Пакетная вставка Doctrine использует 2 Гб оперативной памяти

Я пытаюсь вставить приблизительно 200 000 записей с помощью Symfony2 и Doctrine с помощью Doctrine Fixture Bundle. Я использую flush and clear, но в конце сценарий использует 1,8 ГБ оперативной памяти.

Это класс, который загружает объекты SmartMeter в базу данных:

<?php

namespace HTEC\SmartMeteringAPIBundle\DataFixtures\ORM;

use Doctrine\Common\DataFixtures\AbstractFixture;
use Doctrine\Common\DataFixtures\FixtureInterface;
use Doctrine\Common\Persistence\ObjectManager;
use Doctrine\Common\DataFixtures\OrderedFixtureInterface;

use HTEC\SmartMeteringAPIBundle\Entity\SmartMeter;

use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;

class LoadSmartMeterData extends AbstractFixture implements FixtureInterface, ContainerAwareInterface, OrderedFixtureInterface
{

static $NUMBER_OF_SMART_METERS = 0;
static $MAX_NUM_OF_SM_PER_CONC = 500;

/**
* @var ContainerInterface
*/
private $container;


public function getOrder()
{
return 10; // the order in which fixtures will be loaded
}

/**
* @inheritDoc
*/
public function setContainer(ContainerInterface $container = null)
{
$this->container = $container;
}

/**
* {@inheritDoc}
*/
public function load(ObjectManager $manager)
{

$numberOfUsers = LoadUserData::$NUMBER_OF_USERS;
$numberOfConcentrators = LoadConcentratorData::$NUMBER_OF_CONCENTRATORS;
$numberOfTariffs = LoadTariffData::$NUMBER_OF_TARIFFS;

$numberOfSmartMetersPerConcentrator = 0;

$smartMeter = null;
$concentrator = null;
$user = null;
$tariff = null;

$feeders = array();
$numberOfFeeders = 0;
$feedersRandomKey = array();

$isIpAddressDynamic = false;

$numberOfTransformerFeeders = 0;

$smartMeterType = 0;
$smartMeterProtocol = 0;
$modemType = 0;

$numberOfSmartMeters = 0;


echo "\n\nCreating Smart Meters. This could take a couple of minutes and could take approximately 2 GB of RAM \n\n...";

$startTime = time();

$smartMeterCount = 0;

$lastChunkNumber = 0;
$currentChunkNumber = 0;

$lastSmartMeterNumber = 0;
$smartMetersClearedCurrently = 0;

$concentratorSmartMeters = null;

for($i = 0; $i < LoadConcentratorData::$NUMBER_OF_CONCENTRATORS; $i++)
{

$concentrator = $manager->getRepository('SMAPIBundle:Concentrator')->find($i+1);

$numberOfSmartMetersPerConcentrator = rand(1, self::$MAX_NUM_OF_SM_PER_CONC);

$numberOfSmartMeters += $numberOfSmartMetersPerConcentrator;

for($c = 0; $c < $numberOfSmartMetersPerConcentrator; $c++)
{
$smartMeter = new SmartMeter();
$smartMeter->setSerialNumber(++$smartMeterCount);
$smartMeter->setConcentrator($concentrator);

$smartMeterType = rand(1,3);

switch($smartMeterType)
{
case 1:
$smartMeter->setType('DIRECT');
break;

case 2:
$smartMeter->setType('HALF');
break;

case 3:
$smartMeter->setType('INDIRECT');
break;
}

$user = $manager->getRepository('SMAPIBundle:User')->find(rand(1, $numberOfUsers));
$smartMeter->setCreatedBy($user);

$numberOfTransformerFeeders = $concentrator->getTransformerFeeders()->count();
$feeders = $concentrator->getTransformerFeeders()->toArray();

if($numberOfTransformerFeeders > 0)
{

$feedersRandomKey = array_rand($feeders, 1);

if(isset($feeders[$feedersRandomKey]))
{
$smartMeter->setFeeder($feeders[$feedersRandomKey]);
}
}

$smartMeter->setStatus(rand(0,2));


$tariff = $manager->getRepository('SMAPIBundle:Tariff')->find(rand(1, $numberOfTariffs));
$smartMeter->setTariff($tariff);

$smartMeterProtocol = rand(1,3);

switch($smartMeterProtocol)
{
case 1:
$smartMeter->setProtocol('DLMS');
break;

case 2:
$smartMeter->setProtocol('EURIDIS');
break;

case 3:
$smartMeter->setProtocol('IEC');
break;
}


$smartMeter->setModemSerialNumber(rand(1, 9000000000));

$smartMeter->setManufacture('MAN: ' . ($i + $c));

$modemType = rand(1,2);

if($modemType === 1)
{
$smartMeter->setModemType('PLC');
$smartMeter->setModemIndex(rand(1, 512));
}
else
{

$smartMeter->setModemType('GPRS');
$isIpAddressDynamic = rand(0,1);

if($isIpAddressDynamic === 1)
{
$smartMeter->setModemIpAddress(rand(1000000, 2000000));
}
else
{
$smartMeter->setModemPhoneNumber($this->getRandomIpAddressV4());
}
}


if(rand(0,1) === 1)
{
$smartMeter->setModemRepeaterNumber(rand(10000000, 90000000));
}

$manager->persist($smartMeter);

} // end of FOR numberOfSmartMetersPerConcentrator

// flush smart meters
$manager->flush();
$manager->clear();

$currentChunkNumber = ceil($smartMeterCount / 5000);

if($smartMeterCount > 5000 && $currentChunkNumber > $lastChunkNumber)
{
$lastChunkNumber = $currentChunkNumber;



if($lastSmartMeterNumber > 0)
{
$smartMetersClearedCurrently = $smartMeterCount - $lastSmartMeterNumber;
}
else
{
$smartMetersClearedCurrently = $smartMeterCount;
}


echo "\n\nFlushing and clearing " . number_format($smartMetersClearedCurrently, 0, ',', '.') . " Smart Meters.\nTotal memory used after flush and clear: " . number_format(((memory_get_usage(true) / 1024) / 1024), 2, ',', '.') . " Megabytes\n\n...";

$lastSmartMeterNumber = $smartMeterCount;
}
}// end for NUMBER_OF_CONCENTRATORS

$manager->flush();
$manager->clear();

unset($concentrator);
unset($smartMeter);
unset($tariff);

echo "\n\n-------------------------------------------\n\n";
echo "\n\nTotal memory used after final flush and clear of Smart Meters: " . number_format(((memory_get_usage(true) / 1024) / 1024), 2, ',', '.') . " Megabytes\n\n...";

echo "\n\nCreating Smart Meters complete. Created " . number_format($numberOfSmartMeters, 0, ',', '.') . " Smart Meters.\n\n";

$durationSeconds = (time() - $startTime);
$durationMinutes = $durationSeconds / 60;

$secondsRemainder = $durationSeconds % 60;

echo "\n\nTotal duration time: " . ceil($durationMinutes) . " minutes and " . $secondsRemainder . " seconds.\n\n\n";

self::$NUMBER_OF_SMART_METERS = $numberOfSmartMeters;

}

public function getRandomIpAddressV4()
{
return rand(1, 255) . '.' . rand(0, 255) . '.' . rand(0, 255) . '.' . rand(0, 255);
}
}

Приблизительно на 5000 записей я звоню ясно и ясно, но кажется, что память не высвобождается.

Есть ли рекомендации о том, как запретить Doctrine использовать слишком много оперативной памяти во время пакетных задач?

-1

Решение

Если это команда, попробуйте запустить ее с параметром --no-debug, Или вы можете отключить регистратор, позвонив $manager->getConnection()->getConfiguration()->setSQLLogger(null); в начале вашего loadфункция.

В любом случае отключение регистратора экономит довольно много памяти во время пакетных задач Doctrine.

3

Другие решения

Других решений пока нет …