public async Task <int> ImportStaticData(HashSet <string> staticNinList, IPregDataProvider staticdataSource, IPushPregData targetPusher)
{
var person = await staticdataSource.GetNextPerson();
while (staticdataSource.HasMore())
{
if (staticNinList.Contains(person.NIN))
{
if (person.NewNIN == "")
{
person.NewNIN = null;
}
if (person.OldNIN == "")
{
person.OldNIN = null;
}
targetPusher.AddToSaveQueue(new List <Person> {
person
});
var wasAdded = IdControl.TakenAdd(person.NIN);
if (!wasAdded)
{
throw new Exception("TakenAdd feilet. dette skal ikke skje siden input liste er kun unike. Sjekk at database var tom initielt");
}
staticNinList.Remove(person.NIN);
}
person = await staticdataSource.GetNextPerson();
}
Outputter.WriteLine("Done importing static data. Number of object not found in source was " + staticNinList.Count + ". Items are put on save queue. Length:" + targetPusher.QueueLength());
return(staticNinList.Count);
}
private async Task <int> DoIt(int howMany, int numberOfThreads = 2, int bunchSize = 10000)
{
var saveTask = _pusherPregData.Save();
var tasks = StartExporterTasks(_cancellationTokenSource.Token, howMany, numberOfThreads, bunchSize);
var surveillanceTask = Task.Run(async() =>
{
var start = DateTime.Now;
int previousNumberOfCreated = _pusherPregData.NumberOfCreated();
int preivousNumberOfHandled = _pusherPregData.NumberOfWritten();
int handlingSum = 0;
int creatingSum = 0;
int count = 0;
int secondsBetween = 10;
while (!_pusherPregData.IsDone())
{
if (QueuePregPusher.UnexceptedQuit)
{
Outputter.WriteLine("ERROR: QueuePregPusher.UnexceptedQuit=true, which means that the pusher is dead and the whole thing is in fatal state");
return;
}
count++;
int now = _pusherPregData.QueueLength();
int nowNumberOfCreated = _pusherPregData.NumberOfCreated();
int nowNumberOfHandled = _pusherPregData.NumberOfWritten();
handlingSum += (nowNumberOfHandled - preivousNumberOfHandled) / secondsBetween;
creatingSum += (nowNumberOfCreated - previousNumberOfCreated) / secondsBetween;
var handlingRate = handlingSum / count;
var creatingRate = creatingSum / count;
var diff = nowNumberOfCreated - nowNumberOfHandled;
Outputter.WriteLine($"[DB]: Current saveQueueLength is {now}. Number of created is {nowNumberOfCreated} ({creatingRate} p/s), number of handled is {nowNumberOfHandled} ({handlingRate} p/s)");
previousNumberOfCreated = nowNumberOfCreated;
preivousNumberOfHandled = nowNumberOfHandled;
if (now > 5 * bunchSize)
{
_holdOn = true;
Outputter.WriteLine($"[DB]: Stopping creating-tasks to clear queue");
}
else
{
if (_holdOn)
{
Outputter.WriteLine($"[DB]: Starting creating-tasks again");
_holdOn = false;
}
}
if (handlingRate > 0)
{
var timeSuffered = DateTime.Now.Subtract(start).TotalMinutes;
var remaining = howMany - nowNumberOfHandled;
var remainingTimeInSeconds = (1.0 / ((double)handlingRate)) * ((double)remaining);
var eta = DateTime.Now.AddSeconds(remainingTimeInSeconds);
Outputter.WriteLine("ETA: " + eta.ToLongTimeString());
}
if (_cancellationTokenSource.IsCancellationRequested)
{
return;
}
await Task.Delay(secondsBetween * 1000, _cancellationTokenSource.Token);
}
}, _cancellationTokenSource.Token);
await Task.WhenAll(tasks);
Outputter.WriteLine("Constructing threads done. Waiting for save task");
_pusherPregData.DonePushing();
await Task.WhenAll(saveTask, surveillanceTask);
Outputter.WriteLine("Save-tasks done");
return(_pusherPregData.NumberOfCreated());
}