public void Build(IVocabulary vocabulary)
{
var saveQueue = new BlockingCollection <DatabaseChunkPart>();
PerformAction(() =>
{
if (Settings.Current.Building.ChunksCount == 0)
{
Settings.Current.Building.ChunksCount = _chunkController.CreateChunks();
}
vocabulary.Fill(false, false);
Logger.Write(null, LogMessageTypes.Info,
$"==================== Conversion to CDM was started ====================");
var save = Task.Run(() =>
{
while (!saveQueue.IsCompleted)
{
DatabaseChunkPart data = null;
try
{
data = saveQueue.Take();
}
catch (InvalidOperationException)
{
}
if (data != null)
{
var timer = new Stopwatch();
timer.Start();
data.Save(Settings.Current.Building.Cdm,
Settings.Current.Building.DestinationConnectionString,
Settings.Current.Building.DestinationEngine,
Settings.Current.Building.SourceSchema,
Settings.Current.Building.CdmSchema);
Settings.Current.Building.CompletedChunkIds.Add(data.ChunkData.ChunkId);
timer.Stop();
Logger.Write(data.ChunkData.ChunkId, LogMessageTypes.Info,
$"ChunkId={data.ChunkData.ChunkId} was saved - {timer.ElapsedMilliseconds} ms | {GC.GetTotalMemory(false) / 1024f / 1024f} Mb");
}
if (CurrentState != BuilderState.Running)
{
break;
}
}
CurrentState = BuilderState.Stopped;
});
Parallel.For(0, Settings.Current.Building.ChunksCount,
new ParallelOptions {
MaxDegreeOfParallelism = Settings.Current.DegreeOfParallelism
}, (chunkId, state) =>
{
if (CurrentState != BuilderState.Running)
{
state.Break();
}
if (!Settings.Current.Building.CompletedChunkIds.Contains(chunkId))
{
var chunk = new DatabaseChunkBuilder(chunkId, CreatePersonBuilder);
//using (var connection =
// new OdbcConnection(Settings.Current.Building.SourceConnectionString))
{
//connection.Open();
saveQueue.Add(chunk.Process(Settings.Current.Building.SourceEngine,
Settings.Current.Building.SourceSchema,
Settings.Current.Building.SourceQueryDefinitions,
Settings.Current.Building.SourceConnectionString,
Settings.Current.Building.Vendor));
}
Settings.Current.Save(false);
while (saveQueue.Count > 0)
{
Thread.Sleep(1000);
}
}
});
saveQueue.CompleteAdding();
save.Wait();
});
}
public void Build()
{
if (_taskQueue.Aborted)
{
return;
}
var saveQueue = new BlockingCollection <DatabaseChunkPart>();
var chunkController = new ChunkController(_settings);
var chunksCount = chunkController.CreateChunks();
_vocabulary.Fill(false, false);
_logHub.Clients.All.SendAsync("Log", string.Format("{0}| {1}",
DateTime.Now,
$"Conversion to CDM was started"))
.Wait();
var step = 100.0 / chunksCount;
var total = 0.0;
_logHub.Clients.All.SendAsync("Progress", total.ToString()).Wait();
var save = Task.Run(() =>
{
while (!saveQueue.IsCompleted)
{
DatabaseChunkPart data = null;
try
{
data = saveQueue.Take();
}
catch (InvalidOperationException)
{
}
if (data != null)
{
var timer = new Stopwatch();
timer.Start();
data.Save(_settings.Cdm,
_settings.DestinationConnectionString,
_settings.DestinationEngine,
_settings.ConversionSettings.SourceSchema,
_settings.ConversionSettings.DestinationSchema);
timer.Stop();
_logHub.Clients.All.SendAsync("Log", string.Format("{0}| {1}",
DateTime.Now,
$"ChunkId={data.ChunkData.ChunkId} was saved - {timer.ElapsedMilliseconds} ms | {GC.GetTotalMemory(false) / 1024f / 1024f} Mb"))
.Wait();
total += step;
}
_logHub.Clients.All.SendAsync("Progress", Convert.ToInt32(total).ToString()).Wait();
}
});
var degreeOfParallelism = 1;
Parallel.For(0, chunksCount,
new ParallelOptions {
MaxDegreeOfParallelism = degreeOfParallelism
}, (chunkId, state) =>
{
if (_taskQueue.Aborted)
{
return;
}
var chunk = new DatabaseChunkBuilder(chunkId);
using var connection =
new OdbcConnection(_settings.SourceConnectionString);
connection.Open();
saveQueue.Add(chunk.Process(_settings.SourceEngine,
_settings.ConversionSettings.SourceSchema,
_settings.SourceQueryDefinitions,
connection));
});
saveQueue.CompleteAdding();
save.Wait();
}