public DatabaseChunkPart Process(IDatabaseEngine sourceEngine, string sourceSchemaName, List <QueryDefinition> sourceQueryDefinitions, string sourceConnection, string vendor) { try { Console.WriteLine("DatabaseChunkBuilder"); var part = new DatabaseChunkPart(_chunkId, _createPersonBuilder, "0", 0); var timer = new Stopwatch(); timer.Start(); var result = part.Load(sourceEngine, sourceSchemaName, sourceQueryDefinitions, sourceConnection, vendor); if (result.Value != null) { Logger.Write(_chunkId, LogMessageTypes.Info, result.Key); throw result.Value; } Logger.Write(_chunkId, LogMessageTypes.Info, $"ChunkId={_chunkId} was loaded - {timer.ElapsedMilliseconds} ms | {GC.GetTotalMemory(false) / 1024f / 1024f} Mb"); part.Build(); return(part); } catch (Exception e) { Logger.WriteError(_chunkId, e); throw; } }
//public DatabaseChunkPart Process(IDatabaseEngine sourceEngine, string sourceSchemaName, List<QueryDefinition> sourceQueryDefinitions, OdbcConnection sourceConnection) public DatabaseChunkPart Process(IDatabaseEngine sourceEngine, string sourceSchemaName, List <QueryDefinition> sourceQueryDefinitions, string sourceConnectionString) { try { Console.WriteLine("DatabaseChunkBuilder"); var part = new DatabaseChunkPart(_chunkId, () => new PersonBuilder(), "0", 0); var timer = new Stopwatch(); timer.Start(); //var result = part.Load(sourceEngine, sourceSchemaName, sourceQueryDefinitions, sourceConnection, ""); var result = part.Load(sourceEngine, sourceSchemaName, sourceQueryDefinitions, sourceConnectionString, ""); if (result.Value != null) { //Logger.Write(_chunkId, LogMessageTypes.Info, result.Key); throw result.Value; } part.Build(); return(part); } catch (Exception e) { throw; } }
public void Build(IVocabulary vocabulary) { var saveQueue = new BlockingCollection <DatabaseChunkPart>(); PerformAction(() => { if (Settings.Current.Building.ChunksCount == 0) { Settings.Current.Building.ChunksCount = _chunkController.CreateChunks(); } vocabulary.Fill(false, false); Logger.Write(null, LogMessageTypes.Info, $"==================== Conversion to CDM was started ===================="); var save = Task.Run(() => { while (!saveQueue.IsCompleted) { DatabaseChunkPart data = null; try { data = saveQueue.Take(); } catch (InvalidOperationException) { } if (data != null) { var timer = new Stopwatch(); timer.Start(); data.Save(Settings.Current.Building.Cdm, Settings.Current.Building.DestinationConnectionString, Settings.Current.Building.DestinationEngine, Settings.Current.Building.SourceSchema, Settings.Current.Building.CdmSchema); Settings.Current.Building.CompletedChunkIds.Add(data.ChunkData.ChunkId); timer.Stop(); Logger.Write(data.ChunkData.ChunkId, LogMessageTypes.Info, $"ChunkId={data.ChunkData.ChunkId} was saved - {timer.ElapsedMilliseconds} ms | {GC.GetTotalMemory(false) / 1024f / 1024f} Mb"); } if (CurrentState != BuilderState.Running) { break; } } CurrentState = BuilderState.Stopped; }); Parallel.For(0, Settings.Current.Building.ChunksCount, new ParallelOptions { MaxDegreeOfParallelism = Settings.Current.DegreeOfParallelism }, (chunkId, state) => { if (CurrentState != BuilderState.Running) { state.Break(); } if (!Settings.Current.Building.CompletedChunkIds.Contains(chunkId)) { var chunk = new DatabaseChunkBuilder(chunkId, CreatePersonBuilder); //using (var connection = // new OdbcConnection(Settings.Current.Building.SourceConnectionString)) { //connection.Open(); saveQueue.Add(chunk.Process(Settings.Current.Building.SourceEngine, Settings.Current.Building.SourceSchema, Settings.Current.Building.SourceQueryDefinitions, Settings.Current.Building.SourceConnectionString, Settings.Current.Building.Vendor)); } Settings.Current.Save(false); while (saveQueue.Count > 0) { Thread.Sleep(1000); } } }); saveQueue.CompleteAdding(); save.Wait(); }); }
public void Build() { if (_taskQueue.Aborted) { return; } var saveQueue = new BlockingCollection <DatabaseChunkPart>(); var chunkController = new ChunkController(_settings); var chunksCount = chunkController.CreateChunks(); _vocabulary.Fill(false, false); _logHub.Clients.All.SendAsync("Log", string.Format("{0}| {1}", DateTime.Now, $"Conversion to CDM was started")) .Wait(); var step = 100.0 / chunksCount; var total = 0.0; _logHub.Clients.All.SendAsync("Progress", total.ToString()).Wait(); var save = Task.Run(() => { while (!saveQueue.IsCompleted) { DatabaseChunkPart data = null; try { data = saveQueue.Take(); } catch (InvalidOperationException) { } if (data != null) { var timer = new Stopwatch(); timer.Start(); data.Save(_settings.Cdm, _settings.DestinationConnectionString, _settings.DestinationEngine, _settings.ConversionSettings.SourceSchema, _settings.ConversionSettings.DestinationSchema); timer.Stop(); _logHub.Clients.All.SendAsync("Log", string.Format("{0}| {1}", DateTime.Now, $"ChunkId={data.ChunkData.ChunkId} was saved - {timer.ElapsedMilliseconds} ms | {GC.GetTotalMemory(false) / 1024f / 1024f} Mb")) .Wait(); total += step; } _logHub.Clients.All.SendAsync("Progress", Convert.ToInt32(total).ToString()).Wait(); } }); var degreeOfParallelism = 1; Parallel.For(0, chunksCount, new ParallelOptions { MaxDegreeOfParallelism = degreeOfParallelism }, (chunkId, state) => { if (_taskQueue.Aborted) { return; } var chunk = new DatabaseChunkBuilder(chunkId); using var connection = new OdbcConnection(_settings.SourceConnectionString); connection.Open(); saveQueue.Add(chunk.Process(_settings.SourceEngine, _settings.ConversionSettings.SourceSchema, _settings.SourceQueryDefinitions, connection)); }); saveQueue.CompleteAdding(); save.Wait(); }