private ProcessingScope(ProcessingScope parent) { Depth = parent != null ? parent.Depth + 1 : 0; Parent = parent; FieldLookup = parent != null ? parent.FieldLookup : null; _indices = new int[Depth + 1]; }
public void Run() { if (Status != JobStatus.Pending) { throw new InvalidOperationException("Job is not pending"); } try { Directory.CreateDirectory(TempDirectory); File.WriteAllText(Path.Combine(TempDirectory, "specification.json"), Specification.ToString()); SetStatus(JobStatus.Preparing); Specification.Initialize(this); var source = Specification.CreateDataSource(); PostProcessors = Specification.CreatePostProcessors().ToArray(); var jobDirectory = TempDirectory; //The processors will consume data from this collection; var items = new BlockingCollection <object>(ExecutionSettings.DataSourceBufferSize); //Create the processors var processors = Enumerable.Range(0, ExecutionSettings.ProcessingThreads).Select(i => { IItemFieldLookup lookup = null; try { lookup = new ItemDatabaseFieldLookup(Database.GetDatabase(ExecutionSettings.DatabaseName), Specification.DefaultLanguage, ExecutionSettings.FieldCacheSize); } catch (Exception ex) { Log.Error("Error initailizing item field lookup", ex, this); } var proc = new DataProcessor(Specification.CreateRootMapper()) { BatchSize = ExecutionSettings.BatchSize, FieldLookup = lookup }; var exporter = Specification.CreateExporter(jobDirectory) as CsvExporter; //Move PartititionPrefix so this cast isn't necessary if (exporter == null) { exporter = new CsvExporter(jobDirectory); } exporter.PartitionPrefix = "~" + i + "_"; exporter.KeepOutput = true; //Don't delete the job's main directory proc.BatchWriter = new TableDataBatchWriter(exporter) { SyncLock = this, MaximumSize = ExecutionSettings.SizeLimit }; proc.Initialize(); return(proc); }).ToArray(); var hasUpdates = false; //Allow post processors to validate their conditions (if any). This allows the job to fail before data is processed //Allow post processors to filter data source for updates foreach (var pp in PostProcessors) { pp.Validate(processors[0].Tables, Specification); if (pp.UpdateDataSource(processors[0].Tables, source)) { if (hasUpdates) { throw new InvalidOperationException("Only one post processor can update the data source"); } hasUpdates = true; } } EstimatedClientItemCount = source.Count; //Start the processors var processingThreads = processors.Select(p => { var t = new Thread(() => { try { p.Process(items.GetConsumingEnumerable()); } catch (Exception ex) { LastException = ex; SetStatus(JobStatus.Failed, ex.Message); } }); t.Start(); return(t); }).ToArray(); if (Status == JobStatus.Failed) { throw LastException; } SetStatus(JobStatus.Running); source.ItemLoaded += (sender, args) => { if (StatusUpdateFrequency <= 0 || ItemsProcessed % StatusUpdateFrequency == 0) { OnProgress(); } ItemsProcessed = args; RowsCreated = processors.Sum(p => p.RowsCreated); }; //Add items to the collection that the processors consume foreach (var item in source) { if (Status != JobStatus.Running) { break; } if (processors.Any(p => p.BatchWriter.End)) { break; } items.Add(item); } items.CompleteAdding(); //Wait for processors to finish foreach (var p in processingThreads) { p.Join(); } RowsCreated = processors.Sum(p => p.RowsCreated); if (Status == JobStatus.Running) { //Now we know how many items we got for sure. Update progress to 100% EstimatedClientItemCount = ItemsProcessed; SetStatus(JobStatus.Merging); using (var csvWriter = Specification.CreateExporter(TempDirectory)) { var tables = MergedTableData.FromTableSets(processors.Select(p => p.Tables)).ToArray(); var w = csvWriter as CsvExporter; if (w == null || w.KeepOutput) { tables = csvWriter.Export(tables).ToArray(); } File.WriteAllText(Path.Combine(jobDirectory, "schema.json"), tables.Select(t => t.Schema).Serialize()); foreach (var postProcessor in PostProcessors) { CurrentPostProcessor = postProcessor; SetStatus(JobStatus.PostProcessing, postProcessor.Name); postProcessor.Process(jobDirectory, tables, Specification); } CurrentPostProcessor = null; foreach (var proc in processors) { SizeLimitExceeded = SizeLimitExceeded || proc.BatchWriter.End; proc.BatchWriter.Dispose(); } SetStatus(JobStatus.Completing); } SetStatus(JobStatus.Completed); } } catch (Exception ex) { Log.Error("Job failed", ex, this); LastException = ex; SetStatus(JobStatus.Failed, ex.ToString()); } try { OnJobEnded(); } catch (Exception ex) { Log.Error("Exception occured after job ended", ex, this); LastException = ex; } EndDate = DateTime.Now; try { if (Status == JobStatus.Canceled || Status == JobStatus.Failed) { Delete(); } } catch (Exception ex) { Log.Error("Exception occured while deleting job", ex, this); } }