public EdgarDataset GetDataset(int id) { using (IEdgarDatasetsRepository repository = CreateRepository()) { EdgarDataset ds = repository.GetDataset(id); return(ds); } }
private void ProcessDatasetTask(int id) { //https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/task-based-asynchronous-programming?view=netframework-4.5.2 Task t = new Task(() => { try { using (IEdgarDatasetsRepository repository = CreateRepository()) { EdgarDataset ds = repository.GetDataset(id); if (ds != null) { ////////////////////////////////////////////////////////////////////////////////////////////////////////// ////BEGIN PROCESS Stopwatch watch = System.Diagnostics.Stopwatch.StartNew(); log.Info("Datasetid " + id.ToString() + " -- BEGIN dataset process"); //Load Submissions, Tags and Dimensions EdgarTaskState[] states = LoadSubTagDim(ds, repository); bool ok = ManageErrors(states); log.Info(string.Format("Datasetid {0} -- Variables after LoadSubTagDim(..): ManageErrors: {1}; Submissions: {2}/{3}; Tags: {4}/{5}; Dimensions: {6}/{7}", id, ok, ds.ProcessedSubmissions, ds.TotalSubmissions, ds.ProcessedTags, ds.TotalTags, ds.ProcessedDimensions, ds.TotalDimensions)); if (!ok || ds.ProcessedSubmissions != ds.TotalSubmissions || ds.ProcessedTags != ds.TotalTags || ds.ProcessedDimensions != ds.TotalDimensions) { log.Fatal("Process of sub, tags or dims failed, process can't continue"); return; } //Retrieve all tags, submissions and dimensions to fill the relationship //Load Calculations, Texts and Numbers log.Info("Datasetid " + id.ToString() + " -- loading all tags for LoadCalTxtNum(...)"); ConcurrentDictionary <string, int> tags = tagParser.GetAsConcurrent(id); log.Info("Datasetid " + id.ToString() + " -- loading all subs for LoadCalTxtNum(...)"); ConcurrentDictionary <string, int> subs = submissionParser.GetAsConcurrent(id); log.Info("Datasetid " + id.ToString() + " -- loading all dims for LoadCalTxtNum(...)"); ConcurrentDictionary <string, int> dims = dimensionParser.GetAsConcurrent(id); log.Info("Datasetid " + id.ToString() + " -- Starting LoadCalTxtNum(...)"); states = LoadCalTxtNum(ds, repository, subs, tags, dims); ok = ManageErrors(states); log.Info(string.Format("Datasetid {0} -- Variables after LoadCalTxtNum(..): ManageErrors: {1}; Calculations: {2}/{3}; Texts: {4}/{5}; Numbers: {6}/{7}", id, ok, ds.ProcessedCalculations, ds.TotalCalculations, ds.ProcessedTexts, ds.TotalTexts, ds.ProcessedNumbers, ds.TotalNumbers)); if (!ok || ds.ProcessedCalculations != ds.TotalCalculations || ds.ProcessedTexts != ds.TotalTexts || ds.ProcessedNumbers != ds.TotalNumbers) { log.Fatal("Process of cal, text or nums failed, process can't continue"); return; } log.Info("Datasetid " + id.ToString() + " -- releasing dims"); dims.Clear(); dims = null; //Load Presentations and Renders log.Info("Datasetid " + id.ToString() + " -- loading all nums for LoadRenPre(...)"); ConcurrentDictionary <string, int> nums = numParser.GetAsConcurrent(id); log.Info("Datasetid " + id.ToString() + " -- loading all txt for LoadRenPre(...)"); ConcurrentDictionary <string, int> txts = textParser.GetAsConcurrent(id); log.Info("Datasetid " + id.ToString() + " -- Starting LoadRenPre(...)"); states = LoadRenPre(ds, repository, subs, tags, nums, txts); ManageErrors(states); log.Info(string.Format("Datasetid {0} -- Variables after LoadRenPre(..): ManageErrors: {1}; Reners: {2}/{3}; Presentations: {4}/{5}", id, ok, ds.ProcessedRenders, ds.TotalRenders, ds.ProcessedPresentations, ds.TotalPresentations)); ////END PROCESS ////////////////////////////////////////////////////////////////////////////////////////////////////////// subs.Clear(); subs = null; tags.Clear(); tags = null; nums.Clear(); nums = null; txts.Clear(); txts = null; GC.Collect();//force GC watch.Stop(); TimeSpan ts = watch.Elapsed; string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); log.Info("Datasetid " + id.ToString() + " -- END dataset process -- time: " + elapsedTime); } else { log.Fatal("Datasetid " + id.ToString() + " does not exists"); } } } catch (Exception ex) { log.Fatal("Datasetid " + id.ToString() + " -- Unexpected error in EdgarDatasetService.Run(" + id.ToString() + "): " + ex.Message, ex); log.FatalInner(ex.InnerException); } }); t.Start(); datasetsInProcess.TryAdd(id, t); }