public static void FatalInner(this log4net.ILog logger, Exception ex)
{
if (ex != null)
{
logger.Fatal(ex.Message, ex);
logger.FatalInner(ex.InnerException);
}
}
private void ProcessDatasetTask(int id)
{
//https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/task-based-asynchronous-programming?view=netframework-4.5.2
Task t = new Task(() =>
{
try
{
using (IAnalystEdgarDatasetsRepository repository = CreateRepository())
{
EdgarDataset ds = repository.GetDataset(id);
if (ds != null)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////
////BEGIN PROCESS
Stopwatch watch = System.Diagnostics.Stopwatch.StartNew();
log.Info("Datasetid " + id.ToString() + " -- BEGIN dataset process");
//Load Submissions, Tags and Dimensions
EdgarTaskState[] states = LoadSubTagDim(ds, repository);
bool ok = ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadSubTagDim(..): ManageErrors: {1}; Submissions: {2}/{3}; Tags: {4}/{5}; Dimensions: {6}/{7}", id, ok, ds.ProcessedSubmissions, ds.TotalSubmissions, ds.ProcessedTags, ds.TotalTags, ds.ProcessedDimensions, ds.TotalDimensions));
if (!ok || ds.ProcessedSubmissions != ds.TotalSubmissions || ds.ProcessedTags != ds.TotalTags || ds.ProcessedDimensions != ds.TotalDimensions)
{
log.Fatal("Process of sub, tags or dims failed, process can't continue");
return;
}
//Retrieve all tags, submissions and dimensions to fill the relationship
//Load Calculations, Texts and Numbers
log.Info("Datasetid " + id.ToString() + " -- loading all tags for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> tags = tagService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all subs for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> subs = submissionService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all dims for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> dims = dimensionService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- Starting LoadCalTxtNum(...)");
states = LoadCalTxtNum(ds, repository, subs, tags, dims);
ok = ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadCalTxtNum(..): ManageErrors: {1}; Calculations: {2}/{3}; Texts: {4}/{5}; Numbers: {6}/{7}", id, ok, ds.ProcessedCalculations, ds.TotalCalculations, ds.ProcessedTexts, ds.TotalTexts, ds.ProcessedNumbers, ds.TotalNumbers));
if (!ok || ds.ProcessedCalculations != ds.TotalCalculations || ds.ProcessedTexts != ds.TotalTexts || ds.ProcessedNumbers != ds.TotalNumbers)
{
log.Fatal("Process of cal, text or nums failed, process can't continue");
return;
}
log.Info("Datasetid " + id.ToString() + " -- releasing dims");
dims.Clear();
dims = null;
//Load Presentations and Renders
log.Info("Datasetid " + id.ToString() + " -- loading all nums for LoadRenPre(...)");
ConcurrentDictionary <string, int> nums = numService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all txt for LoadRenPre(...)");
ConcurrentDictionary <string, int> txts = textService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- Starting LoadRenPre(...)");
states = LoadRenPre(ds, repository, subs, tags, nums, txts);
ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadRenPre(..): ManageErrors: {1}; Reners: {2}/{3}; Presentations: {4}/{5}", id, ok, ds.ProcessedRenders, ds.TotalRenders, ds.ProcessedPresentations, ds.TotalPresentations));
////END PROCESS
//////////////////////////////////////////////////////////////////////////////////////////////////////////
subs.Clear(); subs = null;
tags.Clear(); tags = null;
nums.Clear(); nums = null;
txts.Clear(); txts = null;
GC.Collect();//force GC
watch.Stop();
TimeSpan ts = watch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
log.Info("Datasetid " + id.ToString() + " -- END dataset process -- time: " + elapsedTime);
}
else
{
log.Fatal("Datasetid " + id.ToString() + " does not exists");
}
}
}
catch (Exception ex)
{
log.Fatal("Datasetid " + id.ToString() + " -- Unexpected error in EdgarDatasetService.Run(" + id.ToString() + "): " + ex.Message, ex);
log.FatalInner(ex.InnerException);
}
});
t.Start();
datasetsInProcess.TryAdd(id, t);
}
