private EdgarTaskState[] LoadCalTxtNum(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> dims)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
List <Task> tasks = new List <Task>();
//process calc file
EdgarTaskState stateCalc = new EdgarTaskState(EdgarDatasetCalculation.FILE_NAME, ds, repo);
states.Add(stateCalc);
calcService.Submissions = subs;
calcService.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting calcService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
calcService.Process(stateCalc, false, EdgarDatasetCalculation.FILE_NAME, "Calculations")) //false --> to avoid to have too many threads
);
//process text file
EdgarTaskState stateText = new EdgarTaskState(EdgarDatasetText.FILE_NAME, ds, repo);
states.Add(stateText);
textService.Dimensions = dims;
textService.Submissions = subs;
textService.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting textService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
textService.Process(stateText, true, EdgarDatasetText.FILE_NAME, "Texts");
for (int i = 0; i < MAX_TRIALS; i++)
{
if (!string.IsNullOrEmpty(stateText.FileNameToReprocess))
{
string filee = stateText.FileNameToReprocess.Split('\\').Last();
textService.Process(stateText, true, stateText.FileNameToReprocess, "Texts");
}
}
}));
//Process num file
EdgarTaskState stateNum = new EdgarTaskState(EdgarDatasetNumber.FILE_NAME, ds, repo);
states.Add(stateNum);
numService.Dimensions = dims;
numService.Submissions = subs;
numService.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting numService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
bool parallel = ConfigurationManager.AppSettings["run_num_in_parallel"] == "true";
numService.Process(stateNum, parallel, EdgarDatasetNumber.FILE_NAME, "Numbers");
}));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
private EdgarTaskState[] LoadSubTagDim(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
EdgarTaskState stateSubs, stateTag, stateDim;
stateSubs = new EdgarTaskState(EdgarDatasetSubmission.FILE_NAME, ds, repo);
stateTag = new EdgarTaskState(EdgarDatasetTag.FILE_NAME, ds, repo);
stateDim = new EdgarTaskState(EdgarDatasetDimension.FILE_NAME, ds, repo);
states.Add(stateSubs);
states.Add(stateTag);
states.Add(stateDim);
IList <Task> tasks = new List <Task>();
log.Info("Datasetid " + ds.Id.ToString() + " -- starting submissionService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
submissionService.Process(stateSubs, false, EdgarDatasetSubmission.FILE_NAME, "Submissions")//false --> to avoid to have too many threads
));
log.Info("Datasetid " + ds.Id.ToString() + " -- starting tagService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
if (ConfigurationManager.AppSettings["run_tag_in_parallel"] == "true")
{
tagService.Process(stateTag, true, EdgarDatasetTag.FILE_NAME, "Tags");
}
else
{
tagService.Process(stateTag, false, EdgarDatasetTag.FILE_NAME, "Tags");
}
}));
log.Info("Datasetid " + ds.Id.ToString() + " -- starting dimensionService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
dimensionService.Process(stateDim, false, EdgarDatasetDimension.FILE_NAME, "Dimensions");
}));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
private EdgarTaskState[] LoadRenPre(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> nums, ConcurrentDictionary <string, int> texts)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
List <Task> tasks = new List <Task>();
EdgarTaskState stateRen = new EdgarTaskState(EdgarDatasetRender.FILE_NAME, ds, repo);
states.Add(stateRen);
EdgarTaskState statePre = new EdgarTaskState(EdgarDatasetPresentation.FILE_NAME, ds, repo);
states.Add(statePre);
tasks.Add(Task.Factory.StartNew(() =>
{
renderingService.Subs = subs;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting renderingService.Process(...)");
//Presentations has a relationship to renders
renderingService.Process(stateRen, true, EdgarDatasetRender.FILE_NAME, "Renders");
presentationService.Subs = subs;
presentationService.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- loading all rens for presentationService.Process(...)");
presentationService.Renders = renderingService.GetAsConcurrent(ds.Id);
presentationService.Nums = nums;
presentationService.Texts = texts;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting presentationService.Process(...)");
if (ConfigurationManager.AppSettings["run_pre_in_parallel"] == "true")
{
presentationService.Process(statePre, true, EdgarDatasetPresentation.FILE_NAME, "Presentations"); //parallel execution
}
else
{
presentationService.Process(statePre, false, EdgarDatasetPresentation.FILE_NAME, "Presentations");//sequential execution
}
}
));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
