public void Add(EdgarDataset dataset, EdgarDatasetTag tag)
{
SqlParameter dsid = new SqlParameter("@DataSetId", dataset.Id);
SqlParameter tagparam = new SqlParameter("@Tag", tag.Tag);
SqlParameter version = new SqlParameter("@Version", tag.Version);
SqlParameter custom = new SqlParameter("@Custom", tag.Custom);
SqlParameter abstracto = new SqlParameter("@Abstract", tag.Abstract);
SqlParameter datatype = new SqlParameter("@Datatype", tag.ValueType);
if (tag.ValueType == null)
{
datatype.Value = DBNull.Value;
}
SqlParameter LabelText = new SqlParameter("@LabelText", tag.LabelText);
if (tag.LabelText == null)
{
LabelText.Value = DBNull.Value;
}
SqlParameter Documentation = new SqlParameter("@Documentation", tag.Documentation);
if (tag.Documentation == null)
{
Documentation.Value = DBNull.Value;
}
SqlParameter LineNumber = new SqlParameter("@LineNumber", tag.LineNumber);
Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETTAGS_INSERT " +
"@DataSetId, @tag,@version,@custom,@Abstract,@Datatype,@LabelText,@Documentation,@LineNumber",
dsid, tagparam, version, custom, abstracto, datatype, LabelText, Documentation, LineNumber);
}
private void LoadDatasets(EdgarDataset ds, BindingSource bs)
{
IList <EdgarDataset> datasets = new List <EdgarDataset>();
datasets.Add(ds);
LoadDatasets(datasets, bindingSourceDatasetInProcess);
}
public override void Add(IAnalystEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetNumber file)
{
file.SubmissionId = Submissions[file.ADSH];
file.TagId = Tags[file.TagCompoundKey];
file.DimensionId = Dimensions[file.DimensionStr];
repo.Add(dataset, file);
}
public EdgarDataset GetDataset(int id)
{
using (IAnalystEdgarDatasetsRepository repository = CreateRepository())
{
EdgarDataset ds = repository.GetDataset(id);
return(ds);
}
}
public void Add(EdgarDataset dataset, EdgarDatasetPresentation pre)
{
SqlParameter ReportNumber = new SqlParameter("@ReportNumber", pre.ReportNumber);
SqlParameter Line = new SqlParameter("@Line", pre.Line);
SqlParameter FinancialStatement = new SqlParameter("@FinancialStatement", pre.FinancialStatement);
SqlParameter Inpth = new SqlParameter("@Inpth", pre.Inpth);
SqlParameter RenderFile = new SqlParameter("@RenderFile", pre.RenderFileStr);
SqlParameter PreferredLabelXBRLLinkRole = new SqlParameter("@PreferredLabelXBRLLinkRole", pre.PreferredLabelXBRLLinkRole);
SqlParameter PreferredLabel = new SqlParameter("@PreferredLabel", pre.PreferredLabel);
SqlParameter Negating = new SqlParameter("@Negating", pre.Negating);
SqlParameter LineNumber = new SqlParameter("@LineNumber", pre.LineNumber);
SqlParameter DataSetId = new SqlParameter("@DataSetId", dataset.Id);
SqlParameter Submission_Id = new SqlParameter("@Submission_Id", pre.SubmissionId);
SqlParameter Tag_Id = new SqlParameter("@Tag_Id", pre.TagId);
SqlParameter Number_Id;
if (pre.NumberId > 0)
{
Number_Id = new SqlParameter("@Number_Id", pre.NumberId);
}
else
{
Number_Id = new SqlParameter("@Number_Id", DBNull.Value);
}
SqlParameter Text_Id;
if (pre.TextId > 0)
{
Text_Id = new SqlParameter("@Text_Id", pre.TextId);
}
else
{
Text_Id = new SqlParameter("@Text_Id", DBNull.Value);
}
SqlParameter Render_Id;
if (pre.RenderId <= 0)
{
Render_Id = new SqlParameter("@Render_Id", DBNull.Value);
}
else
{
Render_Id = new SqlParameter("@Render_Id", pre.RenderId);
}
SqlParameter adsh_tag_version = new SqlParameter("@adsh_tag_version", pre.ADSH_Tag_Version);
if (string.IsNullOrEmpty(pre.ADSH_Tag_Version))
{
adsh_tag_version.Value = DBNull.Value;
}
Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETPRESENTATIONS_INSERT " +
"@ReportNumber, @Line, @FinancialStatement, @Inpth,@RenderFile, @PreferredLabelXBRLLinkRole, @PreferredLabel, @Negating,@LineNumber, @DataSetId, @Submission_Id, @Tag_Id, @Number_Id, @Text_Id, @Render_Id, @adsh_tag_version",
ReportNumber, Line, FinancialStatement, Inpth, PreferredLabelXBRLLinkRole, PreferredLabel, Negating, LineNumber, DataSetId, Submission_Id, Tag_Id, Number_Id, Text_Id, Render_Id, adsh_tag_version);
}
private EdgarTaskState[] LoadCalTxtNum(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> dims)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
List <Task> tasks = new List <Task>();
//process calc file
EdgarTaskState stateCalc = new EdgarTaskState(EdgarDatasetCalculation.FILE_NAME, ds, repo);
states.Add(stateCalc);
calcService.Submissions = subs;
calcService.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting calcService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
calcService.Process(stateCalc, false, EdgarDatasetCalculation.FILE_NAME, "Calculations")) //false --> to avoid to have too many threads
);
//process text file
EdgarTaskState stateText = new EdgarTaskState(EdgarDatasetText.FILE_NAME, ds, repo);
states.Add(stateText);
textService.Dimensions = dims;
textService.Submissions = subs;
textService.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting textService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
textService.Process(stateText, true, EdgarDatasetText.FILE_NAME, "Texts");
for (int i = 0; i < MAX_TRIALS; i++)
{
if (!string.IsNullOrEmpty(stateText.FileNameToReprocess))
{
string filee = stateText.FileNameToReprocess.Split('\\').Last();
textService.Process(stateText, true, stateText.FileNameToReprocess, "Texts");
}
}
}));
//Process num file
EdgarTaskState stateNum = new EdgarTaskState(EdgarDatasetNumber.FILE_NAME, ds, repo);
states.Add(stateNum);
numService.Dimensions = dims;
numService.Submissions = subs;
numService.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting numService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
bool parallel = ConfigurationManager.AppSettings["run_num_in_parallel"] == "true";
numService.Process(stateNum, parallel, EdgarDatasetNumber.FILE_NAME, "Numbers");
}));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
public void Add(EdgarDataset dataset, EdgarDatasetRender ren)
{
SqlParameter Report = new SqlParameter("@Report", ren.Report);
SqlParameter RenderFile = new SqlParameter("@RenderFile", ren.RenderFileStr);
SqlParameter MenuCategory = new SqlParameter("@MenuCategory", ren.MenuCategory);
SqlParameter ShortName = new SqlParameter("@ShortName", ren.ShortName);
SqlParameter LongName = new SqlParameter("@LongName", ren.LongName);
SqlParameter Roleuri;
if (string.IsNullOrEmpty(ren.RoleURI))
{
Roleuri = new SqlParameter("@Roleuri", DBNull.Value);
}
else
{
Roleuri = new SqlParameter("@Roleuri", ren.RoleURI);
}
SqlParameter ParentRoleuri;
if (string.IsNullOrEmpty(ren.ParentRoleURI))
{
ParentRoleuri = new SqlParameter("@ParentRoleuri", DBNull.Value);
}
else
{
ParentRoleuri = new SqlParameter("@ParentRoleuri", ren.ParentRoleURI);
}
SqlParameter ParentReport = new SqlParameter("@ParentReport", ren.ParentReport);
if (ren.ParentReport == null)
{
ParentReport.Value = DBNull.Value;
}
SqlParameter UltimateParentReport = new SqlParameter("@UltimateParentReport", ren.UltimateParentReport);
if (ren.UltimateParentReport == null)
{
UltimateParentReport.Value = DBNull.Value;
}
SqlParameter Submission_Id = new SqlParameter("@Submission_Id", ren.SubmissionId);
SqlParameter DataSetId = new SqlParameter("@DataSetId", dataset.Id);
SqlParameter lineNumber = new SqlParameter("@LineNumber", ren.LineNumber);
Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETRENDERS_INSERT " +
"@Report, @RenderFile, @MenuCategory, @ShortName, @LongName, @Roleuri, @ParentRoleuri, @ParentReport, @UltimateParentReport, @Submission_Id, @DataSetId, @LineNumber",
Report, MenuCategory, ShortName, LongName, Roleuri, ParentRoleuri, ParentReport, UltimateParentReport, Submission_Id, DataSetId, lineNumber
);
}
public void Add(EdgarDataset dataset, EdgarDatasetSubmission sub)
{
SqlParameter ADSH = new SqlParameter("@ADSH", sub.ADSH);
SqlParameter Period = new SqlParameter("@Period", sub.Period);
SqlParameter Detail = new SqlParameter("@Detail", sub.Detail);
SqlParameter XBRLInstance = new SqlParameter("@XBRLInstance", sub.XBRLInstance);
if (sub.XBRLInstance == null)
{
XBRLInstance.Value = DBNull.Value;
}
SqlParameter NumberOfCIKs = new SqlParameter("@NumberOfCIKs", sub.NumberOfCIKs);
SqlParameter AdditionalCIKs = new SqlParameter("@AdditionalCIKs", sub.AdditionalCIKs);
if (sub.AdditionalCIKs == null)
{
AdditionalCIKs.Value = DBNull.Value;
}
SqlParameter PublicFloatUSD = new SqlParameter("@PublicFloatUSD", sub.PublicFloatUSD);
if (sub.PublicFloatUSD == null)
{
PublicFloatUSD.Value = DBNull.Value;
}
SqlParameter FloatDate = new SqlParameter("@FloatDate", sub.FloatDate);
if (sub.FloatDate == null)
{
FloatDate.Value = DBNull.Value;
}
SqlParameter FloatAxis = new SqlParameter("@FloatAxis", sub.FloatAxis);
if (sub.FloatAxis == null)
{
FloatAxis.Value = DBNull.Value;
}
SqlParameter FloatMems = new SqlParameter("@FloatMems", sub.FloatMems);
if (sub.FloatMems == null)
{
FloatMems.Value = DBNull.Value;
}
SqlParameter lineNumber = new SqlParameter("@LineNumber", sub.LineNumber);
SqlParameter FormId = new SqlParameter("@Form_id", sub.Form.Id);
SqlParameter Registrant_Id = new SqlParameter("@Registrant_Id", sub.Registrant.Id);
SqlParameter EdgarDataset_Id = new SqlParameter("@EdgarDataset_Id", dataset.Id);
Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASSUBMISSIONS_INSERT " +
"@ADSH, @Period, @Detail, @XBRLInstance, @NumberOfCIKs, @AdditionalCIKs, @PublicFloatUSD, @FloatDate, @FloatAxis, @FloatMems,@LineNumber, @Form_Id, @Registrant_Id, @EdgarDataset_Id",
ADSH, Period, Detail, XBRLInstance, NumberOfCIKs, AdditionalCIKs, PublicFloatUSD, FloatDate, FloatAxis, FloatMems, lineNumber, FormId, Registrant_Id, EdgarDataset_Id);
}
public void Add(EdgarDataset dataset, EdgarDatasetCalculation file)
{
SqlParameter LineNumber = new SqlParameter("@LineNumber", file.LineNumber);
SqlParameter SequentialNumberForGrouping = new SqlParameter("@SequentialNumberForGrouping", file.SequentialNumberForGrouping);
SqlParameter SequentialNumberForArc = new SqlParameter("@SequentialNumberForArc", file.SequentialNumberForArc);
SqlParameter Negative = new SqlParameter("@Negative", file.Negative);
SqlParameter ParentTagId = new SqlParameter("@ParentTagId", file.ParentTagId);
SqlParameter ChildTagId = new SqlParameter("@ChildTagId", file.ChildTagId);
SqlParameter Dataset_Id = new SqlParameter("@Dataset_Id", dataset.Id);
SqlParameter Submission_Id = new SqlParameter("@Submission_Id", file.SubmissionId);
Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETCALC_INSERT " +
"@LineNumber, @SequentialNumberForGrouping, @SequentialNumberForArc, @Negative, @ParentTagId, @ChildTagId, @Dataset_Id, @Submission_Id",
LineNumber, SequentialNumberForGrouping, SequentialNumberForArc, Negative, ParentTagId, ChildTagId, Dataset_Id, Submission_Id);
}
private void Timer_Tick(object sender, EventArgs e)
{
using (IEdgarDatasetService edsserv = new EdgarDatasetService(new AnalystEdgarDatasetsBulkRepository()))
{
int id = Convert.ToInt32(lblDatasetInProcess.Text);
EdgarDataset ds = edsserv.GetDataset(id);
LoadDatasets(ds, bindingSourceDatasetInProcess);
if (ds.IsComplete || !edsserv.IsRunning(id))
{
timer.Stop();
btnLoadDataset.Enabled = true;
}
}
lblTimer.Text = "Timer running - " + (DateTime.Now - startTime).ToString();
}
public void Add(EdgarDataset dataset, EdgarDatasetNumber number)
{
SqlParameter DatavalueEnddate = new SqlParameter("@DatavalueEnddate", number.DatavalueEnddate);
SqlParameter countOfNumberOfQuarters = new SqlParameter("@CountOfNumberOfQuarters", number.CountOfNumberOfQuarters);
SqlParameter UnitOfMeasure = new SqlParameter("@UnitOfMeasure", number.UnitOfMeasure);
SqlParameter iprx = new SqlParameter("@IPRX", number.IPRX);
SqlParameter value = new SqlParameter("@Value", number.Value);
if (!number.Value.HasValue)
{
value.Value = DBNull.Value;
}
SqlParameter footNote;
if (string.IsNullOrEmpty(number.FootNote))
{
footNote = new SqlParameter("@FootNote", DBNull.Value);
}
else
{
footNote = new SqlParameter("@FootNote", number.FootNote);
}
SqlParameter footLength = new SqlParameter("@FootLength", number.FootLength);
SqlParameter numberOfDimensions = new SqlParameter("@NumberOfDimensions", number.NumberOfDimensions);
SqlParameter coRegistrant;
if (string.IsNullOrEmpty(number.CoRegistrant))
{
coRegistrant = new SqlParameter("@CoRegistrant", DBNull.Value);
}
else
{
coRegistrant = new SqlParameter("@CoRegistrant", number.CoRegistrant);
}
SqlParameter durp = new SqlParameter("@durp", number.Durp);
SqlParameter datp = new SqlParameter("@datp", number.Datp);
SqlParameter decimals = new SqlParameter("@Decimals", number.Decimals);
SqlParameter dimensionId = new SqlParameter("@Dimension_Id", number.DimensionId);
SqlParameter submissionId = new SqlParameter("@Submission_Id", number.SubmissionId);
SqlParameter tagId = new SqlParameter("@Tag_Id", number.TagId);
SqlParameter lineNumber = new SqlParameter("@LineNumber", number.LineNumber);
SqlParameter edgarDatasetId = new SqlParameter("@EdgarDataset_Id", dataset.Id);
Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETNUMBER_INSERT " +
"@DatavalueEnddate, @CountOfNumberOfQuarters,@UnitOfMeasure, @IPRX, @Value, @FootNote, @FootLength, @NumberOfDimensions, @CoRegistrant, @durp, @datp, @Decimals, @Dimension_Id, @Submission_Id, @Tag_Id,@LineNumber, @EdgarDataset_Id",
DatavalueEnddate, countOfNumberOfQuarters, UnitOfMeasure, iprx, value, footNote, footLength, numberOfDimensions, coRegistrant, durp, datp, decimals, dimensionId, submissionId, tagId, lineNumber, edgarDatasetId
);
}
public void Add(EdgarDataset dataset, EdgarDatasetText file)
{
SqlParameter LineNumber = new SqlParameter("@LineNumber", file.LineNumber);
SqlParameter DatavalueEnddate = new SqlParameter("@DatavalueEnddate", file.DatavalueEnddate);
SqlParameter CountOfNumberOfQuarters = new SqlParameter("@CountOfNumberOfQuarters", file.CountOfNumberOfQuarters);
SqlParameter Iprx = new SqlParameter("@Iprx", file.Iprx);
SqlParameter Language = new SqlParameter("@Language", file.Language);
SqlParameter Dcml = new SqlParameter("@Dcml", file.Dcml);
SqlParameter Durp = new SqlParameter("@Durp", file.Durp);
SqlParameter Datp = new SqlParameter("@Datp", file.Datp);
SqlParameter DimensionNumber = new SqlParameter("@DimensionNumber", file.DimensionNumber);
if (!file.DimensionNumber.HasValue)
{
DimensionNumber.Value = DBNull.Value;
}
SqlParameter CoRegistrant;
if (!string.IsNullOrEmpty(file.CoRegistrant))
{
CoRegistrant = new SqlParameter("@CoRegistrant", file.CoRegistrant);
}
else
{
CoRegistrant = new SqlParameter("@CoRegistrant", DBNull.Value);
}
SqlParameter Escaped = new SqlParameter("@Escaped", file.Escaped);
SqlParameter SourceLength = new SqlParameter("@SourceLength", file.SourceLength);
SqlParameter TextLength = new SqlParameter("@TextLength", file.TextLength);
SqlParameter FootNote = new SqlParameter("@FootNote", file.FootNote);
SqlParameter FootLength = new SqlParameter("@FootLength", file.FootLength);
if (!file.FootLength.HasValue)
{
FootLength.Value = DBNull.Value;
}
SqlParameter paramContext = new SqlParameter("@Context", file.Context);
SqlParameter Value = new SqlParameter("@Value", file.Value);
SqlParameter Dimension_Id = new SqlParameter("@Dimension_Id", file.DimensionId);
SqlParameter Submission_Id = new SqlParameter("@Submission_Id", file.SubmissionId);
SqlParameter Tag_Id = new SqlParameter("@Tag_Id", file.TagId);
SqlParameter DatasetId = new SqlParameter("@DatasetId", dataset.Id);
Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETTEXT_INSERT " +
"@LineNumber, @DatavalueEnddate, @CountOfNumberOfQuarters, @Iprx, @Language, @Dcml, @Durp, @Datp, @DimensionNumber, @CoRegistrant, @Escaped, @SourceLength, @TextLength, @FootNote, @FootLength, @Context, @Value, @Dimension_Id, @Submission_Id, @Tag_Id, @DatasetId",
LineNumber, DatavalueEnddate, CountOfNumberOfQuarters, Iprx, Language, Dcml, Durp, Datp, DimensionNumber, CoRegistrant, Escaped, SourceLength, TextLength, FootNote, FootLength, paramContext, Value, Dimension_Id, Submission_Id, Tag_Id, DatasetId);
}
public void Add(EdgarDataset dataset, EdgarDatasetDimension dim)
{
SqlParameter DimensionH = new SqlParameter("@DimensionH", SqlDbType.NVarChar, EdgarDatasetDimension.LENGHT_FIELD_DIMENSIONH);
DimensionH.Value = dim.DimensionH;
SqlParameter Segments = new SqlParameter("@Segments", dim.Segments);
if (string.IsNullOrEmpty(dim.Segments))
{
Segments.Value = DBNull.Value;
}
SqlParameter SegmentTruncated = new SqlParameter("@SegmentTruncated", dim.SegmentTruncated);
SqlParameter DataSetId = new SqlParameter("@DataSetId", dataset.Id);
SqlParameter LineNumber = new SqlParameter("@LineNumber", dim.LineNumber);
Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETDIMENSIONS_INSERT " +
"@DimensionH , @Segments, @SegmentTruncated, @DataSetId,@LineNumber",
DimensionH, Segments, SegmentTruncated, DataSetId, LineNumber)
;
}
private EdgarTaskState[] LoadSubTagDim(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
EdgarTaskState stateSubs, stateTag, stateDim;
stateSubs = new EdgarTaskState(EdgarDatasetSubmission.FILE_NAME, ds, repo);
stateTag = new EdgarTaskState(EdgarDatasetTag.FILE_NAME, ds, repo);
stateDim = new EdgarTaskState(EdgarDatasetDimension.FILE_NAME, ds, repo);
states.Add(stateSubs);
states.Add(stateTag);
states.Add(stateDim);
IList <Task> tasks = new List <Task>();
log.Info("Datasetid " + ds.Id.ToString() + " -- starting submissionService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
submissionService.Process(stateSubs, false, EdgarDatasetSubmission.FILE_NAME, "Submissions")//false --> to avoid to have too many threads
));
log.Info("Datasetid " + ds.Id.ToString() + " -- starting tagService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
if (ConfigurationManager.AppSettings["run_tag_in_parallel"] == "true")
{
tagService.Process(stateTag, true, EdgarDatasetTag.FILE_NAME, "Tags");
}
else
{
tagService.Process(stateTag, false, EdgarDatasetTag.FILE_NAME, "Tags");
}
}));
log.Info("Datasetid " + ds.Id.ToString() + " -- starting dimensionService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
dimensionService.Process(stateDim, false, EdgarDatasetDimension.FILE_NAME, "Dimensions");
}));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
private EdgarTaskState[] LoadRenPre(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> nums, ConcurrentDictionary <string, int> texts)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
List <Task> tasks = new List <Task>();
EdgarTaskState stateRen = new EdgarTaskState(EdgarDatasetRender.FILE_NAME, ds, repo);
states.Add(stateRen);
EdgarTaskState statePre = new EdgarTaskState(EdgarDatasetPresentation.FILE_NAME, ds, repo);
states.Add(statePre);
tasks.Add(Task.Factory.StartNew(() =>
{
renderingService.Subs = subs;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting renderingService.Process(...)");
//Presentations has a relationship to renders
renderingService.Process(stateRen, true, EdgarDatasetRender.FILE_NAME, "Renders");
presentationService.Subs = subs;
presentationService.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- loading all rens for presentationService.Process(...)");
presentationService.Renders = renderingService.GetAsConcurrent(ds.Id);
presentationService.Nums = nums;
presentationService.Texts = texts;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting presentationService.Process(...)");
if (ConfigurationManager.AppSettings["run_pre_in_parallel"] == "true")
{
presentationService.Process(statePre, true, EdgarDatasetPresentation.FILE_NAME, "Presentations"); //parallel execution
}
else
{
presentationService.Process(statePre, false, EdgarDatasetPresentation.FILE_NAME, "Presentations");//sequential execution
}
}
));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetRender file)
{
repo.Add(dataset, file);
}
public EdgarTaskState(string processName, EdgarDataset ds, IEdgarDatasetsRepository dsRepo)
{
this.ds = ds;
this.dsRepo = dsRepo;
this.processName = processName;
}
public override void Add(IAnalystEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetText file)
{
repo.Add(dataset, file);
}
private void ProcessDatasetTask(int id)
{
//https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/task-based-asynchronous-programming?view=netframework-4.5.2
Task t = new Task(() =>
{
try
{
using (IAnalystEdgarDatasetsRepository repository = CreateRepository())
{
EdgarDataset ds = repository.GetDataset(id);
if (ds != null)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////
////BEGIN PROCESS
Stopwatch watch = System.Diagnostics.Stopwatch.StartNew();
log.Info("Datasetid " + id.ToString() + " -- BEGIN dataset process");
//Load Submissions, Tags and Dimensions
EdgarTaskState[] states = LoadSubTagDim(ds, repository);
bool ok = ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadSubTagDim(..): ManageErrors: {1}; Submissions: {2}/{3}; Tags: {4}/{5}; Dimensions: {6}/{7}", id, ok, ds.ProcessedSubmissions, ds.TotalSubmissions, ds.ProcessedTags, ds.TotalTags, ds.ProcessedDimensions, ds.TotalDimensions));
if (!ok || ds.ProcessedSubmissions != ds.TotalSubmissions || ds.ProcessedTags != ds.TotalTags || ds.ProcessedDimensions != ds.TotalDimensions)
{
log.Fatal("Process of sub, tags or dims failed, process can't continue");
return;
}
//Retrieve all tags, submissions and dimensions to fill the relationship
//Load Calculations, Texts and Numbers
log.Info("Datasetid " + id.ToString() + " -- loading all tags for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> tags = tagService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all subs for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> subs = submissionService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all dims for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> dims = dimensionService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- Starting LoadCalTxtNum(...)");
states = LoadCalTxtNum(ds, repository, subs, tags, dims);
ok = ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadCalTxtNum(..): ManageErrors: {1}; Calculations: {2}/{3}; Texts: {4}/{5}; Numbers: {6}/{7}", id, ok, ds.ProcessedCalculations, ds.TotalCalculations, ds.ProcessedTexts, ds.TotalTexts, ds.ProcessedNumbers, ds.TotalNumbers));
if (!ok || ds.ProcessedCalculations != ds.TotalCalculations || ds.ProcessedTexts != ds.TotalTexts || ds.ProcessedNumbers != ds.TotalNumbers)
{
log.Fatal("Process of cal, text or nums failed, process can't continue");
return;
}
log.Info("Datasetid " + id.ToString() + " -- releasing dims");
dims.Clear();
dims = null;
//Load Presentations and Renders
log.Info("Datasetid " + id.ToString() + " -- loading all nums for LoadRenPre(...)");
ConcurrentDictionary <string, int> nums = numService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all txt for LoadRenPre(...)");
ConcurrentDictionary <string, int> txts = textService.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- Starting LoadRenPre(...)");
states = LoadRenPre(ds, repository, subs, tags, nums, txts);
ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadRenPre(..): ManageErrors: {1}; Reners: {2}/{3}; Presentations: {4}/{5}", id, ok, ds.ProcessedRenders, ds.TotalRenders, ds.ProcessedPresentations, ds.TotalPresentations));
////END PROCESS
//////////////////////////////////////////////////////////////////////////////////////////////////////////
subs.Clear(); subs = null;
tags.Clear(); tags = null;
nums.Clear(); nums = null;
txts.Clear(); txts = null;
GC.Collect();//force GC
watch.Stop();
TimeSpan ts = watch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
log.Info("Datasetid " + id.ToString() + " -- END dataset process -- time: " + elapsedTime);
}
else
{
log.Fatal("Datasetid " + id.ToString() + " does not exists");
}
}
}
catch (Exception ex)
{
log.Fatal("Datasetid " + id.ToString() + " -- Unexpected error in EdgarDatasetService.Run(" + id.ToString() + "): " + ex.Message, ex);
log.FatalInner(ex.InnerException);
}
});
t.Start();
datasetsInProcess.TryAdd(id, t);
}
public void UpdateEdgarDataset(EdgarDataset dataset, string property)
{
Context.Entry <EdgarDataset>(dataset).Property(property).IsModified = true;
Context.SaveChanges();
}
public override void Add(IAnalystEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetPresentation file)
{
repo.Add(dataset, file);
}
public abstract void Add(IAnalystEdgarDatasetsRepository repo, EdgarDataset dataset, T file);
public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetCalculation file)
{
repo.Add(dataset, file);
}
public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetSubmission file)
{
repo.Add(dataset, file);
}
public EdgarDataset GetDataset(int id)
{
EdgarDataset ds = Context.DataSets.Where(x => x.Id == id).SingleOrDefault();
return(ds);
}
public void Add(EdgarDataset dataset)
{
Context.DataSets.Add(dataset);
Context.SaveChanges();
}