public void Add(EdgarDataset dataset, EdgarDatasetTag tag)
        {
            SqlParameter dsid     = new SqlParameter("@DataSetId", dataset.Id);
            SqlParameter tagparam = new SqlParameter("@Tag", tag.Tag);
            SqlParameter version  = new SqlParameter("@Version", tag.Version);

            SqlParameter custom    = new SqlParameter("@Custom", tag.Custom);
            SqlParameter abstracto = new SqlParameter("@Abstract", tag.Abstract);
            SqlParameter datatype  = new SqlParameter("@Datatype", tag.ValueType);

            if (tag.ValueType == null)
            {
                datatype.Value = DBNull.Value;
            }
            SqlParameter LabelText = new SqlParameter("@LabelText", tag.LabelText);

            if (tag.LabelText == null)
            {
                LabelText.Value = DBNull.Value;
            }
            SqlParameter Documentation = new SqlParameter("@Documentation", tag.Documentation);

            if (tag.Documentation == null)
            {
                Documentation.Value = DBNull.Value;
            }
            SqlParameter LineNumber = new SqlParameter("@LineNumber", tag.LineNumber);

            Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETTAGS_INSERT " +
                                               "@DataSetId, @tag,@version,@custom,@Abstract,@Datatype,@LabelText,@Documentation,@LineNumber",
                                               dsid, tagparam, version, custom, abstracto, datatype, LabelText, Documentation, LineNumber);
        }
Esempio n. 2
0
        private void LoadDatasets(EdgarDataset ds, BindingSource bs)
        {
            IList <EdgarDataset> datasets = new List <EdgarDataset>();

            datasets.Add(ds);
            LoadDatasets(datasets, bindingSourceDatasetInProcess);
        }
Esempio n. 3
0
 public override void Add(IAnalystEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetNumber file)
 {
     file.SubmissionId = Submissions[file.ADSH];
     file.TagId        = Tags[file.TagCompoundKey];
     file.DimensionId  = Dimensions[file.DimensionStr];
     repo.Add(dataset, file);
 }
Esempio n. 4
0
 public EdgarDataset GetDataset(int id)
 {
     using (IAnalystEdgarDatasetsRepository repository = CreateRepository())
     {
         EdgarDataset ds = repository.GetDataset(id);
         return(ds);
     }
 }
        public void Add(EdgarDataset dataset, EdgarDatasetPresentation pre)
        {
            SqlParameter ReportNumber               = new SqlParameter("@ReportNumber", pre.ReportNumber);
            SqlParameter Line                       = new SqlParameter("@Line", pre.Line);
            SqlParameter FinancialStatement         = new SqlParameter("@FinancialStatement", pre.FinancialStatement);
            SqlParameter Inpth                      = new SqlParameter("@Inpth", pre.Inpth);
            SqlParameter RenderFile                 = new SqlParameter("@RenderFile", pre.RenderFileStr);
            SqlParameter PreferredLabelXBRLLinkRole = new SqlParameter("@PreferredLabelXBRLLinkRole", pre.PreferredLabelXBRLLinkRole);
            SqlParameter PreferredLabel             = new SqlParameter("@PreferredLabel", pre.PreferredLabel);
            SqlParameter Negating                   = new SqlParameter("@Negating", pre.Negating);
            SqlParameter LineNumber                 = new SqlParameter("@LineNumber", pre.LineNumber);
            SqlParameter DataSetId                  = new SqlParameter("@DataSetId", dataset.Id);
            SqlParameter Submission_Id              = new SqlParameter("@Submission_Id", pre.SubmissionId);
            SqlParameter Tag_Id                     = new SqlParameter("@Tag_Id", pre.TagId);
            SqlParameter Number_Id;

            if (pre.NumberId > 0)
            {
                Number_Id = new SqlParameter("@Number_Id", pre.NumberId);
            }
            else
            {
                Number_Id = new SqlParameter("@Number_Id", DBNull.Value);
            }
            SqlParameter Text_Id;

            if (pre.TextId > 0)
            {
                Text_Id = new SqlParameter("@Text_Id", pre.TextId);
            }
            else
            {
                Text_Id = new SqlParameter("@Text_Id", DBNull.Value);
            }
            SqlParameter Render_Id;

            if (pre.RenderId <= 0)
            {
                Render_Id = new SqlParameter("@Render_Id", DBNull.Value);
            }
            else
            {
                Render_Id = new SqlParameter("@Render_Id", pre.RenderId);
            }
            SqlParameter adsh_tag_version = new SqlParameter("@adsh_tag_version", pre.ADSH_Tag_Version);

            if (string.IsNullOrEmpty(pre.ADSH_Tag_Version))
            {
                adsh_tag_version.Value = DBNull.Value;
            }

            Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETPRESENTATIONS_INSERT " +
                                               "@ReportNumber, @Line, @FinancialStatement, @Inpth,@RenderFile, @PreferredLabelXBRLLinkRole, @PreferredLabel, @Negating,@LineNumber, @DataSetId, @Submission_Id, @Tag_Id, @Number_Id, @Text_Id, @Render_Id, @adsh_tag_version",
                                               ReportNumber, Line, FinancialStatement, Inpth, PreferredLabelXBRLLinkRole, PreferredLabel, Negating, LineNumber, DataSetId, Submission_Id, Tag_Id, Number_Id, Text_Id, Render_Id, adsh_tag_version);
        }
Esempio n. 6
0
        private EdgarTaskState[] LoadCalTxtNum(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> dims)
        {
            List <EdgarTaskState> states = new List <EdgarTaskState>();
            List <Task>           tasks  = new List <Task>();

            //process calc file
            EdgarTaskState stateCalc = new EdgarTaskState(EdgarDatasetCalculation.FILE_NAME, ds, repo);

            states.Add(stateCalc);
            calcService.Submissions = subs;
            calcService.Tags        = tags;
            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  calcService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
                                            calcService.Process(stateCalc, false, EdgarDatasetCalculation.FILE_NAME, "Calculations")) //false --> to avoid to have too many threads
                      );

            //process text file
            EdgarTaskState stateText = new EdgarTaskState(EdgarDatasetText.FILE_NAME, ds, repo);

            states.Add(stateText);
            textService.Dimensions  = dims;
            textService.Submissions = subs;
            textService.Tags        = tags;
            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  textService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
            {
                textService.Process(stateText, true, EdgarDatasetText.FILE_NAME, "Texts");
                for (int i = 0; i < MAX_TRIALS; i++)
                {
                    if (!string.IsNullOrEmpty(stateText.FileNameToReprocess))
                    {
                        string filee = stateText.FileNameToReprocess.Split('\\').Last();
                        textService.Process(stateText, true, stateText.FileNameToReprocess, "Texts");
                    }
                }
            }));

            //Process num file
            EdgarTaskState stateNum = new EdgarTaskState(EdgarDatasetNumber.FILE_NAME, ds, repo);

            states.Add(stateNum);
            numService.Dimensions  = dims;
            numService.Submissions = subs;
            numService.Tags        = tags;
            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  numService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
            {
                bool parallel = ConfigurationManager.AppSettings["run_num_in_parallel"] == "true";
                numService.Process(stateNum, parallel, EdgarDatasetNumber.FILE_NAME, "Numbers");
            }));

            Task.WaitAll(tasks.ToArray());
            return(states.ToArray());
        }
        public void Add(EdgarDataset dataset, EdgarDatasetRender ren)
        {
            SqlParameter Report       = new SqlParameter("@Report", ren.Report);
            SqlParameter RenderFile   = new SqlParameter("@RenderFile", ren.RenderFileStr);
            SqlParameter MenuCategory = new SqlParameter("@MenuCategory", ren.MenuCategory);
            SqlParameter ShortName    = new SqlParameter("@ShortName", ren.ShortName);
            SqlParameter LongName     = new SqlParameter("@LongName", ren.LongName);

            SqlParameter Roleuri;

            if (string.IsNullOrEmpty(ren.RoleURI))
            {
                Roleuri = new SqlParameter("@Roleuri", DBNull.Value);
            }
            else
            {
                Roleuri = new SqlParameter("@Roleuri", ren.RoleURI);
            }

            SqlParameter ParentRoleuri;

            if (string.IsNullOrEmpty(ren.ParentRoleURI))
            {
                ParentRoleuri = new SqlParameter("@ParentRoleuri", DBNull.Value);
            }
            else
            {
                ParentRoleuri = new SqlParameter("@ParentRoleuri", ren.ParentRoleURI);
            }

            SqlParameter ParentReport = new SqlParameter("@ParentReport", ren.ParentReport);

            if (ren.ParentReport == null)
            {
                ParentReport.Value = DBNull.Value;
            }

            SqlParameter UltimateParentReport = new SqlParameter("@UltimateParentReport", ren.UltimateParentReport);

            if (ren.UltimateParentReport == null)
            {
                UltimateParentReport.Value = DBNull.Value;
            }

            SqlParameter Submission_Id = new SqlParameter("@Submission_Id", ren.SubmissionId);
            SqlParameter DataSetId     = new SqlParameter("@DataSetId", dataset.Id);
            SqlParameter lineNumber    = new SqlParameter("@LineNumber", ren.LineNumber);

            Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETRENDERS_INSERT " +
                                               "@Report, @RenderFile, @MenuCategory, @ShortName, @LongName, @Roleuri, @ParentRoleuri, @ParentReport, @UltimateParentReport, @Submission_Id, @DataSetId, @LineNumber",
                                               Report, MenuCategory, ShortName, LongName, Roleuri, ParentRoleuri, ParentReport, UltimateParentReport, Submission_Id, DataSetId, lineNumber
                                               );
        }
        public void Add(EdgarDataset dataset, EdgarDatasetSubmission sub)
        {
            SqlParameter ADSH         = new SqlParameter("@ADSH", sub.ADSH);
            SqlParameter Period       = new SqlParameter("@Period", sub.Period);
            SqlParameter Detail       = new SqlParameter("@Detail", sub.Detail);
            SqlParameter XBRLInstance = new SqlParameter("@XBRLInstance", sub.XBRLInstance);

            if (sub.XBRLInstance == null)
            {
                XBRLInstance.Value = DBNull.Value;
            }
            SqlParameter NumberOfCIKs   = new SqlParameter("@NumberOfCIKs", sub.NumberOfCIKs);
            SqlParameter AdditionalCIKs = new SqlParameter("@AdditionalCIKs", sub.AdditionalCIKs);

            if (sub.AdditionalCIKs == null)
            {
                AdditionalCIKs.Value = DBNull.Value;
            }
            SqlParameter PublicFloatUSD = new SqlParameter("@PublicFloatUSD", sub.PublicFloatUSD);

            if (sub.PublicFloatUSD == null)
            {
                PublicFloatUSD.Value = DBNull.Value;
            }
            SqlParameter FloatDate = new SqlParameter("@FloatDate", sub.FloatDate);

            if (sub.FloatDate == null)
            {
                FloatDate.Value = DBNull.Value;
            }
            SqlParameter FloatAxis = new SqlParameter("@FloatAxis", sub.FloatAxis);

            if (sub.FloatAxis == null)
            {
                FloatAxis.Value = DBNull.Value;
            }
            SqlParameter FloatMems = new SqlParameter("@FloatMems", sub.FloatMems);

            if (sub.FloatMems == null)
            {
                FloatMems.Value = DBNull.Value;
            }
            SqlParameter lineNumber      = new SqlParameter("@LineNumber", sub.LineNumber);
            SqlParameter FormId          = new SqlParameter("@Form_id", sub.Form.Id);
            SqlParameter Registrant_Id   = new SqlParameter("@Registrant_Id", sub.Registrant.Id);
            SqlParameter EdgarDataset_Id = new SqlParameter("@EdgarDataset_Id", dataset.Id);


            Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASSUBMISSIONS_INSERT " +
                                               "@ADSH, @Period, @Detail, @XBRLInstance, @NumberOfCIKs, @AdditionalCIKs, @PublicFloatUSD, @FloatDate, @FloatAxis, @FloatMems,@LineNumber, @Form_Id, @Registrant_Id, @EdgarDataset_Id",
                                               ADSH, Period, Detail, XBRLInstance, NumberOfCIKs, AdditionalCIKs, PublicFloatUSD, FloatDate, FloatAxis, FloatMems, lineNumber, FormId, Registrant_Id, EdgarDataset_Id);
        }
        public void Add(EdgarDataset dataset, EdgarDatasetCalculation file)
        {
            SqlParameter LineNumber = new SqlParameter("@LineNumber", file.LineNumber);
            SqlParameter SequentialNumberForGrouping = new SqlParameter("@SequentialNumberForGrouping", file.SequentialNumberForGrouping);
            SqlParameter SequentialNumberForArc      = new SqlParameter("@SequentialNumberForArc", file.SequentialNumberForArc);
            SqlParameter Negative      = new SqlParameter("@Negative", file.Negative);
            SqlParameter ParentTagId   = new SqlParameter("@ParentTagId", file.ParentTagId);
            SqlParameter ChildTagId    = new SqlParameter("@ChildTagId", file.ChildTagId);
            SqlParameter Dataset_Id    = new SqlParameter("@Dataset_Id", dataset.Id);
            SqlParameter Submission_Id = new SqlParameter("@Submission_Id", file.SubmissionId);

            Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETCALC_INSERT " +
                                               "@LineNumber, @SequentialNumberForGrouping, @SequentialNumberForArc, @Negative, @ParentTagId, @ChildTagId, @Dataset_Id, @Submission_Id",
                                               LineNumber, SequentialNumberForGrouping, SequentialNumberForArc, Negative, ParentTagId, ChildTagId, Dataset_Id, Submission_Id);
        }
Esempio n. 10
0
 private void Timer_Tick(object sender, EventArgs e)
 {
     using (IEdgarDatasetService edsserv = new EdgarDatasetService(new AnalystEdgarDatasetsBulkRepository()))
     {
         int          id = Convert.ToInt32(lblDatasetInProcess.Text);
         EdgarDataset ds = edsserv.GetDataset(id);
         LoadDatasets(ds, bindingSourceDatasetInProcess);
         if (ds.IsComplete || !edsserv.IsRunning(id))
         {
             timer.Stop();
             btnLoadDataset.Enabled = true;
         }
     }
     lblTimer.Text = "Timer running - " + (DateTime.Now - startTime).ToString();
 }
        public void Add(EdgarDataset dataset, EdgarDatasetNumber number)
        {
            SqlParameter DatavalueEnddate        = new SqlParameter("@DatavalueEnddate", number.DatavalueEnddate);
            SqlParameter countOfNumberOfQuarters = new SqlParameter("@CountOfNumberOfQuarters", number.CountOfNumberOfQuarters);
            SqlParameter UnitOfMeasure           = new SqlParameter("@UnitOfMeasure", number.UnitOfMeasure);
            SqlParameter iprx  = new SqlParameter("@IPRX", number.IPRX);
            SqlParameter value = new SqlParameter("@Value", number.Value);

            if (!number.Value.HasValue)
            {
                value.Value = DBNull.Value;
            }
            SqlParameter footNote;

            if (string.IsNullOrEmpty(number.FootNote))
            {
                footNote = new SqlParameter("@FootNote", DBNull.Value);
            }
            else
            {
                footNote = new SqlParameter("@FootNote", number.FootNote);
            }
            SqlParameter footLength         = new SqlParameter("@FootLength", number.FootLength);
            SqlParameter numberOfDimensions = new SqlParameter("@NumberOfDimensions", number.NumberOfDimensions);
            SqlParameter coRegistrant;

            if (string.IsNullOrEmpty(number.CoRegistrant))
            {
                coRegistrant = new SqlParameter("@CoRegistrant", DBNull.Value);
            }
            else
            {
                coRegistrant = new SqlParameter("@CoRegistrant", number.CoRegistrant);
            }
            SqlParameter durp           = new SqlParameter("@durp", number.Durp);
            SqlParameter datp           = new SqlParameter("@datp", number.Datp);
            SqlParameter decimals       = new SqlParameter("@Decimals", number.Decimals);
            SqlParameter dimensionId    = new SqlParameter("@Dimension_Id", number.DimensionId);
            SqlParameter submissionId   = new SqlParameter("@Submission_Id", number.SubmissionId);
            SqlParameter tagId          = new SqlParameter("@Tag_Id", number.TagId);
            SqlParameter lineNumber     = new SqlParameter("@LineNumber", number.LineNumber);
            SqlParameter edgarDatasetId = new SqlParameter("@EdgarDataset_Id", dataset.Id);

            Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETNUMBER_INSERT " +
                                               "@DatavalueEnddate, @CountOfNumberOfQuarters,@UnitOfMeasure, @IPRX, @Value, @FootNote, @FootLength, @NumberOfDimensions, @CoRegistrant, @durp, @datp, @Decimals, @Dimension_Id, @Submission_Id, @Tag_Id,@LineNumber, @EdgarDataset_Id",
                                               DatavalueEnddate, countOfNumberOfQuarters, UnitOfMeasure, iprx, value, footNote, footLength, numberOfDimensions, coRegistrant, durp, datp, decimals, dimensionId, submissionId, tagId, lineNumber, edgarDatasetId
                                               );
        }
        public void Add(EdgarDataset dataset, EdgarDatasetText file)
        {
            SqlParameter LineNumber              = new SqlParameter("@LineNumber", file.LineNumber);
            SqlParameter DatavalueEnddate        = new SqlParameter("@DatavalueEnddate", file.DatavalueEnddate);
            SqlParameter CountOfNumberOfQuarters = new SqlParameter("@CountOfNumberOfQuarters", file.CountOfNumberOfQuarters);
            SqlParameter Iprx            = new SqlParameter("@Iprx", file.Iprx);
            SqlParameter Language        = new SqlParameter("@Language", file.Language);
            SqlParameter Dcml            = new SqlParameter("@Dcml", file.Dcml);
            SqlParameter Durp            = new SqlParameter("@Durp", file.Durp);
            SqlParameter Datp            = new SqlParameter("@Datp", file.Datp);
            SqlParameter DimensionNumber = new SqlParameter("@DimensionNumber", file.DimensionNumber);

            if (!file.DimensionNumber.HasValue)
            {
                DimensionNumber.Value = DBNull.Value;
            }
            SqlParameter CoRegistrant;

            if (!string.IsNullOrEmpty(file.CoRegistrant))
            {
                CoRegistrant = new SqlParameter("@CoRegistrant", file.CoRegistrant);
            }
            else
            {
                CoRegistrant = new SqlParameter("@CoRegistrant", DBNull.Value);
            }
            SqlParameter Escaped      = new SqlParameter("@Escaped", file.Escaped);
            SqlParameter SourceLength = new SqlParameter("@SourceLength", file.SourceLength);
            SqlParameter TextLength   = new SqlParameter("@TextLength", file.TextLength);
            SqlParameter FootNote     = new SqlParameter("@FootNote", file.FootNote);
            SqlParameter FootLength   = new SqlParameter("@FootLength", file.FootLength);

            if (!file.FootLength.HasValue)
            {
                FootLength.Value = DBNull.Value;
            }
            SqlParameter paramContext  = new SqlParameter("@Context", file.Context);
            SqlParameter Value         = new SqlParameter("@Value", file.Value);
            SqlParameter Dimension_Id  = new SqlParameter("@Dimension_Id", file.DimensionId);
            SqlParameter Submission_Id = new SqlParameter("@Submission_Id", file.SubmissionId);
            SqlParameter Tag_Id        = new SqlParameter("@Tag_Id", file.TagId);
            SqlParameter DatasetId     = new SqlParameter("@DatasetId", dataset.Id);

            Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETTEXT_INSERT " +
                                               "@LineNumber, @DatavalueEnddate, @CountOfNumberOfQuarters, @Iprx, @Language, @Dcml, @Durp, @Datp, @DimensionNumber, @CoRegistrant, @Escaped, @SourceLength, @TextLength, @FootNote, @FootLength, @Context, @Value, @Dimension_Id, @Submission_Id, @Tag_Id, @DatasetId",
                                               LineNumber, DatavalueEnddate, CountOfNumberOfQuarters, Iprx, Language, Dcml, Durp, Datp, DimensionNumber, CoRegistrant, Escaped, SourceLength, TextLength, FootNote, FootLength, paramContext, Value, Dimension_Id, Submission_Id, Tag_Id, DatasetId);
        }
        public void Add(EdgarDataset dataset, EdgarDatasetDimension dim)
        {
            SqlParameter DimensionH = new SqlParameter("@DimensionH", SqlDbType.NVarChar, EdgarDatasetDimension.LENGHT_FIELD_DIMENSIONH);

            DimensionH.Value = dim.DimensionH;
            SqlParameter Segments = new SqlParameter("@Segments", dim.Segments);

            if (string.IsNullOrEmpty(dim.Segments))
            {
                Segments.Value = DBNull.Value;
            }
            SqlParameter SegmentTruncated = new SqlParameter("@SegmentTruncated", dim.SegmentTruncated);
            SqlParameter DataSetId        = new SqlParameter("@DataSetId", dataset.Id);
            SqlParameter LineNumber       = new SqlParameter("@LineNumber", dim.LineNumber);

            Context.Database.ExecuteSqlCommand("exec SP_EDGARDATASETDIMENSIONS_INSERT " +
                                               "@DimensionH , @Segments, @SegmentTruncated, @DataSetId,@LineNumber",
                                               DimensionH, Segments, SegmentTruncated, DataSetId, LineNumber)
            ;
        }
Esempio n. 14
0
        private EdgarTaskState[] LoadSubTagDim(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo)
        {
            List <EdgarTaskState> states = new List <EdgarTaskState>();
            EdgarTaskState        stateSubs, stateTag, stateDim;

            stateSubs = new EdgarTaskState(EdgarDatasetSubmission.FILE_NAME, ds, repo);
            stateTag  = new EdgarTaskState(EdgarDatasetTag.FILE_NAME, ds, repo);
            stateDim  = new EdgarTaskState(EdgarDatasetDimension.FILE_NAME, ds, repo);
            states.Add(stateSubs);
            states.Add(stateTag);
            states.Add(stateDim);
            IList <Task> tasks = new List <Task>();

            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  submissionService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
                                            submissionService.Process(stateSubs, false, EdgarDatasetSubmission.FILE_NAME, "Submissions")//false --> to avoid to have too many threads
                                            ));

            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  tagService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
            {
                if (ConfigurationManager.AppSettings["run_tag_in_parallel"] == "true")
                {
                    tagService.Process(stateTag, true, EdgarDatasetTag.FILE_NAME, "Tags");
                }
                else
                {
                    tagService.Process(stateTag, false, EdgarDatasetTag.FILE_NAME, "Tags");
                }
            }));

            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  dimensionService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
            {
                dimensionService.Process(stateDim, false, EdgarDatasetDimension.FILE_NAME, "Dimensions");
            }));

            Task.WaitAll(tasks.ToArray());
            return(states.ToArray());
        }
Esempio n. 15
0
        private EdgarTaskState[] LoadRenPre(EdgarDataset ds, IAnalystEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> nums, ConcurrentDictionary <string, int> texts)
        {
            List <EdgarTaskState> states = new List <EdgarTaskState>();
            List <Task>           tasks  = new List <Task>();

            EdgarTaskState stateRen = new EdgarTaskState(EdgarDatasetRender.FILE_NAME, ds, repo);

            states.Add(stateRen);

            EdgarTaskState statePre = new EdgarTaskState(EdgarDatasetPresentation.FILE_NAME, ds, repo);

            states.Add(statePre);

            tasks.Add(Task.Factory.StartNew(() =>
            {
                renderingService.Subs = subs;
                log.Info("Datasetid " + ds.Id.ToString() + " -- starting  renderingService.Process(...)");
                //Presentations has a relationship to renders
                renderingService.Process(stateRen, true, EdgarDatasetRender.FILE_NAME, "Renders");
                presentationService.Subs = subs;
                presentationService.Tags = tags;
                log.Info("Datasetid " + ds.Id.ToString() + " -- loading all rens for presentationService.Process(...)");
                presentationService.Renders = renderingService.GetAsConcurrent(ds.Id);
                presentationService.Nums    = nums;
                presentationService.Texts   = texts;
                log.Info("Datasetid " + ds.Id.ToString() + " -- starting  presentationService.Process(...)");
                if (ConfigurationManager.AppSettings["run_pre_in_parallel"] == "true")
                {
                    presentationService.Process(statePre, true, EdgarDatasetPresentation.FILE_NAME, "Presentations"); //parallel execution
                }
                else
                {
                    presentationService.Process(statePre, false, EdgarDatasetPresentation.FILE_NAME, "Presentations");//sequential execution
                }
            }
                                            ));
            Task.WaitAll(tasks.ToArray());
            return(states.ToArray());
        }
 public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetRender file)
 {
     repo.Add(dataset, file);
 }
Esempio n. 17
0
 public EdgarTaskState(string processName, EdgarDataset ds, IEdgarDatasetsRepository dsRepo)
 {
     this.ds          = ds;
     this.dsRepo      = dsRepo;
     this.processName = processName;
 }
 public override void Add(IAnalystEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetText file)
 {
     repo.Add(dataset, file);
 }
Esempio n. 19
0
        private void ProcessDatasetTask(int id)
        {
            //https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/task-based-asynchronous-programming?view=netframework-4.5.2
            Task t = new Task(() =>
            {
                try
                {
                    using (IAnalystEdgarDatasetsRepository repository = CreateRepository())
                    {
                        EdgarDataset ds = repository.GetDataset(id);
                        if (ds != null)
                        {
                            //////////////////////////////////////////////////////////////////////////////////////////////////////////
                            ////BEGIN PROCESS
                            Stopwatch watch = System.Diagnostics.Stopwatch.StartNew();
                            log.Info("Datasetid " + id.ToString() + " -- BEGIN dataset process");

                            //Load Submissions, Tags and Dimensions
                            EdgarTaskState[] states = LoadSubTagDim(ds, repository);
                            bool ok = ManageErrors(states);
                            log.Info(string.Format("Datasetid {0} -- Variables after LoadSubTagDim(..): ManageErrors: {1}; Submissions: {2}/{3}; Tags: {4}/{5}; Dimensions: {6}/{7}", id, ok, ds.ProcessedSubmissions, ds.TotalSubmissions, ds.ProcessedTags, ds.TotalTags, ds.ProcessedDimensions, ds.TotalDimensions));
                            if (!ok || ds.ProcessedSubmissions != ds.TotalSubmissions || ds.ProcessedTags != ds.TotalTags || ds.ProcessedDimensions != ds.TotalDimensions)
                            {
                                log.Fatal("Process of sub, tags or dims failed, process can't continue");
                                return;
                            }
                            //Retrieve all tags, submissions and dimensions to fill the relationship
                            //Load Calculations, Texts and Numbers
                            log.Info("Datasetid " + id.ToString() + " -- loading all tags for LoadCalTxtNum(...)");
                            ConcurrentDictionary <string, int> tags = tagService.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- loading all subs for LoadCalTxtNum(...)");
                            ConcurrentDictionary <string, int> subs = submissionService.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- loading all dims for LoadCalTxtNum(...)");
                            ConcurrentDictionary <string, int> dims = dimensionService.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- Starting LoadCalTxtNum(...)");
                            states = LoadCalTxtNum(ds, repository, subs, tags, dims);
                            ok     = ManageErrors(states);
                            log.Info(string.Format("Datasetid {0} -- Variables after LoadCalTxtNum(..): ManageErrors: {1}; Calculations: {2}/{3}; Texts: {4}/{5}; Numbers: {6}/{7}", id, ok, ds.ProcessedCalculations, ds.TotalCalculations, ds.ProcessedTexts, ds.TotalTexts, ds.ProcessedNumbers, ds.TotalNumbers));
                            if (!ok || ds.ProcessedCalculations != ds.TotalCalculations || ds.ProcessedTexts != ds.TotalTexts || ds.ProcessedNumbers != ds.TotalNumbers)
                            {
                                log.Fatal("Process of cal, text or nums failed, process can't continue");
                                return;
                            }
                            log.Info("Datasetid " + id.ToString() + " -- releasing dims");
                            dims.Clear();
                            dims = null;

                            //Load Presentations and Renders
                            log.Info("Datasetid " + id.ToString() + " -- loading all nums for LoadRenPre(...)");
                            ConcurrentDictionary <string, int> nums = numService.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- loading all txt for LoadRenPre(...)");
                            ConcurrentDictionary <string, int> txts = textService.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- Starting LoadRenPre(...)");
                            states = LoadRenPre(ds, repository, subs, tags, nums, txts);
                            ManageErrors(states);
                            log.Info(string.Format("Datasetid {0} -- Variables after LoadRenPre(..): ManageErrors: {1}; Reners: {2}/{3}; Presentations: {4}/{5}", id, ok, ds.ProcessedRenders, ds.TotalRenders, ds.ProcessedPresentations, ds.TotalPresentations));
                            ////END PROCESS
                            //////////////////////////////////////////////////////////////////////////////////////////////////////////
                            subs.Clear(); subs = null;
                            tags.Clear(); tags = null;
                            nums.Clear(); nums = null;
                            txts.Clear(); txts = null;
                            GC.Collect();//force GC

                            watch.Stop();
                            TimeSpan ts        = watch.Elapsed;
                            string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
                            log.Info("Datasetid " + id.ToString() + " -- END dataset process -- time: " + elapsedTime);
                        }
                        else
                        {
                            log.Fatal("Datasetid " + id.ToString() + " does not exists");
                        }
                    }
                }
                catch (Exception ex)
                {
                    log.Fatal("Datasetid " + id.ToString() + " -- Unexpected error in EdgarDatasetService.Run(" + id.ToString() + "): " + ex.Message, ex);
                    log.FatalInner(ex.InnerException);
                }
            });

            t.Start();
            datasetsInProcess.TryAdd(id, t);
        }
 public void UpdateEdgarDataset(EdgarDataset dataset, string property)
 {
     Context.Entry <EdgarDataset>(dataset).Property(property).IsModified = true;
     Context.SaveChanges();
 }
 public override void Add(IAnalystEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetPresentation file)
 {
     repo.Add(dataset, file);
 }
 public abstract void Add(IAnalystEdgarDatasetsRepository repo, EdgarDataset dataset, T file);
 public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetCalculation file)
 {
     repo.Add(dataset, file);
 }
 public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetSubmission file)
 {
     repo.Add(dataset, file);
 }
        public EdgarDataset GetDataset(int id)
        {
            EdgarDataset ds = Context.DataSets.Where(x => x.Id == id).SingleOrDefault();

            return(ds);
        }
 public void Add(EdgarDataset dataset)
 {
     Context.DataSets.Add(dataset);
     Context.SaveChanges();
 }