public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetNumber file)
 {
     file.SubmissionId = Submissions[file.ADSH];
     file.TagId        = Tags[file.TagCompoundKey];
     file.DimensionId  = Dimensions[file.DimensionStr];
     repo.Add(dataset, file);
 }
        private Registrant ParseRegistrant(IEdgarDatasetsRepository repository, string cik, List <string> fieldNames, List <string> fields)
        {
            Registrant r = repository.GetRegistrant(cik);

            if (r == null)
            {
                r      = new Registrant();
                r.CIK  = int.Parse(cik);
                r.Name = fields[fieldNames.IndexOf("name")];
                string value = fields[fieldNames.IndexOf("sic")];
                r.SIC        = string.IsNullOrEmpty(value) ? null : repository.GetSIC(value);
                value        = fields[fieldNames.IndexOf("countryba")];
                r.CountryBA  = String.IsNullOrEmpty(value) ? null : value;
                value        = fields[fieldNames.IndexOf("cityba")];
                r.CityBA     = String.IsNullOrEmpty(value) ? null : value;
                value        = fields[fieldNames.IndexOf("countryinc")];
                r.CountryInc = String.IsNullOrEmpty(value) ? null : value;
                value        = fields[fieldNames.IndexOf("ein")];
                r.EmployeeIdentificationNumber = string.IsNullOrEmpty(value) ? (int?)null : int.Parse(value);
                r.FilerStatus             = fields[fieldNames.IndexOf("afs")];
                r.WellKnownSeasonedIssuer = fields[fieldNames.IndexOf("wksi")] == "1";
                value = fields[fieldNames.IndexOf("fye")];
                if (string.IsNullOrEmpty(value))
                {
                    r.FiscalYearEndDate = null;
                }
                else
                {
                    r.FiscalYearEndDate = Convert.ToInt16(value);
                }
                repository.Add(r);
            }
            return(r);
        }
        public override EdgarDatasetCalculation Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
        {
            try
            {
                EdgarDatasetCalculation calc = new EdgarDatasetCalculation();

                string adsh = fields[fieldNames.IndexOf("adsh")];
                calc.SubmissionId = Submissions[adsh];

                calc.SequentialNumberForGrouping = Convert.ToInt16(fields[fieldNames.IndexOf("grp")]);
                calc.SequentialNumberForArc      = Convert.ToInt16(fields[fieldNames.IndexOf("arc")]);

                string pTag     = fields[fieldNames.IndexOf("ptag")];
                string pVersion = fields[fieldNames.IndexOf("pversion")];
                calc.ParentTagId = Tags[pTag + pVersion];

                //Indicates a weight of -1 (TRUE if the arc is negative), but typically +1 (FALSE).
                calc.Negative = fields[fieldNames.IndexOf("negative")] == "-1" ? true : false;

                string cTag     = fields[fieldNames.IndexOf("ctag")];
                string cVersion = fields[fieldNames.IndexOf("cversion")];
                calc.ChildTagId = Tags[cTag + cVersion];

                calc.LineNumber = lineNumber;
                return(calc);
            }
            catch (Exception ex)
            {
                throw ex;
            }
        }
        public override EdgarDatasetRender Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
        {
            /*
             * adsh	report	rfile	menucat	shortname	longname	roleuri	parentroleuri	parentreport	ultparentrpt
             * 0001163302-16-000148	1	H	Cover	Document and Entity Information	0001000 - Document - Document and Entity Information	http://www.ussteel.com/role/DocumentAndEntityInformation
             * 0001163302-16-000148	2	H	Statements	Consolidated Statement Of Operations	1001000 - Statement - Consolidated Statement Of Operations	http://www.ussteel.com/role/ConsolidatedStatementOfOperations
             */
            EdgarDatasetRender ren  = new EdgarDatasetRender();
            string             adsh = fields[fieldNames.IndexOf("adsh")];

            ren.SubmissionId = Subs[adsh];
            ren.Report       = Convert.ToInt32(fields[fieldNames.IndexOf("report")]);
            string value = "";

            ren.RenderFile    = fields[fieldNames.IndexOf("rfile")][0];
            ren.RenderFileStr = ren.RenderFile.ToString();
            ren.MenuCategory  = fields[fieldNames.IndexOf("menucat")];
            ren.ShortName     = fields[fieldNames.IndexOf("shortname")];
            ren.LongName      = fields[fieldNames.IndexOf("longname")];
            ren.RoleURI       = fields[fieldNames.IndexOf("roleuri")];
            ren.ParentRoleURI = fields[fieldNames.IndexOf("parentroleuri")];
            value             = fields[fieldNames.IndexOf("parentreport")];
            if (!string.IsNullOrEmpty(value))
            {
                ren.ParentReport = Convert.ToInt32(value);
            }
            value = fields[fieldNames.IndexOf("ultparentrpt")];
            if (!string.IsNullOrEmpty(value))
            {
                ren.UltimateParentReport = Convert.ToInt32(value);
            }
            ren.LineNumber = lineNumber;
            return(ren);
        }
 public IList <EdgarDataset> GetDatasets()
 {
     using (IEdgarDatasetsRepository repository = CreateRepository())
     {
         IList <EdgarDataset> datasets = repository.Get <EdgarDataset>();
         return(datasets);
     }
 }
 public EdgarDataset GetDataset(int id)
 {
     using (IEdgarDatasetsRepository repository = CreateRepository())
     {
         EdgarDataset ds = repository.GetDataset(id);
         return(ds);
     }
 }
        public override EdgarDatasetText Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
        {
            /*
             * adsh	tag	version	ddate	qtrs	iprx	lang	dcml	durp	datp	dimh	dimn	coreg	escaped	srclen	txtlen	footnote	footlen	context	value
             * 0001163302-16-000148	AssetRetirementObligationDisclosureTextBlock	us-gaap/2016	20160930	3	0	en-US	32767	0.008219957	0.0	0x00000000	0		1	17336	907		0	FD2016Q3YTD	Asset Retirement Obligations U. S. Steels asset retirement obligations (AROs) primarily relate to mine and landfill closure and post-closure costs. The following table reflects changes in the carrying values of AROs: (In millions) September�30, 2016 December�31, 2015 Balance at beginning of year $ 89 $ 48 Additional obligations incurred 3 45 (a) Obligations settled (9 ) (6 ) Foreign currency translation effects  (1 ) Accretion expense 2 3 Balance at end of period $ 85 $ 89 (a) Additional AROs relate to the permanent closure of the coke production facilities at Gary Works and Granite City Works. Certain AROs related to disposal costs of the majority of fixed assets at our integrated steel facilities have not been recorded because they have an indeterminate settlement date. These AROs will be initially recognized in the period in which sufficient information exists to estimate their fair value.
             * 0001163302-16-000148	BasisOfPresentationAndSignificantAccountingPoliciesTextBlock	us-gaap/2016	20160930	3	0	en-US	32767	0.008219957	0.0	0x00000000	0		1	3331	2042		0	FD2016Q3YTD	Basis of Presentation and Significant Accounting Policies United States Steel Corporation produces and sells steel products, including flat-rolled and tubular products, in North America and Central Europe. Operations in North America also include iron ore and coke production facilities, railroad services and real estate operations. Operations in Europe also include coke production facilities. The year-end Consolidated Balance Sheet data was derived from audited statements but does not include all disclosures required for complete financial statements by accounting principles generally accepted in the United States of America (U.S. GAAP). The other information in these financial statements is unaudited but, in the opinion of management, reflects all adjustments necessary for a fair statement of the results for the periods covered. All such adjustments are of a normal recurring nature unless disclosed otherwise. These financial statements, including notes, have been prepared in accordance with the applicable rules of the Securities and Exchange Commission and do not include all of the information and disclosures required by U.S. GAAP for complete financial statements. Additional information is contained in the United States Steel Corporation Annual Report on Form�10-K for the fiscal year ended December�31, 2015 , which should be read in conjunction with these financial statements. Revision of Prior Period Financial Statements During 2015, the Company identified a prior period error related to the classification of unpaid capital expenditures in the Consolidated Statements of Cash Flows that impacted the quarterly interim financial statements in 2015. As a result, the Consolidated Statement of Cash Flows for the the nine months ended September 30, 2015 has been revised to reflect a decrease in cash provided by operating activities and a decrease in cash used in investing activities of $55 million . The Company has concluded the impact of this error was not material to the previously filed financial statements.
             */

            //aca puede iterar sobre la coleccion como en el proyecto sec q db

            EdgarDatasetText text = new EdgarDatasetText();
            string           adsh = fields[fieldNames.IndexOf("adsh")];

            text.SubmissionId = Submissions[adsh];
            string tag     = fields[fieldNames.IndexOf("tag")];
            string version = fields[fieldNames.IndexOf("version")];

            text.TagId = Tags[tag + version];
            string value = fields[fieldNames.IndexOf("ddate")];

            text.DatavalueEnddate        = new DateTime(int.Parse(value.Substring(0, 4)), int.Parse(value.Substring(4, 2)), int.Parse(value.Substring(6, 2)));
            text.CountOfNumberOfQuarters = Convert.ToInt32(fields[fieldNames.IndexOf("qtrs")]);
            text.Iprx        = Convert.ToInt16(fields[fieldNames.IndexOf("iprx")]);
            text.Language    = fields[fieldNames.IndexOf("lang")];
            text.Dcml        = Convert.ToInt32(fields[fieldNames.IndexOf("dcml")]);
            text.Durp        = float.Parse(fields[fieldNames.IndexOf("durp")], CultureInfo.GetCultureInfo("en-us").NumberFormat);
            text.Datp        = float.Parse(fields[fieldNames.IndexOf("datp")], CultureInfo.GetCultureInfo("en-us").NumberFormat);
            text.DimensionId = Dimensions[fields[fieldNames.IndexOf("dimh")]];
            value            = fields[fieldNames.IndexOf("dimn")];
            if (!string.IsNullOrEmpty(value))
            {
                text.DimensionNumber = Convert.ToInt16(value);
            }
            value = fields[fieldNames.IndexOf("coreg")];
            if (string.IsNullOrEmpty(value))
            {
                text.CoRegistrant = null;
            }
            else
            {
                text.CoRegistrant = value;
            }
            text.Escaped      = fields[fieldNames.IndexOf("escaped")] != "0";
            text.SourceLength = Convert.ToInt32(fields[fieldNames.IndexOf("srclen")]);
            text.TextLength   = Convert.ToInt32(fields[fieldNames.IndexOf("txtlen")]);
            text.FootNote     = fields[fieldNames.IndexOf("footnote")];
            value             = fields[fieldNames.IndexOf("footlen")];
            if (!string.IsNullOrEmpty(value))
            {
                text.FootLength = Convert.ToInt32(value);
            }
            ;
            text.Context    = fields[fieldNames.IndexOf("context")];
            text.Value      = fields[fieldNames.IndexOf("value")];
            text.LineNumber = lineNumber;
            return(text);
        }
        private EdgarTaskState[] LoadCalTxtNum(EdgarDataset ds, IEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> dims)
        {
            List <EdgarTaskState> states = new List <EdgarTaskState>();
            List <Task>           tasks  = new List <Task>();

            //process calc file
            EdgarTaskState stateCalc = new EdgarTaskState(EdgarDatasetCalculation.FILE_NAME, ds, repo);

            states.Add(stateCalc);
            calcParser.Submissions = subs;
            calcParser.Tags        = tags;
            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  calcService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
                                            calcParser.Process(stateCalc, false, EdgarDatasetCalculation.FILE_NAME, "Calculations")) //false --> to avoid to have too many threads
                      );

            //process text file
            EdgarTaskState stateText = new EdgarTaskState(EdgarDatasetText.FILE_NAME, ds, repo);

            states.Add(stateText);
            textParser.Dimensions  = dims;
            textParser.Submissions = subs;
            textParser.Tags        = tags;
            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  textService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
            {
                textParser.Process(stateText, true, EdgarDatasetText.FILE_NAME, "Texts");
                for (int i = 0; i < MAX_TRIALS; i++)
                {
                    if (!string.IsNullOrEmpty(stateText.FileNameToReprocess))
                    {
                        string filee = stateText.FileNameToReprocess.Split('\\').Last();
                        textParser.Process(stateText, true, stateText.FileNameToReprocess, "Texts");
                    }
                }
            }));

            //Process num file
            EdgarTaskState stateNum = new EdgarTaskState(EdgarDatasetNumber.FILE_NAME, ds, repo);

            states.Add(stateNum);
            numParser.Dimensions  = dims;
            numParser.Submissions = subs;
            numParser.Tags        = tags;
            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  numService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
            {
                bool parallel = ConfigurationManager.AppSettings["run_num_in_parallel"] == "true";
                numParser.Process(stateNum, parallel, EdgarDatasetNumber.FILE_NAME, "Numbers");
            }));

            Task.WaitAll(tasks.ToArray());
            return(states.ToArray());
        }
        public override EdgarDatasetDimension Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
        {
            string dimhash = fields[fieldNames.IndexOf("dimhash")];
            EdgarDatasetDimension dim;

            dim                  = new EdgarDatasetDimension();
            dim.DimensionH       = dimhash;
            dim.Segments         = fields[fieldNames.IndexOf("segments")];
            dim.SegmentTruncated = !(fields[fieldNames.IndexOf("segt")] == "0");
            dim.LineNumber       = lineNumber;
            return(dim);
        }
        public override EdgarDatasetSubmission Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
        {
            //Example
            //adsh	cik	name	sic	countryba	stprba	cityba	zipba	bas1	bas2	baph	countryma	stprma	cityma	zipma	mas1	mas2	countryinc	stprinc	ein	former	changed	afs	wksi	fye	form	period	fy	fp	filed	accepted	prevrpt	detail	instance	nciks	aciks	pubfloatusd	floatdate	floataxis	floatmems
            //0000002178 - 16 - 000103    2178    ADAMS RESOURCES &ENERGY, INC.  5172    US TX  HOUSTON 77027   17 S.BRIAR HOLLOW LN.      7138813600  US TX  HOUSTON 77001   P O BOX 844     US DE  741753147   ADAMS RESOURCES &ENERGY INC    19920703    2 - ACC   0   1231    10 - Q    20160930    2016    Q3  20161109    2016 - 11 - 09 12:49:00.0   0   1   ae - 20160930.xml 1

            EdgarDatasetSubmission sub;
            string adsh = fields[fieldNames.IndexOf("adsh")];

            sub      = new EdgarDatasetSubmission();
            sub.ADSH = adsh;

            sub.Registrant = ParseRegistrant(repository, fields[fieldNames.IndexOf("cik")], fieldNames, fields);

            sub.Form = repository.GetSECForm(fields[fieldNames.IndexOf("form")]);

            string period = fields[fieldNames.IndexOf("period")];

            sub.Period = new DateTime(int.Parse(period.Substring(0, 4)), int.Parse(period.Substring(4, 2)), int.Parse(period.Substring(6, 2)));

            sub.Detail = fields[fieldNames.IndexOf("period")] == "1";

            sub.XBRLInstance = fields[fieldNames.IndexOf("instance")];

            sub.NumberOfCIKs = int.Parse(fields[fieldNames.IndexOf("nciks")]);

            string value = fields[fieldNames.IndexOf("aciks")];

            sub.AdditionalCIKs = String.IsNullOrEmpty(value) ? null : value;

            value = fields[fieldNames.IndexOf("pubfloatusd")];
            sub.PublicFloatUSD = string.IsNullOrEmpty(value) ? (float?)null : float.Parse(value, CultureInfo.GetCultureInfo("en-us").NumberFormat);

            string floatdate = fields[fieldNames.IndexOf("floatdate")];

            sub.FloatDate = String.IsNullOrEmpty(floatdate) ? (DateTime?)null : new DateTime(int.Parse(floatdate.Substring(0, 4)), int.Parse(floatdate.Substring(4, 2)), int.Parse(floatdate.Substring(6, 2)));

            value         = fields[fieldNames.IndexOf("floataxis")];
            sub.FloatAxis = String.IsNullOrEmpty(value) ? null : value;

            value         = fields[fieldNames.IndexOf("floatmems")];
            sub.FloatMems = string.IsNullOrEmpty(value) ? (int?)null : int.Parse(value);

            sub.LineNumber = lineNumber;

            return(sub);
        }
        private EdgarTaskState[] LoadSubTagDim(EdgarDataset ds, IEdgarDatasetsRepository repo)
        {
            List <EdgarTaskState> states = new List <EdgarTaskState>();
            EdgarTaskState        stateSubs, stateTag, stateDim;

            stateSubs = new EdgarTaskState(EdgarDatasetSubmission.FILE_NAME, ds, repo);
            stateTag  = new EdgarTaskState(EdgarDatasetTag.FILE_NAME, ds, repo);
            stateDim  = new EdgarTaskState(EdgarDatasetDimension.FILE_NAME, ds, repo);
            states.Add(stateSubs);
            states.Add(stateTag);
            states.Add(stateDim);
            IList <Task> tasks = new List <Task>();

            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  submissionService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
                                            submissionParser.Process(stateSubs, false, EdgarDatasetSubmission.FILE_NAME, "Submissions")//false --> to avoid to have too many threads
                                            ));

            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  tagService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
            {
                if (ConfigurationManager.AppSettings["run_tag_in_parallel"] == "true")
                {
                    tagParser.Process(stateTag, true, EdgarDatasetTag.FILE_NAME, "Tags");
                }
                else
                {
                    tagParser.Process(stateTag, false, EdgarDatasetTag.FILE_NAME, "Tags");
                }
            }));

            log.Info("Datasetid " + ds.Id.ToString() + " -- starting  dimensionService.Process(...)");
            tasks.Add(Task.Factory.StartNew(() =>
            {
                dimensionParser.Process(stateDim, false, EdgarDatasetDimension.FILE_NAME, "Dimensions");
            }));

            Task.WaitAll(tasks.ToArray());
            return(states.ToArray());
        }
        private EdgarTaskState[] LoadRenPre(EdgarDataset ds, IEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> nums, ConcurrentDictionary <string, int> texts)
        {
            List <EdgarTaskState> states = new List <EdgarTaskState>();
            List <Task>           tasks  = new List <Task>();

            EdgarTaskState stateRen = new EdgarTaskState(EdgarDatasetRender.FILE_NAME, ds, repo);

            states.Add(stateRen);

            EdgarTaskState statePre = new EdgarTaskState(EdgarDatasetPresentation.FILE_NAME, ds, repo);

            states.Add(statePre);

            tasks.Add(Task.Factory.StartNew(() =>
            {
                renderingParser.Subs = subs;
                log.Info("Datasetid " + ds.Id.ToString() + " -- starting  renderingService.Process(...)");
                //Presentations has a relationship to renders
                renderingParser.Process(stateRen, true, EdgarDatasetRender.FILE_NAME, "Renders");
                presentationParser.Subs = subs;
                presentationParser.Tags = tags;
                log.Info("Datasetid " + ds.Id.ToString() + " -- loading all rens for presentationService.Process(...)");
                presentationParser.Renders = renderingParser.GetAsConcurrent(ds.Id);
                presentationParser.Nums    = nums;
                presentationParser.Texts   = texts;
                log.Info("Datasetid " + ds.Id.ToString() + " -- starting  presentationService.Process(...)");
                if (ConfigurationManager.AppSettings["run_pre_in_parallel"] == "true")
                {
                    presentationParser.Process(statePre, true, EdgarDatasetPresentation.FILE_NAME, "Presentations"); //parallel execution
                }
                else
                {
                    presentationParser.Process(statePre, false, EdgarDatasetPresentation.FILE_NAME, "Presentations");//sequential execution
                }
            }
                                            ));
            Task.WaitAll(tasks.ToArray());
            return(states.ToArray());
        }
Example #13
0
        public override EdgarDatasetTag Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int linenumber)
        {
            /*
             * File content:
             * tag	version	custom	abstract	datatype	iord	crdr	tlabel	doc
             * AccountsPayableCurrent	us-gaap/2015	0	0	monetary	I	C	Accounts Payable, Current	Carrying value as of the balance sheet date of liabilities incurred (and for which invoices have typically been received) and payable to vendors for goods and services received that are used in an entity's business. Used to reflect the current portion of the liabilities (due within one year or within the normal operating cycle if longer).
             * AccountsPayableRelatedPartiesCurrent	us-gaap/2015	0	0	monetary	I	C	Accounts Payable, Related Parties, Current	Amount for accounts payable to related parties. Used to reflect the current portion of the liabilities (due within one year or within the normal operating cycle if longer).
             * ...
             */


            string strTag  = fields[fieldNames.IndexOf("tag")];
            string version = fields[fieldNames.IndexOf("version")];

            EdgarDatasetTag tag;

            tag         = new EdgarDatasetTag();
            tag.Tag     = strTag;
            tag.Version = version;
            string value = fields[fieldNames.IndexOf("custom")];

            tag.Custom    = value == "1" ? true : false;
            value         = fields[fieldNames.IndexOf("abstract")];
            tag.Abstract  = value == "1" ? true : false;
            value         = fields[fieldNames.IndexOf("datatype")];
            tag.Datatype  = string.IsNullOrEmpty(value) ? null : value;
            value         = fields[fieldNames.IndexOf("iord")];
            tag.ValueType = string.IsNullOrEmpty(value) ? (char?)null : value[0];
            value         = fields[fieldNames.IndexOf("crdr")];
            tag.NaturalAccountingBalance = string.IsNullOrEmpty(value) ? (char?)null : value[0];
            value             = fields[fieldNames.IndexOf("tlabel")];
            tag.LabelText     = string.IsNullOrEmpty(value) ? null : value;
            value             = fields[fieldNames.IndexOf("doc")];
            tag.Documentation = string.IsNullOrEmpty(value) ? null : value;
            tag.LineNumber    = linenumber;

            return(tag);
        }
Example #14
0
 public abstract T Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber);
Example #15
0
 public EdgarService(IEdgarRepository edgarRepo, IEdgarDatasetsRepository datasetsRepo)
 {
     this.datasetsRepo = datasetsRepo;
     this.edgarRepo    = edgarRepo;
 }
 public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetText file)
 {
     repo.Add(dataset, file);
 }
Example #17
0
 public abstract IList <EdgarTuple> GetKeys(IEdgarDatasetsRepository repository, int datasetId);
Example #18
0
 public override IList <EdgarTuple> GetKeys(IEdgarDatasetsRepository repository, int datasetId)
 {
     return(repository.GetPresentationsKeys(datasetId));
 }
 public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetSubmission file)
 {
     repo.Add(dataset, file);
 }
Example #20
0
        public override EdgarDatasetPresentation Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
        {
            /*
             * adsh	report	line	stmt	inpth	rfile	tag	version	prole	plabel	negating
             * 0001163302-16-000148	1	4	CP	0	H	DocumentFiscalYearFocus	dei/2014	terseLabel	Document Fiscal Year Focus	0
             * 0001163302-16-000148	1	3	CP	0	H	DocumentPeriodEndDate	dei/2014	terseLabel	Document Period End Date	0
             * 0001125345-17-000041	6	1	CF	0	H	NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract	us-gaap/2015	terseLabel	Operating activities	0
             * 0001104659-17-016575	111	6		0	H	NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract	us-gaap/2015	terseLabel	Cash flows from operating activities:	0
             * ...
             */
            try
            {
                EdgarDatasetPresentation pre;
                string adsh   = fields[fieldNames.IndexOf("adsh")];
                string line   = fields[fieldNames.IndexOf("line")];
                string report = fields[fieldNames.IndexOf("report")];
                //select S.ADSH + CAST(p.ReportNumber as varchar) + cast(p.Line as varchar) [key] ...
                //string key = adsh + report + line;

                pre = new EdgarDatasetPresentation();
                pre.SubmissionId       = Subs[adsh];
                pre.ReportNumber       = Convert.ToInt32(report);
                pre.RenderId           = Renders[adsh + report];
                pre.Line               = Convert.ToInt32(line);
                pre.FinancialStatement = fields[fieldNames.IndexOf("stmt")];
                pre.Inpth              = fields[fieldNames.IndexOf("inpth")] == "1";
                int index = fieldNames.IndexOf("rfile");
                if (index >= 0)
                {
                    pre.RenderFile    = fields[fieldNames.IndexOf("rfile")][0];
                    pre.RenderFileStr = fields[fieldNames.IndexOf("rfile")];
                }
                else
                {
                    pre.RenderFileStr = null;
                    pre.RenderFile    = char.MinValue;
                }
                string tag     = fields[fieldNames.IndexOf("tag")];
                string version = fields[fieldNames.IndexOf("version")];
                if (Tags.ContainsKey(tag + version))
                {
                    pre.TagId = Tags[tag + version];
                }
                else
                {
                    string tag2 = Encoding.GetEncoding(1252).GetString(Encoding.GetEncoding("iso-8859-7").GetBytes(tag));
                    if (Tags.ContainsKey(tag + version))
                    {
                        pre.TagId = Tags[tag + version];
                    }
                    else
                    {
                        throw new EdgarLineException(EdgarDatasetPresentation.FILE_NAME, lineNumber, "Error retrieving key: " + tag + version);
                    }
                }
                pre.PreferredLabelXBRLLinkRole = fields[fieldNames.IndexOf("prole")];
                pre.PreferredLabel             = fields[fieldNames.IndexOf("plabel")];
                pre.Negating   = !(fields[fieldNames.IndexOf("negating")] == "0");
                pre.LineNumber = lineNumber;

                string numKey = adsh + tag + version;
                if (Nums.ContainsKey(numKey))
                {
                    pre.NumberId = Nums[numKey];
                }
                else
                {
                    pre.ADSH_Tag_Version = adsh + "|" + tag + "|" + version;
                }

                if (Texts.ContainsKey(numKey))
                {
                    pre.TextId = Texts[numKey];
                }
                else
                {
                    pre.ADSH_Tag_Version = adsh + "|" + tag + "|" + version;
                }

                return(pre);
            }
            catch (Exception ex)
            {
                throw ex;
            }
        }
 public EdgarTaskState(string processName, EdgarDataset ds, IEdgarDatasetsRepository dsRepo)
 {
     this.ds          = ds;
     this.dsRepo      = dsRepo;
     this.processName = processName;
 }
        public override EdgarDatasetNumber Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
        {
            /*
             * Ejemplo
             * adsh	tag	version	ddate	qtrs	uom	dimh	iprx	value	footnote	footlen	dimn	coreg	durp	datp	dcml
             * 0000846913-16-000146	ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsVestedAndExpectedToVestExercisableAggregateIntrinsicValue	us-gaap/2015	20160930	0	USD	0x00000000	0	0.0000		0	0		0.0	0.0	-3
             * 0000846913-16-000146	WeightedAverageNumberOfSharesRestrictedStock	us-gaap/2015	20150930	1	shares	0x00000000	0	0.0000		0	0		0.0027400255	0.0	-3
             */
            EdgarDatasetNumber number = new EdgarDatasetNumber();
            String             value  = "";

            number.LineNumber              = lineNumber;
            number.ADSH                    = fields[fieldNames.IndexOf("adsh")];
            number.TagStr                  = fields[fieldNames.IndexOf("tag")];
            number.Version                 = fields[fieldNames.IndexOf("version")];
            value                          = fields[fieldNames.IndexOf("ddate")];
            number.DatavalueEnddate        = new DateTime(int.Parse(value.Substring(0, 4)), int.Parse(value.Substring(4, 2)), int.Parse(value.Substring(6, 2)));
            value                          = fields[fieldNames.IndexOf("qtrs")];
            number.CountOfNumberOfQuarters = Convert.ToInt32(value);
            number.UnitOfMeasure           = fields[fieldNames.IndexOf("uom")];
            number.DimensionStr            = fields[fieldNames.IndexOf("dimh")];
            value                          = fields[fieldNames.IndexOf("iprx")];
            number.IPRX                    = Convert.ToInt16(value);
            value                          = fields[fieldNames.IndexOf("value")];
            if (String.IsNullOrEmpty(value))
            {
                number.Value = null;
            }
            else
            {
                number.Value = double.Parse(value, CultureInfo.GetCultureInfo("en-us").NumberFormat);
            }
            value = fields[fieldNames.IndexOf("footnote")];
            if (string.IsNullOrEmpty(value))
            {
                number.FootNote = null;
            }
            else
            {
                number.FootNote = value;
            }
            value                     = fields[fieldNames.IndexOf("footlen")];
            number.FootLength         = Convert.ToInt16(value);
            value                     = fields[fieldNames.IndexOf("dimn")];
            number.NumberOfDimensions = Convert.ToInt16(value);
            value                     = fields[fieldNames.IndexOf("coreg")];
            if (string.IsNullOrEmpty(value))
            {
                number.CoRegistrant = null;
            }
            else
            {
                number.CoRegistrant = value;
            }
            value           = fields[fieldNames.IndexOf("durp")];
            number.Durp     = float.Parse(value, CultureInfo.GetCultureInfo("en-us").NumberFormat);
            value           = fields[fieldNames.IndexOf("datp")];
            number.Datp     = float.Parse(value, CultureInfo.GetCultureInfo("en-us").NumberFormat);
            value           = fields[fieldNames.IndexOf("dcml")];
            number.Decimals = Convert.ToInt32(value);

            number.LineNumber = lineNumber;

            return(number);
        }
Example #23
0
 public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetPresentation file)
 {
     repo.Add(dataset, file);
 }
        private void ProcessDatasetTask(int id)
        {
            //https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/task-based-asynchronous-programming?view=netframework-4.5.2
            Task t = new Task(() =>
            {
                try
                {
                    using (IEdgarDatasetsRepository repository = CreateRepository())
                    {
                        EdgarDataset ds = repository.GetDataset(id);
                        if (ds != null)
                        {
                            //////////////////////////////////////////////////////////////////////////////////////////////////////////
                            ////BEGIN PROCESS
                            Stopwatch watch = System.Diagnostics.Stopwatch.StartNew();
                            log.Info("Datasetid " + id.ToString() + " -- BEGIN dataset process");

                            //Load Submissions, Tags and Dimensions
                            EdgarTaskState[] states = LoadSubTagDim(ds, repository);
                            bool ok = ManageErrors(states);
                            log.Info(string.Format("Datasetid {0} -- Variables after LoadSubTagDim(..): ManageErrors: {1}; Submissions: {2}/{3}; Tags: {4}/{5}; Dimensions: {6}/{7}", id, ok, ds.ProcessedSubmissions, ds.TotalSubmissions, ds.ProcessedTags, ds.TotalTags, ds.ProcessedDimensions, ds.TotalDimensions));
                            if (!ok || ds.ProcessedSubmissions != ds.TotalSubmissions || ds.ProcessedTags != ds.TotalTags || ds.ProcessedDimensions != ds.TotalDimensions)
                            {
                                log.Fatal("Process of sub, tags or dims failed, process can't continue");
                                return;
                            }
                            //Retrieve all tags, submissions and dimensions to fill the relationship
                            //Load Calculations, Texts and Numbers
                            log.Info("Datasetid " + id.ToString() + " -- loading all tags for LoadCalTxtNum(...)");
                            ConcurrentDictionary <string, int> tags = tagParser.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- loading all subs for LoadCalTxtNum(...)");
                            ConcurrentDictionary <string, int> subs = submissionParser.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- loading all dims for LoadCalTxtNum(...)");
                            ConcurrentDictionary <string, int> dims = dimensionParser.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- Starting LoadCalTxtNum(...)");
                            states = LoadCalTxtNum(ds, repository, subs, tags, dims);
                            ok     = ManageErrors(states);
                            log.Info(string.Format("Datasetid {0} -- Variables after LoadCalTxtNum(..): ManageErrors: {1}; Calculations: {2}/{3}; Texts: {4}/{5}; Numbers: {6}/{7}", id, ok, ds.ProcessedCalculations, ds.TotalCalculations, ds.ProcessedTexts, ds.TotalTexts, ds.ProcessedNumbers, ds.TotalNumbers));
                            if (!ok || ds.ProcessedCalculations != ds.TotalCalculations || ds.ProcessedTexts != ds.TotalTexts || ds.ProcessedNumbers != ds.TotalNumbers)
                            {
                                log.Fatal("Process of cal, text or nums failed, process can't continue");
                                return;
                            }
                            log.Info("Datasetid " + id.ToString() + " -- releasing dims");
                            dims.Clear();
                            dims = null;

                            //Load Presentations and Renders
                            log.Info("Datasetid " + id.ToString() + " -- loading all nums for LoadRenPre(...)");
                            ConcurrentDictionary <string, int> nums = numParser.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- loading all txt for LoadRenPre(...)");
                            ConcurrentDictionary <string, int> txts = textParser.GetAsConcurrent(id);
                            log.Info("Datasetid " + id.ToString() + " -- Starting LoadRenPre(...)");
                            states = LoadRenPre(ds, repository, subs, tags, nums, txts);
                            ManageErrors(states);
                            log.Info(string.Format("Datasetid {0} -- Variables after LoadRenPre(..): ManageErrors: {1}; Reners: {2}/{3}; Presentations: {4}/{5}", id, ok, ds.ProcessedRenders, ds.TotalRenders, ds.ProcessedPresentations, ds.TotalPresentations));
                            ////END PROCESS
                            //////////////////////////////////////////////////////////////////////////////////////////////////////////
                            subs.Clear(); subs = null;
                            tags.Clear(); tags = null;
                            nums.Clear(); nums = null;
                            txts.Clear(); txts = null;
                            GC.Collect();//force GC

                            watch.Stop();
                            TimeSpan ts        = watch.Elapsed;
                            string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
                            log.Info("Datasetid " + id.ToString() + " -- END dataset process -- time: " + elapsedTime);
                        }
                        else
                        {
                            log.Fatal("Datasetid " + id.ToString() + " does not exists");
                        }
                    }
                }
                catch (Exception ex)
                {
                    log.Fatal("Datasetid " + id.ToString() + " -- Unexpected error in EdgarDatasetService.Run(" + id.ToString() + "): " + ex.Message, ex);
                    log.FatalInner(ex.InnerException);
                }
            });

            t.Start();
            datasetsInProcess.TryAdd(id, t);
        }
 public override IList <EdgarTuple> GetKeys(IEdgarDatasetsRepository repository, int datasetId)
 {
     return(repository.GetNumberKeys(datasetId));
 }
Example #26
0
 public abstract void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, T file);