public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetNumber file)
{
file.SubmissionId = Submissions[file.ADSH];
file.TagId = Tags[file.TagCompoundKey];
file.DimensionId = Dimensions[file.DimensionStr];
repo.Add(dataset, file);
}
private Registrant ParseRegistrant(IEdgarDatasetsRepository repository, string cik, List <string> fieldNames, List <string> fields)
{
Registrant r = repository.GetRegistrant(cik);
if (r == null)
{
r = new Registrant();
r.CIK = int.Parse(cik);
r.Name = fields[fieldNames.IndexOf("name")];
string value = fields[fieldNames.IndexOf("sic")];
r.SIC = string.IsNullOrEmpty(value) ? null : repository.GetSIC(value);
value = fields[fieldNames.IndexOf("countryba")];
r.CountryBA = String.IsNullOrEmpty(value) ? null : value;
value = fields[fieldNames.IndexOf("cityba")];
r.CityBA = String.IsNullOrEmpty(value) ? null : value;
value = fields[fieldNames.IndexOf("countryinc")];
r.CountryInc = String.IsNullOrEmpty(value) ? null : value;
value = fields[fieldNames.IndexOf("ein")];
r.EmployeeIdentificationNumber = string.IsNullOrEmpty(value) ? (int?)null : int.Parse(value);
r.FilerStatus = fields[fieldNames.IndexOf("afs")];
r.WellKnownSeasonedIssuer = fields[fieldNames.IndexOf("wksi")] == "1";
value = fields[fieldNames.IndexOf("fye")];
if (string.IsNullOrEmpty(value))
{
r.FiscalYearEndDate = null;
}
else
{
r.FiscalYearEndDate = Convert.ToInt16(value);
}
repository.Add(r);
}
return(r);
}
public override EdgarDatasetCalculation Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
{
try
{
EdgarDatasetCalculation calc = new EdgarDatasetCalculation();
string adsh = fields[fieldNames.IndexOf("adsh")];
calc.SubmissionId = Submissions[adsh];
calc.SequentialNumberForGrouping = Convert.ToInt16(fields[fieldNames.IndexOf("grp")]);
calc.SequentialNumberForArc = Convert.ToInt16(fields[fieldNames.IndexOf("arc")]);
string pTag = fields[fieldNames.IndexOf("ptag")];
string pVersion = fields[fieldNames.IndexOf("pversion")];
calc.ParentTagId = Tags[pTag + pVersion];
//Indicates a weight of -1 (TRUE if the arc is negative), but typically +1 (FALSE).
calc.Negative = fields[fieldNames.IndexOf("negative")] == "-1" ? true : false;
string cTag = fields[fieldNames.IndexOf("ctag")];
string cVersion = fields[fieldNames.IndexOf("cversion")];
calc.ChildTagId = Tags[cTag + cVersion];
calc.LineNumber = lineNumber;
return(calc);
}
catch (Exception ex)
{
throw ex;
}
}
public override EdgarDatasetRender Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
{
/*
* adsh report rfile menucat shortname longname roleuri parentroleuri parentreport ultparentrpt
* 0001163302-16-000148 1 H Cover Document and Entity Information 0001000 - Document - Document and Entity Information http://www.ussteel.com/role/DocumentAndEntityInformation
* 0001163302-16-000148 2 H Statements Consolidated Statement Of Operations 1001000 - Statement - Consolidated Statement Of Operations http://www.ussteel.com/role/ConsolidatedStatementOfOperations
*/
EdgarDatasetRender ren = new EdgarDatasetRender();
string adsh = fields[fieldNames.IndexOf("adsh")];
ren.SubmissionId = Subs[adsh];
ren.Report = Convert.ToInt32(fields[fieldNames.IndexOf("report")]);
string value = "";
ren.RenderFile = fields[fieldNames.IndexOf("rfile")][0];
ren.RenderFileStr = ren.RenderFile.ToString();
ren.MenuCategory = fields[fieldNames.IndexOf("menucat")];
ren.ShortName = fields[fieldNames.IndexOf("shortname")];
ren.LongName = fields[fieldNames.IndexOf("longname")];
ren.RoleURI = fields[fieldNames.IndexOf("roleuri")];
ren.ParentRoleURI = fields[fieldNames.IndexOf("parentroleuri")];
value = fields[fieldNames.IndexOf("parentreport")];
if (!string.IsNullOrEmpty(value))
{
ren.ParentReport = Convert.ToInt32(value);
}
value = fields[fieldNames.IndexOf("ultparentrpt")];
if (!string.IsNullOrEmpty(value))
{
ren.UltimateParentReport = Convert.ToInt32(value);
}
ren.LineNumber = lineNumber;
return(ren);
}
public IList <EdgarDataset> GetDatasets()
{
using (IEdgarDatasetsRepository repository = CreateRepository())
{
IList <EdgarDataset> datasets = repository.Get <EdgarDataset>();
return(datasets);
}
}
public EdgarDataset GetDataset(int id)
{
using (IEdgarDatasetsRepository repository = CreateRepository())
{
EdgarDataset ds = repository.GetDataset(id);
return(ds);
}
}
public override EdgarDatasetText Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
{
/*
* adsh tag version ddate qtrs iprx lang dcml durp datp dimh dimn coreg escaped srclen txtlen footnote footlen context value
* 0001163302-16-000148 AssetRetirementObligationDisclosureTextBlock us-gaap/2016 20160930 3 0 en-US 32767 0.008219957 0.0 0x00000000 0 1 17336 907 0 FD2016Q3YTD Asset Retirement Obligations U. S. Steels asset retirement obligations (AROs) primarily relate to mine and landfill closure and post-closure costs. The following table reflects changes in the carrying values of AROs: (In millions) September�30, 2016 December�31, 2015 Balance at beginning of year $ 89 $ 48 Additional obligations incurred 3 45 (a) Obligations settled (9 ) (6 ) Foreign currency translation effects (1 ) Accretion expense 2 3 Balance at end of period $ 85 $ 89 (a) Additional AROs relate to the permanent closure of the coke production facilities at Gary Works and Granite City Works. Certain AROs related to disposal costs of the majority of fixed assets at our integrated steel facilities have not been recorded because they have an indeterminate settlement date. These AROs will be initially recognized in the period in which sufficient information exists to estimate their fair value.
* 0001163302-16-000148 BasisOfPresentationAndSignificantAccountingPoliciesTextBlock us-gaap/2016 20160930 3 0 en-US 32767 0.008219957 0.0 0x00000000 0 1 3331 2042 0 FD2016Q3YTD Basis of Presentation and Significant Accounting Policies United States Steel Corporation produces and sells steel products, including flat-rolled and tubular products, in North America and Central Europe. Operations in North America also include iron ore and coke production facilities, railroad services and real estate operations. Operations in Europe also include coke production facilities. The year-end Consolidated Balance Sheet data was derived from audited statements but does not include all disclosures required for complete financial statements by accounting principles generally accepted in the United States of America (U.S. GAAP). The other information in these financial statements is unaudited but, in the opinion of management, reflects all adjustments necessary for a fair statement of the results for the periods covered. All such adjustments are of a normal recurring nature unless disclosed otherwise. These financial statements, including notes, have been prepared in accordance with the applicable rules of the Securities and Exchange Commission and do not include all of the information and disclosures required by U.S. GAAP for complete financial statements. Additional information is contained in the United States Steel Corporation Annual Report on Form�10-K for the fiscal year ended December�31, 2015 , which should be read in conjunction with these financial statements. Revision of Prior Period Financial Statements During 2015, the Company identified a prior period error related to the classification of unpaid capital expenditures in the Consolidated Statements of Cash Flows that impacted the quarterly interim financial statements in 2015. As a result, the Consolidated Statement of Cash Flows for the the nine months ended September 30, 2015 has been revised to reflect a decrease in cash provided by operating activities and a decrease in cash used in investing activities of $55 million . The Company has concluded the impact of this error was not material to the previously filed financial statements.
*/
//aca puede iterar sobre la coleccion como en el proyecto sec q db
EdgarDatasetText text = new EdgarDatasetText();
string adsh = fields[fieldNames.IndexOf("adsh")];
text.SubmissionId = Submissions[adsh];
string tag = fields[fieldNames.IndexOf("tag")];
string version = fields[fieldNames.IndexOf("version")];
text.TagId = Tags[tag + version];
string value = fields[fieldNames.IndexOf("ddate")];
text.DatavalueEnddate = new DateTime(int.Parse(value.Substring(0, 4)), int.Parse(value.Substring(4, 2)), int.Parse(value.Substring(6, 2)));
text.CountOfNumberOfQuarters = Convert.ToInt32(fields[fieldNames.IndexOf("qtrs")]);
text.Iprx = Convert.ToInt16(fields[fieldNames.IndexOf("iprx")]);
text.Language = fields[fieldNames.IndexOf("lang")];
text.Dcml = Convert.ToInt32(fields[fieldNames.IndexOf("dcml")]);
text.Durp = float.Parse(fields[fieldNames.IndexOf("durp")], CultureInfo.GetCultureInfo("en-us").NumberFormat);
text.Datp = float.Parse(fields[fieldNames.IndexOf("datp")], CultureInfo.GetCultureInfo("en-us").NumberFormat);
text.DimensionId = Dimensions[fields[fieldNames.IndexOf("dimh")]];
value = fields[fieldNames.IndexOf("dimn")];
if (!string.IsNullOrEmpty(value))
{
text.DimensionNumber = Convert.ToInt16(value);
}
value = fields[fieldNames.IndexOf("coreg")];
if (string.IsNullOrEmpty(value))
{
text.CoRegistrant = null;
}
else
{
text.CoRegistrant = value;
}
text.Escaped = fields[fieldNames.IndexOf("escaped")] != "0";
text.SourceLength = Convert.ToInt32(fields[fieldNames.IndexOf("srclen")]);
text.TextLength = Convert.ToInt32(fields[fieldNames.IndexOf("txtlen")]);
text.FootNote = fields[fieldNames.IndexOf("footnote")];
value = fields[fieldNames.IndexOf("footlen")];
if (!string.IsNullOrEmpty(value))
{
text.FootLength = Convert.ToInt32(value);
}
;
text.Context = fields[fieldNames.IndexOf("context")];
text.Value = fields[fieldNames.IndexOf("value")];
text.LineNumber = lineNumber;
return(text);
}
private EdgarTaskState[] LoadCalTxtNum(EdgarDataset ds, IEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> dims)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
List <Task> tasks = new List <Task>();
//process calc file
EdgarTaskState stateCalc = new EdgarTaskState(EdgarDatasetCalculation.FILE_NAME, ds, repo);
states.Add(stateCalc);
calcParser.Submissions = subs;
calcParser.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting calcService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
calcParser.Process(stateCalc, false, EdgarDatasetCalculation.FILE_NAME, "Calculations")) //false --> to avoid to have too many threads
);
//process text file
EdgarTaskState stateText = new EdgarTaskState(EdgarDatasetText.FILE_NAME, ds, repo);
states.Add(stateText);
textParser.Dimensions = dims;
textParser.Submissions = subs;
textParser.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting textService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
textParser.Process(stateText, true, EdgarDatasetText.FILE_NAME, "Texts");
for (int i = 0; i < MAX_TRIALS; i++)
{
if (!string.IsNullOrEmpty(stateText.FileNameToReprocess))
{
string filee = stateText.FileNameToReprocess.Split('\\').Last();
textParser.Process(stateText, true, stateText.FileNameToReprocess, "Texts");
}
}
}));
//Process num file
EdgarTaskState stateNum = new EdgarTaskState(EdgarDatasetNumber.FILE_NAME, ds, repo);
states.Add(stateNum);
numParser.Dimensions = dims;
numParser.Submissions = subs;
numParser.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting numService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
bool parallel = ConfigurationManager.AppSettings["run_num_in_parallel"] == "true";
numParser.Process(stateNum, parallel, EdgarDatasetNumber.FILE_NAME, "Numbers");
}));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
public override EdgarDatasetDimension Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
{
string dimhash = fields[fieldNames.IndexOf("dimhash")];
EdgarDatasetDimension dim;
dim = new EdgarDatasetDimension();
dim.DimensionH = dimhash;
dim.Segments = fields[fieldNames.IndexOf("segments")];
dim.SegmentTruncated = !(fields[fieldNames.IndexOf("segt")] == "0");
dim.LineNumber = lineNumber;
return(dim);
}
public override EdgarDatasetSubmission Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
{
//Example
//adsh cik name sic countryba stprba cityba zipba bas1 bas2 baph countryma stprma cityma zipma mas1 mas2 countryinc stprinc ein former changed afs wksi fye form period fy fp filed accepted prevrpt detail instance nciks aciks pubfloatusd floatdate floataxis floatmems
//0000002178 - 16 - 000103 2178 ADAMS RESOURCES &ENERGY, INC. 5172 US TX HOUSTON 77027 17 S.BRIAR HOLLOW LN. 7138813600 US TX HOUSTON 77001 P O BOX 844 US DE 741753147 ADAMS RESOURCES &ENERGY INC 19920703 2 - ACC 0 1231 10 - Q 20160930 2016 Q3 20161109 2016 - 11 - 09 12:49:00.0 0 1 ae - 20160930.xml 1
EdgarDatasetSubmission sub;
string adsh = fields[fieldNames.IndexOf("adsh")];
sub = new EdgarDatasetSubmission();
sub.ADSH = adsh;
sub.Registrant = ParseRegistrant(repository, fields[fieldNames.IndexOf("cik")], fieldNames, fields);
sub.Form = repository.GetSECForm(fields[fieldNames.IndexOf("form")]);
string period = fields[fieldNames.IndexOf("period")];
sub.Period = new DateTime(int.Parse(period.Substring(0, 4)), int.Parse(period.Substring(4, 2)), int.Parse(period.Substring(6, 2)));
sub.Detail = fields[fieldNames.IndexOf("period")] == "1";
sub.XBRLInstance = fields[fieldNames.IndexOf("instance")];
sub.NumberOfCIKs = int.Parse(fields[fieldNames.IndexOf("nciks")]);
string value = fields[fieldNames.IndexOf("aciks")];
sub.AdditionalCIKs = String.IsNullOrEmpty(value) ? null : value;
value = fields[fieldNames.IndexOf("pubfloatusd")];
sub.PublicFloatUSD = string.IsNullOrEmpty(value) ? (float?)null : float.Parse(value, CultureInfo.GetCultureInfo("en-us").NumberFormat);
string floatdate = fields[fieldNames.IndexOf("floatdate")];
sub.FloatDate = String.IsNullOrEmpty(floatdate) ? (DateTime?)null : new DateTime(int.Parse(floatdate.Substring(0, 4)), int.Parse(floatdate.Substring(4, 2)), int.Parse(floatdate.Substring(6, 2)));
value = fields[fieldNames.IndexOf("floataxis")];
sub.FloatAxis = String.IsNullOrEmpty(value) ? null : value;
value = fields[fieldNames.IndexOf("floatmems")];
sub.FloatMems = string.IsNullOrEmpty(value) ? (int?)null : int.Parse(value);
sub.LineNumber = lineNumber;
return(sub);
}
private EdgarTaskState[] LoadSubTagDim(EdgarDataset ds, IEdgarDatasetsRepository repo)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
EdgarTaskState stateSubs, stateTag, stateDim;
stateSubs = new EdgarTaskState(EdgarDatasetSubmission.FILE_NAME, ds, repo);
stateTag = new EdgarTaskState(EdgarDatasetTag.FILE_NAME, ds, repo);
stateDim = new EdgarTaskState(EdgarDatasetDimension.FILE_NAME, ds, repo);
states.Add(stateSubs);
states.Add(stateTag);
states.Add(stateDim);
IList <Task> tasks = new List <Task>();
log.Info("Datasetid " + ds.Id.ToString() + " -- starting submissionService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
submissionParser.Process(stateSubs, false, EdgarDatasetSubmission.FILE_NAME, "Submissions")//false --> to avoid to have too many threads
));
log.Info("Datasetid " + ds.Id.ToString() + " -- starting tagService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
if (ConfigurationManager.AppSettings["run_tag_in_parallel"] == "true")
{
tagParser.Process(stateTag, true, EdgarDatasetTag.FILE_NAME, "Tags");
}
else
{
tagParser.Process(stateTag, false, EdgarDatasetTag.FILE_NAME, "Tags");
}
}));
log.Info("Datasetid " + ds.Id.ToString() + " -- starting dimensionService.Process(...)");
tasks.Add(Task.Factory.StartNew(() =>
{
dimensionParser.Process(stateDim, false, EdgarDatasetDimension.FILE_NAME, "Dimensions");
}));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
private EdgarTaskState[] LoadRenPre(EdgarDataset ds, IEdgarDatasetsRepository repo, ConcurrentDictionary <string, int> subs, ConcurrentDictionary <string, int> tags, ConcurrentDictionary <string, int> nums, ConcurrentDictionary <string, int> texts)
{
List <EdgarTaskState> states = new List <EdgarTaskState>();
List <Task> tasks = new List <Task>();
EdgarTaskState stateRen = new EdgarTaskState(EdgarDatasetRender.FILE_NAME, ds, repo);
states.Add(stateRen);
EdgarTaskState statePre = new EdgarTaskState(EdgarDatasetPresentation.FILE_NAME, ds, repo);
states.Add(statePre);
tasks.Add(Task.Factory.StartNew(() =>
{
renderingParser.Subs = subs;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting renderingService.Process(...)");
//Presentations has a relationship to renders
renderingParser.Process(stateRen, true, EdgarDatasetRender.FILE_NAME, "Renders");
presentationParser.Subs = subs;
presentationParser.Tags = tags;
log.Info("Datasetid " + ds.Id.ToString() + " -- loading all rens for presentationService.Process(...)");
presentationParser.Renders = renderingParser.GetAsConcurrent(ds.Id);
presentationParser.Nums = nums;
presentationParser.Texts = texts;
log.Info("Datasetid " + ds.Id.ToString() + " -- starting presentationService.Process(...)");
if (ConfigurationManager.AppSettings["run_pre_in_parallel"] == "true")
{
presentationParser.Process(statePre, true, EdgarDatasetPresentation.FILE_NAME, "Presentations"); //parallel execution
}
else
{
presentationParser.Process(statePre, false, EdgarDatasetPresentation.FILE_NAME, "Presentations");//sequential execution
}
}
));
Task.WaitAll(tasks.ToArray());
return(states.ToArray());
}
public override EdgarDatasetTag Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int linenumber)
{
/*
* File content:
* tag version custom abstract datatype iord crdr tlabel doc
* AccountsPayableCurrent us-gaap/2015 0 0 monetary I C Accounts Payable, Current Carrying value as of the balance sheet date of liabilities incurred (and for which invoices have typically been received) and payable to vendors for goods and services received that are used in an entity's business. Used to reflect the current portion of the liabilities (due within one year or within the normal operating cycle if longer).
* AccountsPayableRelatedPartiesCurrent us-gaap/2015 0 0 monetary I C Accounts Payable, Related Parties, Current Amount for accounts payable to related parties. Used to reflect the current portion of the liabilities (due within one year or within the normal operating cycle if longer).
* ...
*/
string strTag = fields[fieldNames.IndexOf("tag")];
string version = fields[fieldNames.IndexOf("version")];
EdgarDatasetTag tag;
tag = new EdgarDatasetTag();
tag.Tag = strTag;
tag.Version = version;
string value = fields[fieldNames.IndexOf("custom")];
tag.Custom = value == "1" ? true : false;
value = fields[fieldNames.IndexOf("abstract")];
tag.Abstract = value == "1" ? true : false;
value = fields[fieldNames.IndexOf("datatype")];
tag.Datatype = string.IsNullOrEmpty(value) ? null : value;
value = fields[fieldNames.IndexOf("iord")];
tag.ValueType = string.IsNullOrEmpty(value) ? (char?)null : value[0];
value = fields[fieldNames.IndexOf("crdr")];
tag.NaturalAccountingBalance = string.IsNullOrEmpty(value) ? (char?)null : value[0];
value = fields[fieldNames.IndexOf("tlabel")];
tag.LabelText = string.IsNullOrEmpty(value) ? null : value;
value = fields[fieldNames.IndexOf("doc")];
tag.Documentation = string.IsNullOrEmpty(value) ? null : value;
tag.LineNumber = linenumber;
return(tag);
}
public abstract T Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber);
public EdgarService(IEdgarRepository edgarRepo, IEdgarDatasetsRepository datasetsRepo)
{
this.datasetsRepo = datasetsRepo;
this.edgarRepo = edgarRepo;
}
public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetText file)
{
repo.Add(dataset, file);
}
public abstract IList <EdgarTuple> GetKeys(IEdgarDatasetsRepository repository, int datasetId);
public override IList <EdgarTuple> GetKeys(IEdgarDatasetsRepository repository, int datasetId)
{
return(repository.GetPresentationsKeys(datasetId));
}
public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetSubmission file)
{
repo.Add(dataset, file);
}
public override EdgarDatasetPresentation Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
{
/*
* adsh report line stmt inpth rfile tag version prole plabel negating
* 0001163302-16-000148 1 4 CP 0 H DocumentFiscalYearFocus dei/2014 terseLabel Document Fiscal Year Focus 0
* 0001163302-16-000148 1 3 CP 0 H DocumentPeriodEndDate dei/2014 terseLabel Document Period End Date 0
* 0001125345-17-000041 6 1 CF 0 H NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract us-gaap/2015 terseLabel Operating activities 0
* 0001104659-17-016575 111 6 0 H NetCashProvidedByUsedInOperatingActivitiesContinuingOperationsAbstract us-gaap/2015 terseLabel Cash flows from operating activities: 0
* ...
*/
try
{
EdgarDatasetPresentation pre;
string adsh = fields[fieldNames.IndexOf("adsh")];
string line = fields[fieldNames.IndexOf("line")];
string report = fields[fieldNames.IndexOf("report")];
//select S.ADSH + CAST(p.ReportNumber as varchar) + cast(p.Line as varchar) [key] ...
//string key = adsh + report + line;
pre = new EdgarDatasetPresentation();
pre.SubmissionId = Subs[adsh];
pre.ReportNumber = Convert.ToInt32(report);
pre.RenderId = Renders[adsh + report];
pre.Line = Convert.ToInt32(line);
pre.FinancialStatement = fields[fieldNames.IndexOf("stmt")];
pre.Inpth = fields[fieldNames.IndexOf("inpth")] == "1";
int index = fieldNames.IndexOf("rfile");
if (index >= 0)
{
pre.RenderFile = fields[fieldNames.IndexOf("rfile")][0];
pre.RenderFileStr = fields[fieldNames.IndexOf("rfile")];
}
else
{
pre.RenderFileStr = null;
pre.RenderFile = char.MinValue;
}
string tag = fields[fieldNames.IndexOf("tag")];
string version = fields[fieldNames.IndexOf("version")];
if (Tags.ContainsKey(tag + version))
{
pre.TagId = Tags[tag + version];
}
else
{
string tag2 = Encoding.GetEncoding(1252).GetString(Encoding.GetEncoding("iso-8859-7").GetBytes(tag));
if (Tags.ContainsKey(tag + version))
{
pre.TagId = Tags[tag + version];
}
else
{
throw new EdgarLineException(EdgarDatasetPresentation.FILE_NAME, lineNumber, "Error retrieving key: " + tag + version);
}
}
pre.PreferredLabelXBRLLinkRole = fields[fieldNames.IndexOf("prole")];
pre.PreferredLabel = fields[fieldNames.IndexOf("plabel")];
pre.Negating = !(fields[fieldNames.IndexOf("negating")] == "0");
pre.LineNumber = lineNumber;
string numKey = adsh + tag + version;
if (Nums.ContainsKey(numKey))
{
pre.NumberId = Nums[numKey];
}
else
{
pre.ADSH_Tag_Version = adsh + "|" + tag + "|" + version;
}
if (Texts.ContainsKey(numKey))
{
pre.TextId = Texts[numKey];
}
else
{
pre.ADSH_Tag_Version = adsh + "|" + tag + "|" + version;
}
return(pre);
}
catch (Exception ex)
{
throw ex;
}
}
public EdgarTaskState(string processName, EdgarDataset ds, IEdgarDatasetsRepository dsRepo)
{
this.ds = ds;
this.dsRepo = dsRepo;
this.processName = processName;
}
public override EdgarDatasetNumber Parse(IEdgarDatasetsRepository repository, List <string> fieldNames, List <string> fields, int lineNumber)
{
/*
* Ejemplo
* adsh tag version ddate qtrs uom dimh iprx value footnote footlen dimn coreg durp datp dcml
* 0000846913-16-000146 ShareBasedCompensationArrangementByShareBasedPaymentAwardOptionsVestedAndExpectedToVestExercisableAggregateIntrinsicValue us-gaap/2015 20160930 0 USD 0x00000000 0 0.0000 0 0 0.0 0.0 -3
* 0000846913-16-000146 WeightedAverageNumberOfSharesRestrictedStock us-gaap/2015 20150930 1 shares 0x00000000 0 0.0000 0 0 0.0027400255 0.0 -3
*/
EdgarDatasetNumber number = new EdgarDatasetNumber();
String value = "";
number.LineNumber = lineNumber;
number.ADSH = fields[fieldNames.IndexOf("adsh")];
number.TagStr = fields[fieldNames.IndexOf("tag")];
number.Version = fields[fieldNames.IndexOf("version")];
value = fields[fieldNames.IndexOf("ddate")];
number.DatavalueEnddate = new DateTime(int.Parse(value.Substring(0, 4)), int.Parse(value.Substring(4, 2)), int.Parse(value.Substring(6, 2)));
value = fields[fieldNames.IndexOf("qtrs")];
number.CountOfNumberOfQuarters = Convert.ToInt32(value);
number.UnitOfMeasure = fields[fieldNames.IndexOf("uom")];
number.DimensionStr = fields[fieldNames.IndexOf("dimh")];
value = fields[fieldNames.IndexOf("iprx")];
number.IPRX = Convert.ToInt16(value);
value = fields[fieldNames.IndexOf("value")];
if (String.IsNullOrEmpty(value))
{
number.Value = null;
}
else
{
number.Value = double.Parse(value, CultureInfo.GetCultureInfo("en-us").NumberFormat);
}
value = fields[fieldNames.IndexOf("footnote")];
if (string.IsNullOrEmpty(value))
{
number.FootNote = null;
}
else
{
number.FootNote = value;
}
value = fields[fieldNames.IndexOf("footlen")];
number.FootLength = Convert.ToInt16(value);
value = fields[fieldNames.IndexOf("dimn")];
number.NumberOfDimensions = Convert.ToInt16(value);
value = fields[fieldNames.IndexOf("coreg")];
if (string.IsNullOrEmpty(value))
{
number.CoRegistrant = null;
}
else
{
number.CoRegistrant = value;
}
value = fields[fieldNames.IndexOf("durp")];
number.Durp = float.Parse(value, CultureInfo.GetCultureInfo("en-us").NumberFormat);
value = fields[fieldNames.IndexOf("datp")];
number.Datp = float.Parse(value, CultureInfo.GetCultureInfo("en-us").NumberFormat);
value = fields[fieldNames.IndexOf("dcml")];
number.Decimals = Convert.ToInt32(value);
number.LineNumber = lineNumber;
return(number);
}
public override void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, EdgarDatasetPresentation file)
{
repo.Add(dataset, file);
}
private void ProcessDatasetTask(int id)
{
//https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/task-based-asynchronous-programming?view=netframework-4.5.2
Task t = new Task(() =>
{
try
{
using (IEdgarDatasetsRepository repository = CreateRepository())
{
EdgarDataset ds = repository.GetDataset(id);
if (ds != null)
{
//////////////////////////////////////////////////////////////////////////////////////////////////////////
////BEGIN PROCESS
Stopwatch watch = System.Diagnostics.Stopwatch.StartNew();
log.Info("Datasetid " + id.ToString() + " -- BEGIN dataset process");
//Load Submissions, Tags and Dimensions
EdgarTaskState[] states = LoadSubTagDim(ds, repository);
bool ok = ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadSubTagDim(..): ManageErrors: {1}; Submissions: {2}/{3}; Tags: {4}/{5}; Dimensions: {6}/{7}", id, ok, ds.ProcessedSubmissions, ds.TotalSubmissions, ds.ProcessedTags, ds.TotalTags, ds.ProcessedDimensions, ds.TotalDimensions));
if (!ok || ds.ProcessedSubmissions != ds.TotalSubmissions || ds.ProcessedTags != ds.TotalTags || ds.ProcessedDimensions != ds.TotalDimensions)
{
log.Fatal("Process of sub, tags or dims failed, process can't continue");
return;
}
//Retrieve all tags, submissions and dimensions to fill the relationship
//Load Calculations, Texts and Numbers
log.Info("Datasetid " + id.ToString() + " -- loading all tags for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> tags = tagParser.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all subs for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> subs = submissionParser.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all dims for LoadCalTxtNum(...)");
ConcurrentDictionary <string, int> dims = dimensionParser.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- Starting LoadCalTxtNum(...)");
states = LoadCalTxtNum(ds, repository, subs, tags, dims);
ok = ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadCalTxtNum(..): ManageErrors: {1}; Calculations: {2}/{3}; Texts: {4}/{5}; Numbers: {6}/{7}", id, ok, ds.ProcessedCalculations, ds.TotalCalculations, ds.ProcessedTexts, ds.TotalTexts, ds.ProcessedNumbers, ds.TotalNumbers));
if (!ok || ds.ProcessedCalculations != ds.TotalCalculations || ds.ProcessedTexts != ds.TotalTexts || ds.ProcessedNumbers != ds.TotalNumbers)
{
log.Fatal("Process of cal, text or nums failed, process can't continue");
return;
}
log.Info("Datasetid " + id.ToString() + " -- releasing dims");
dims.Clear();
dims = null;
//Load Presentations and Renders
log.Info("Datasetid " + id.ToString() + " -- loading all nums for LoadRenPre(...)");
ConcurrentDictionary <string, int> nums = numParser.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- loading all txt for LoadRenPre(...)");
ConcurrentDictionary <string, int> txts = textParser.GetAsConcurrent(id);
log.Info("Datasetid " + id.ToString() + " -- Starting LoadRenPre(...)");
states = LoadRenPre(ds, repository, subs, tags, nums, txts);
ManageErrors(states);
log.Info(string.Format("Datasetid {0} -- Variables after LoadRenPre(..): ManageErrors: {1}; Reners: {2}/{3}; Presentations: {4}/{5}", id, ok, ds.ProcessedRenders, ds.TotalRenders, ds.ProcessedPresentations, ds.TotalPresentations));
////END PROCESS
//////////////////////////////////////////////////////////////////////////////////////////////////////////
subs.Clear(); subs = null;
tags.Clear(); tags = null;
nums.Clear(); nums = null;
txts.Clear(); txts = null;
GC.Collect();//force GC
watch.Stop();
TimeSpan ts = watch.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
log.Info("Datasetid " + id.ToString() + " -- END dataset process -- time: " + elapsedTime);
}
else
{
log.Fatal("Datasetid " + id.ToString() + " does not exists");
}
}
}
catch (Exception ex)
{
log.Fatal("Datasetid " + id.ToString() + " -- Unexpected error in EdgarDatasetService.Run(" + id.ToString() + "): " + ex.Message, ex);
log.FatalInner(ex.InnerException);
}
});
t.Start();
datasetsInProcess.TryAdd(id, t);
}
public override IList <EdgarTuple> GetKeys(IEdgarDatasetsRepository repository, int datasetId)
{
return(repository.GetNumberKeys(datasetId));
}
public abstract void Add(IEdgarDatasetsRepository repo, EdgarDataset dataset, T file);