/// <summary> /// Get a list of string dates less that the given date. /// </summary> /// <param name="taxonomyId">Taxonomy identifier.</param> /// <param name="entityId">Entity identifier.</param> /// <param name="maxDate">Date to compare.</param> /// <param name="abaxXBRLCellStoreMongo">Data access object</param> /// <returns>List with the string dates.</returns> public static async Task <IList <string> > FindLowestDatesAsync(string taxonomyId, string entityId, DateTime maxDate, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo) { var db = abaxXBRLCellStoreMongo.GetMongoDB(); var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME); var match = new BsonDocument { { "dimensionMap.taxonomyId", taxonomyId }, { "dimensionMap.instanceDocmentEntity", entityId } }; var groupDistinctDates = new BsonDocument { { "_id", "$dimensionMap.instanceDocmentReportedDate" } }; var projectParseDate = new BsonDocument { { "_id", "$_id" }, { "date", new BsonDocument { { "$dateFromString", new BsonDocument { { "dateString", "$_id" } } } } } }; FilterDefinition <BsonDocument> matchLessDate = "{\"date\":{\"$lt\":" + ModeloBase.ParseJson(maxDate) + "}}"; var aggregate = collection.Aggregate(). Match(match).Group(groupDistinctDates).Project(projectParseDate).Match(matchLessDate); var documentsList = await aggregate.ToListAsync(); var listDates = new List <string>(); foreach (var document in documentsList) { listDates.Add(document["_id"].AsString); } return(listDates); }
/// <summary> /// Checks if the send exists in the data base. /// </summary> /// <param name="reportId">Identifier of the send.</param> /// <param name="abaxXBRLCellStoreMongo">Data access object.</param> /// <returns>If the send exists.</returns> public static BsonDocument GetRecentlyReportSend(string reportId, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo) { var db = abaxXBRLCellStoreMongo.GetMongoDB(); var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME); var match = new BsonDocument { { "reportId", reportId }, { "isReplaced", false } }; var groupDistinctDates = new BsonDocument { { "_id", "$registrationDate" } }; var groupMaxDate = new BsonDocument { { "_id", "_id" }, { "maxDate", new BsonDocument { { "$max", "$_id" } } } }; var projectAsStrig = new BsonDocument { { "_id", 0 }, { "maxDate", new BsonDocument { { "$dateToString", new BsonDocument { { "date", "$maxDate" }, { "format", "%Y-%m-%dT%H:%M:%S.%LZ" } } } } } }; var aggregate = collection.Aggregate().Match(match). Group(groupDistinctDates).Group(groupMaxDate).Project(projectAsStrig); return(aggregate.FirstOrDefault()); }
/// <summary> /// Finds the max reported date of the entity. /// </summary> /// <param name="taxonomyId">Taxonomy reported.</param> /// <param name="entityId">Entity identifier.</param> /// <param name="abaxXBRLCellStoreMongo">Data access object.</param> /// <returns>Date found or null if not date was found.</returns> public static async Task <DateTime?> GetMaxReportedDateAsync(string taxonomyId, string entityId, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo) { var db = abaxXBRLCellStoreMongo.GetMongoDB(); var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME); var match = new BsonDocument { { "dimensionMap.taxonomyId", taxonomyId }, { "dimensionMap.instanceDocmentEntity", entityId } }; var groupDistinctDates = new BsonDocument { { "_id", "$dimensionMap.instanceDocmentReportedDate" } }; var projectParseDate = new BsonDocument { { "_id", "$_id" }, { "date", new BsonDocument { { "$dateFromString", new BsonDocument { { "dateString", "$_id" } } } } } }; var groupMaxDate = new BsonDocument { { "_id", "_id" }, { "maxDate", new BsonDocument { { "$max", "$date" } } } }; var projectAsStrig = new BsonDocument { { "_id", 0 }, { "maxDate", new BsonDocument { { "$dateToString", new BsonDocument { { "date", "$maxDate" }, { "format", "%Y-%m-%dT%H:%M:%S.%LZ" } } } } } }; var aggregate = collection.Aggregate(). Match(match).Group(groupDistinctDates). Project(projectParseDate).Group(groupMaxDate).Project(projectAsStrig); var document = await aggregate.FirstOrDefaultAsync(); DateTime?dateFound = null; if (document != null && document.Contains("maxDate")) { var stringDate = document["maxDate"].AsString; if (!String.IsNullOrWhiteSpace(stringDate)) { dateFound = DateTime.Parse(stringDate); } } return(dateFound); }
public void TestProcesarDistribucionDocumentosXBRL() { LogUtil.LogDirPath = @"..\..\TestOutput\"; LogUtil.Inicializa(); var DirectorioProcesar = @"..\..\TestInput\XBRL\JBRL\Auxiliar"; var xpe = XPEServiceImpl.GetInstance(true); var abaxXBRLCellStoreMongo = new AbaxXBRLCellStoreMongo(); abaxXBRLCellStoreMongo.JSONOutDirectory = @"..\..\TestOutput\"; abaxXBRLCellStoreMongo.ConnectionString = "mongodb://localhost/jbrl"; abaxXBRLCellStoreMongo.DataBaseName = "jbrl"; //abaxXBRLCellStoreMongo.ConnectionString = "mongodb+srv://oscarloyola:[email protected]/xbrl?retryWrites=true"; //abaxXBRLCellStoreMongo.DataBaseName = "xbrl"; abaxXBRLCellStoreMongo.Init(); var mongoDB = abaxXBRLCellStoreMongo.GetMongoDB(); XPEService xpeService = XPEServiceImpl.GetInstance(); string[] fileEntries = Directory.GetFiles(DirectorioProcesar); var taskList = new List <Task>(); foreach (string rutaArchivo in fileEntries) { var registrationDate = System.IO.File.GetLastWriteTime(rutaArchivo); var fileName = Path.GetFileName(rutaArchivo); using (var reader = File.OpenRead(rutaArchivo)) { var reportParams = new Dictionary <String, String>(); var factsList = new List <FactJBRL>(); DocumentoInstanciaXbrlDto documentoInstanciXbrlDto = xpeService.CargaInstanciaXBRLStreamFile(reader, fileName); if (documentoInstanciXbrlDto.Taxonomia == null) { JBRLUtils.ObtenerTaxonomia(documentoInstanciXbrlDto); } JBRLUtils.InsertFacts( documentoInstanciXbrlDto, reportParams, factsList, taskList, registrationDate, abaxXBRLCellStoreMongo); } System.GC.Collect(); } Task.WaitAll(taskList.ToArray()); }
/// <summary> /// Updates the attributes "isReplaced" and "replacementId" with the given parameters. /// </summary> /// <param name="reportId">Report id to evaluate.</param> /// <param name="reportRecordId">Version that replacement previous versions.</param> /// <param name="abaxXBRLCellStoreMongo">Data access object of the data base.</param> public static async Task UpdateReportsReplacementVersionAsync(string reportId, string reportRecordId, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo) { FilterDefinition <BsonDocument> filter = new BsonDocument { { "reportId", reportId }, { "isReplaced", false }, { "reportRecordId", new BsonDocument { { "$ne", reportRecordId } } } }; UpdateDefinition <BsonDocument> updateStatement = new BsonDocument { { "$set", new BsonDocument { { "isReplaced", true }, { "replacementId", reportRecordId }, } } }; var db = abaxXBRLCellStoreMongo.GetMongoDB(); var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME); await collection.UpdateManyAsync(filter, updateStatement); }
/// <summary> /// Insert the facts of a document into the mongo database. /// </summary> /// <param name="documentoInstanciXbrlDto">Documento to evaluate.</param> /// <param name="reportParams">Extra params to include.</param> /// <param name="factsList">Extra facts to include.</param> /// <param name="taskList">List of task to wait befor ends.</param> /// <param name="registrationDate">Registration date of the document.</param> /// <param name="abaxXBRLCellStoreMongo">Data access object of mongo.</param> public static void InsertFacts( DocumentoInstanciaXbrlDto documentoInstanciXbrlDto, IDictionary <String, String> reportParams, IList <FactJBRL> factsList, IList <Task> taskList, DateTime registrationDate, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo) { var mongoDB = abaxXBRLCellStoreMongo.GetMongoDB(); JBRLUtils.SetReportParams(documentoInstanciXbrlDto, reportParams); var reportId = JBRLUtils.CreateReportId(reportParams); var reportRecordId = JBRLUtils.CreateReportRecordId(reportId, registrationDate); if (JBRLUtils.ExistsReportRecord(reportRecordId, abaxXBRLCellStoreMongo)) { return; } var replacementId = JBRLUtils.SarchReplacementId(reportId, reportRecordId, registrationDate, abaxXBRLCellStoreMongo); if (replacementId == null) { var taskVersion = JBRLUtils.UpdateReportsReplacementVersionAsync(reportId, reportRecordId, abaxXBRLCellStoreMongo); taskList.Add(taskVersion); } foreach (var hechoId in documentoInstanciXbrlDto.HechosPorId.Keys) { var hecho = documentoInstanciXbrlDto.HechosPorId[hechoId]; if (hecho.TipoDatoXbrl.Contains("64")) { continue; } var hechoJbrl = FactJBRL.Parse( hecho, documentoInstanciXbrlDto, reportId, reportRecordId, registrationDate, reportParams, replacementId); if (hechoJbrl != null) { factsList.Add(hechoJbrl); } else { LogUtil.Error(new Dictionary <string, object>() { { "Error", "No fué posible convertir el hecho a JBRL" }, { "HechoId", hecho.Id ?? "null" }, { "Concepto", hecho.IdConcepto ?? "null" }, { "Hecho", hecho } }); } } var modeloBaseList = new List <IModeloBase>(); modeloBaseList.AddRange(factsList); abaxXBRLCellStoreMongo.InserttChunksCollection(mongoDB, JBRLUtils.COLLECTION_NAME, modeloBaseList); var taskFactsEvaluation = JBRLUtils. UpdatePreviousFactsReplacementVersionAsync(factsList, abaxXBRLCellStoreMongo); taskList.Add(taskFactsEvaluation); }
/// <summary> /// Set the replacement to same facts reported in distinct documents. /// </summary> /// <param name="factsList">The list of facts to evaluate.</param> /// <param name="abaxXBRLCellStoreMongo">Data access object.</param> public static async Task UpdatePreviousFactsReplacementVersionAsync(IList <FactJBRL> factsList, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo) { if (factsList.Count == 0) { return; } var first = factsList.First(); if (first.isReplaced) { return; } string entityId = null; string taxonomyId = null; if (!first.dimensionMap.TryGetValue("instanceDocmentEntity", out entityId) || string.IsNullOrEmpty(entityId) || !first.dimensionMap.TryGetValue("taxonomyId", out taxonomyId) || string.IsNullOrEmpty(taxonomyId)) { return; } var maxReportedDate = await GetMaxReportedDateAsync(taxonomyId, entityId, abaxXBRLCellStoreMongo); if (maxReportedDate == null) { return; } var reportedDatesList = await FindLowestDatesAsync(taxonomyId, entityId, maxReportedDate ?? DateTime.MinValue, abaxXBRLCellStoreMongo); string currentDateString = null; first.dimensionMap.TryGetValue("instanceDocmentReportedDate", out currentDateString); if (currentDateString != null) { var currentDate = DateTime.Parse(currentDateString); if (currentDate < maxReportedDate) { reportedDatesList.Add(currentDateString); } } if (reportedDatesList.Count == 0) { return; } FilterDefinition <BsonDocument> fechasAceptadas = Builders <BsonDocument> .Filter.In("dimensionMap.instanceDocmentReportedDate", reportedDatesList); FilterDefinition <BsonDocument> filterNoReplaced = Builders <BsonDocument> .Filter.Eq("isReplaced", false); FilterDefinition <BsonDocument> filterNotCurrent = Builders <BsonDocument> .Filter.And(fechasAceptadas, filterNoReplaced); var reportRecordId = first.reportRecordId; UpdateDefinition <BsonDocument> updateStatement = new BsonDocument { { "$set", new BsonDocument { { "isReplaced", true }, { "replacementId", reportRecordId }, } } }; var db = abaxXBRLCellStoreMongo.GetMongoDB(); var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME); var idsHEchosEnviados = new List <string>(); foreach (var fact in factsList) { idsHEchosEnviados.Add(fact.factId); if (idsHEchosEnviados.Count >= 500) { FilterDefinition <BsonDocument> filterIdsHechosActualizar = Builders <BsonDocument> .Filter.In("factId", idsHEchosEnviados); FilterDefinition <BsonDocument> filterHechosReenviados = Builders <BsonDocument> .Filter.And(filterIdsHechosActualizar, filterNotCurrent); await collection.UpdateManyAsync(filterHechosReenviados, updateStatement); idsHEchosEnviados = new List <string>(); } } if (idsHEchosEnviados.Count > 0) { FilterDefinition <BsonDocument> filterIdsHechosActualizar = Builders <BsonDocument> .Filter.In("factId", idsHEchosEnviados); FilterDefinition <BsonDocument> filterHechosReenviados = Builders <BsonDocument> .Filter.And(filterIdsHechosActualizar, filterNotCurrent); await collection.UpdateManyAsync(filterHechosReenviados, updateStatement); } }