/// <summary>
/// Get a list of string dates less that the given date.
/// </summary>
/// <param name="taxonomyId">Taxonomy identifier.</param>
/// <param name="entityId">Entity identifier.</param>
/// <param name="maxDate">Date to compare.</param>
/// <param name="abaxXBRLCellStoreMongo">Data access object</param>
/// <returns>List with the string dates.</returns>
public static async Task <IList <string> > FindLowestDatesAsync(string taxonomyId, string entityId, DateTime maxDate, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo)
{
var db = abaxXBRLCellStoreMongo.GetMongoDB();
var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME);
var match = new BsonDocument
{
{ "dimensionMap.taxonomyId", taxonomyId },
{ "dimensionMap.instanceDocmentEntity", entityId }
};
var groupDistinctDates = new BsonDocument {
{ "_id", "$dimensionMap.instanceDocmentReportedDate" }
};
var projectParseDate = new BsonDocument
{
{ "_id", "$_id" },
{ "date", new BsonDocument
{
{ "$dateFromString", new BsonDocument {
{ "dateString", "$_id" }
} }
} }
};
FilterDefinition <BsonDocument> matchLessDate = "{\"date\":{\"$lt\":" + ModeloBase.ParseJson(maxDate) + "}}";
var aggregate = collection.Aggregate().
Match(match).Group(groupDistinctDates).Project(projectParseDate).Match(matchLessDate);
var documentsList = await aggregate.ToListAsync();
var listDates = new List <string>();
foreach (var document in documentsList)
{
listDates.Add(document["_id"].AsString);
}
return(listDates);
}
/// <summary>
/// Checks if the send exists in the data base.
/// </summary>
/// <param name="reportId">Identifier of the send.</param>
/// <param name="abaxXBRLCellStoreMongo">Data access object.</param>
/// <returns>If the send exists.</returns>
public static BsonDocument GetRecentlyReportSend(string reportId, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo)
{
var db = abaxXBRLCellStoreMongo.GetMongoDB();
var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME);
var match = new BsonDocument {
{ "reportId", reportId }, { "isReplaced", false }
};
var groupDistinctDates = new BsonDocument {
{ "_id", "$registrationDate" }
};
var groupMaxDate = new BsonDocument
{
{ "_id", "_id" },
{ "maxDate", new BsonDocument {
{ "$max", "$_id" }
} }
};
var projectAsStrig = new BsonDocument
{
{ "_id", 0 },
{ "maxDate", new BsonDocument
{
{ "$dateToString", new BsonDocument {
{ "date", "$maxDate" }, { "format", "%Y-%m-%dT%H:%M:%S.%LZ" }
} }
} }
};
var aggregate = collection.Aggregate().Match(match).
Group(groupDistinctDates).Group(groupMaxDate).Project(projectAsStrig);
return(aggregate.FirstOrDefault());
}
/// <summary>
/// Finds the max reported date of the entity.
/// </summary>
/// <param name="taxonomyId">Taxonomy reported.</param>
/// <param name="entityId">Entity identifier.</param>
/// <param name="abaxXBRLCellStoreMongo">Data access object.</param>
/// <returns>Date found or null if not date was found.</returns>
public static async Task <DateTime?> GetMaxReportedDateAsync(string taxonomyId, string entityId, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo)
{
var db = abaxXBRLCellStoreMongo.GetMongoDB();
var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME);
var match = new BsonDocument
{
{ "dimensionMap.taxonomyId", taxonomyId },
{ "dimensionMap.instanceDocmentEntity", entityId }
};
var groupDistinctDates = new BsonDocument {
{ "_id", "$dimensionMap.instanceDocmentReportedDate" }
};
var projectParseDate = new BsonDocument
{
{ "_id", "$_id" },
{ "date", new BsonDocument
{
{ "$dateFromString", new BsonDocument {
{ "dateString", "$_id" }
} }
} }
};
var groupMaxDate = new BsonDocument
{
{ "_id", "_id" },
{ "maxDate", new BsonDocument {
{ "$max", "$date" }
} }
};
var projectAsStrig = new BsonDocument
{
{ "_id", 0 },
{ "maxDate", new BsonDocument
{
{ "$dateToString", new BsonDocument {
{ "date", "$maxDate" }, { "format", "%Y-%m-%dT%H:%M:%S.%LZ" }
} }
} }
};
var aggregate = collection.Aggregate().
Match(match).Group(groupDistinctDates).
Project(projectParseDate).Group(groupMaxDate).Project(projectAsStrig);
var document = await aggregate.FirstOrDefaultAsync();
DateTime?dateFound = null;
if (document != null && document.Contains("maxDate"))
{
var stringDate = document["maxDate"].AsString;
if (!String.IsNullOrWhiteSpace(stringDate))
{
dateFound = DateTime.Parse(stringDate);
}
}
return(dateFound);
}
public void TestProcesarDistribucionDocumentosXBRL()
{
LogUtil.LogDirPath = @"..\..\TestOutput\";
LogUtil.Inicializa();
var DirectorioProcesar = @"..\..\TestInput\XBRL\JBRL\Auxiliar";
var xpe = XPEServiceImpl.GetInstance(true);
var abaxXBRLCellStoreMongo = new AbaxXBRLCellStoreMongo();
abaxXBRLCellStoreMongo.JSONOutDirectory = @"..\..\TestOutput\";
abaxXBRLCellStoreMongo.ConnectionString = "mongodb://localhost/jbrl";
abaxXBRLCellStoreMongo.DataBaseName = "jbrl";
//abaxXBRLCellStoreMongo.ConnectionString = "mongodb+srv://oscarloyola:[email protected]/xbrl?retryWrites=true";
//abaxXBRLCellStoreMongo.DataBaseName = "xbrl";
abaxXBRLCellStoreMongo.Init();
var mongoDB = abaxXBRLCellStoreMongo.GetMongoDB();
XPEService xpeService = XPEServiceImpl.GetInstance();
string[] fileEntries = Directory.GetFiles(DirectorioProcesar);
var taskList = new List <Task>();
foreach (string rutaArchivo in fileEntries)
{
var registrationDate = System.IO.File.GetLastWriteTime(rutaArchivo);
var fileName = Path.GetFileName(rutaArchivo);
using (var reader = File.OpenRead(rutaArchivo))
{
var reportParams = new Dictionary <String, String>();
var factsList = new List <FactJBRL>();
DocumentoInstanciaXbrlDto documentoInstanciXbrlDto = xpeService.CargaInstanciaXBRLStreamFile(reader, fileName);
if (documentoInstanciXbrlDto.Taxonomia == null)
{
JBRLUtils.ObtenerTaxonomia(documentoInstanciXbrlDto);
}
JBRLUtils.InsertFacts(
documentoInstanciXbrlDto,
reportParams,
factsList,
taskList,
registrationDate,
abaxXBRLCellStoreMongo);
}
System.GC.Collect();
}
Task.WaitAll(taskList.ToArray());
}
/// <summary>
/// Updates the attributes "isReplaced" and "replacementId" with the given parameters.
/// </summary>
/// <param name="reportId">Report id to evaluate.</param>
/// <param name="reportRecordId">Version that replacement previous versions.</param>
/// <param name="abaxXBRLCellStoreMongo">Data access object of the data base.</param>
public static async Task UpdateReportsReplacementVersionAsync(string reportId, string reportRecordId, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo)
{
FilterDefinition <BsonDocument> filter = new BsonDocument
{
{ "reportId", reportId },
{ "isReplaced", false },
{ "reportRecordId", new BsonDocument {
{ "$ne", reportRecordId }
} }
};
UpdateDefinition <BsonDocument> updateStatement = new BsonDocument
{
{ "$set", new BsonDocument
{
{ "isReplaced", true },
{ "replacementId", reportRecordId },
} }
};
var db = abaxXBRLCellStoreMongo.GetMongoDB();
var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME);
await collection.UpdateManyAsync(filter, updateStatement);
}
/// <summary>
/// Insert the facts of a document into the mongo database.
/// </summary>
/// <param name="documentoInstanciXbrlDto">Documento to evaluate.</param>
/// <param name="reportParams">Extra params to include.</param>
/// <param name="factsList">Extra facts to include.</param>
/// <param name="taskList">List of task to wait befor ends.</param>
/// <param name="registrationDate">Registration date of the document.</param>
/// <param name="abaxXBRLCellStoreMongo">Data access object of mongo.</param>
public static void InsertFacts(
DocumentoInstanciaXbrlDto documentoInstanciXbrlDto,
IDictionary <String, String> reportParams,
IList <FactJBRL> factsList,
IList <Task> taskList,
DateTime registrationDate,
AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo)
{
var mongoDB = abaxXBRLCellStoreMongo.GetMongoDB();
JBRLUtils.SetReportParams(documentoInstanciXbrlDto, reportParams);
var reportId = JBRLUtils.CreateReportId(reportParams);
var reportRecordId = JBRLUtils.CreateReportRecordId(reportId, registrationDate);
if (JBRLUtils.ExistsReportRecord(reportRecordId, abaxXBRLCellStoreMongo))
{
return;
}
var replacementId =
JBRLUtils.SarchReplacementId(reportId, reportRecordId, registrationDate, abaxXBRLCellStoreMongo);
if (replacementId == null)
{
var taskVersion =
JBRLUtils.UpdateReportsReplacementVersionAsync(reportId, reportRecordId, abaxXBRLCellStoreMongo);
taskList.Add(taskVersion);
}
foreach (var hechoId in documentoInstanciXbrlDto.HechosPorId.Keys)
{
var hecho = documentoInstanciXbrlDto.HechosPorId[hechoId];
if (hecho.TipoDatoXbrl.Contains("64"))
{
continue;
}
var hechoJbrl = FactJBRL.Parse(
hecho, documentoInstanciXbrlDto, reportId, reportRecordId,
registrationDate, reportParams, replacementId);
if (hechoJbrl != null)
{
factsList.Add(hechoJbrl);
}
else
{
LogUtil.Error(new Dictionary <string, object>()
{
{ "Error", "No fué posible convertir el hecho a JBRL" },
{ "HechoId", hecho.Id ?? "null" },
{ "Concepto", hecho.IdConcepto ?? "null" },
{ "Hecho", hecho }
});
}
}
var modeloBaseList = new List <IModeloBase>();
modeloBaseList.AddRange(factsList);
abaxXBRLCellStoreMongo.InserttChunksCollection(mongoDB, JBRLUtils.COLLECTION_NAME, modeloBaseList);
var taskFactsEvaluation = JBRLUtils.
UpdatePreviousFactsReplacementVersionAsync(factsList, abaxXBRLCellStoreMongo);
taskList.Add(taskFactsEvaluation);
}
/// <summary>
/// Set the replacement to same facts reported in distinct documents.
/// </summary>
/// <param name="factsList">The list of facts to evaluate.</param>
/// <param name="abaxXBRLCellStoreMongo">Data access object.</param>
public static async Task UpdatePreviousFactsReplacementVersionAsync(IList <FactJBRL> factsList, AbaxXBRLCellStoreMongo abaxXBRLCellStoreMongo)
{
if (factsList.Count == 0)
{
return;
}
var first = factsList.First();
if (first.isReplaced)
{
return;
}
string entityId = null;
string taxonomyId = null;
if (!first.dimensionMap.TryGetValue("instanceDocmentEntity", out entityId) || string.IsNullOrEmpty(entityId) ||
!first.dimensionMap.TryGetValue("taxonomyId", out taxonomyId) || string.IsNullOrEmpty(taxonomyId))
{
return;
}
var maxReportedDate = await GetMaxReportedDateAsync(taxonomyId, entityId, abaxXBRLCellStoreMongo);
if (maxReportedDate == null)
{
return;
}
var reportedDatesList = await FindLowestDatesAsync(taxonomyId, entityId, maxReportedDate ?? DateTime.MinValue, abaxXBRLCellStoreMongo);
string currentDateString = null;
first.dimensionMap.TryGetValue("instanceDocmentReportedDate", out currentDateString);
if (currentDateString != null)
{
var currentDate = DateTime.Parse(currentDateString);
if (currentDate < maxReportedDate)
{
reportedDatesList.Add(currentDateString);
}
}
if (reportedDatesList.Count == 0)
{
return;
}
FilterDefinition <BsonDocument> fechasAceptadas = Builders <BsonDocument> .Filter.In("dimensionMap.instanceDocmentReportedDate", reportedDatesList);
FilterDefinition <BsonDocument> filterNoReplaced = Builders <BsonDocument> .Filter.Eq("isReplaced", false);
FilterDefinition <BsonDocument> filterNotCurrent = Builders <BsonDocument> .Filter.And(fechasAceptadas, filterNoReplaced);
var reportRecordId = first.reportRecordId;
UpdateDefinition <BsonDocument> updateStatement = new BsonDocument
{
{ "$set", new BsonDocument
{
{ "isReplaced", true },
{ "replacementId", reportRecordId },
} }
};
var db = abaxXBRLCellStoreMongo.GetMongoDB();
var collection = db.GetCollection <BsonDocument>(COLLECTION_NAME);
var idsHEchosEnviados = new List <string>();
foreach (var fact in factsList)
{
idsHEchosEnviados.Add(fact.factId);
if (idsHEchosEnviados.Count >= 500)
{
FilterDefinition <BsonDocument> filterIdsHechosActualizar = Builders <BsonDocument> .Filter.In("factId", idsHEchosEnviados);
FilterDefinition <BsonDocument> filterHechosReenviados = Builders <BsonDocument> .Filter.And(filterIdsHechosActualizar, filterNotCurrent);
await collection.UpdateManyAsync(filterHechosReenviados, updateStatement);
idsHEchosEnviados = new List <string>();
}
}
if (idsHEchosEnviados.Count > 0)
{
FilterDefinition <BsonDocument> filterIdsHechosActualizar = Builders <BsonDocument> .Filter.In("factId", idsHEchosEnviados);
FilterDefinition <BsonDocument> filterHechosReenviados = Builders <BsonDocument> .Filter.And(filterIdsHechosActualizar, filterNotCurrent);
await collection.UpdateManyAsync(filterHechosReenviados, updateStatement);
}
}
