public override void Cleanup()
{
base.Cleanup();
TypeSelectedItem = null;
page = 0;
lastPage = 0;
}
///<summary>
///Elimina un discoverItem
///</summary>
///<param name="identifier">Identificador del item</param>
public bool RemoveDiscoverItem(Guid identifier)
{
try
{
DiscoverItem discoverItem = GetDiscoverItemById(identifier);
if (discoverItem != null)
{
if (discoverItem.DissambiguationProblems != null)
{
foreach (var dissambiguationProblem in discoverItem.DissambiguationProblems)
{
_context.Entry(dissambiguationProblem).State = EntityState.Deleted;
if (dissambiguationProblem.DissambiguationCandiates != null)
{
foreach (var dissambiguationCandidate in dissambiguationProblem.DissambiguationCandiates)
{
_context.Entry(dissambiguationCandidate).State = EntityState.Deleted;
}
}
}
}
_context.Entry(discoverItem).State = EntityState.Deleted;
_context.SaveChanges();
}
return(true);
}
catch (Exception ex)
{
return(false);
}
}
public void CheckInitialized()
{
if (TypeSelectedItem == null)
{
TypeSelectedItem = DisconverItems[0];
}
}
/// <summary>
/// Obtiene los detalles de un rdf
/// </summary>
/// <returns></returns>
public IActionResult DetailsRdf(Guid itemId)
{
//Cargamos la ontología
RohGraph ontologyGraph = new RohGraph();
ontologyGraph = _callEDtlPublishService.CallGetOntology();
SparqlResultSet sparqlResultSetNombresPropiedades = (SparqlResultSet)ontologyGraph.ExecuteQuery("select distinct ?entidad ?nombre where { ?entidad <http://www.w3.org/2000/01/rdf-schema#label> ?nombre. FILTER(lang(?nombre) = 'es')}");
//Guardamos todos los nombres de las propiedades en un diccionario
Dictionary <string, string> communNamePropierties = new Dictionary <string, string>();
foreach (SparqlResult sparqlResult in sparqlResultSetNombresPropiedades.Results)
{
communNamePropierties.Add(sparqlResult["entidad"].ToString(), ((LiteralNode)(sparqlResult["nombre"])).Value);
}
//Cargamos los datos
DiscoverItem discoveryGraph = _discoverItemService.GetDiscoverItemById(itemId);
RohGraph dataGraph = new RohGraph();
dataGraph.LoadFromString(discoveryGraph.DiscoverRdf, new RdfXmlParser());
//Guardamos todas las entidades que no son blankNodes
List <String> entities = new List <string>();
SparqlResultSet sparqlResultSetListaEntidadesNotBN = (SparqlResultSet)dataGraph.ExecuteQuery("select ?s count(?p) as ?num where { ?s ?p ?o. FILTER (!isBlank(?s)) }group by ?s order by desc(?num) ");
foreach (SparqlResult sparqlResult in sparqlResultSetListaEntidadesNotBN.Results)
{
entities.Add(sparqlResult["s"].ToString());
}
List <DiscoverRdfViewModel> model = new List <DiscoverRdfViewModel>();
//Guardamos todas las entidades
List <String> allEntities = new List <string>();
SparqlResultSet sparqlResultSetEntidades = (SparqlResultSet)dataGraph.ExecuteQuery("select distinct ?s where { ?s ?p ?o }");
foreach (SparqlResult sparqlResult in sparqlResultSetEntidades.Results)
{
allEntities.Add(sparqlResult["s"].ToString());
}
SparqlResultSet sparqlResultSet = (SparqlResultSet)dataGraph.ExecuteQuery("select ?s ?p ?o where { ?s ?p ?o }");
Dictionary <string, List <SparqlResult> > entitySparqlResult = new Dictionary <string, List <SparqlResult> >();
foreach (SparqlResult sparqlResult in sparqlResultSet.Results)
{
if (!entitySparqlResult.ContainsKey(sparqlResult["s"].ToString()))
{
entitySparqlResult.Add(sparqlResult["s"].ToString(), new List <SparqlResult>());
}
entitySparqlResult[sparqlResult["s"].ToString()].Add(sparqlResult);
}
foreach (var idEntity in entities)
{
DiscoverRdfViewModel entidad = createDiscoverRdfViewModel(idEntity, entitySparqlResult, new List <string>(), allEntities, communNamePropierties, discoveryGraph.LoadedEntities);
model.Add(entidad);
}
return(View(model));
}
///<summary>
///Añade un item de descubrimiento
///</summary>
///<param name="discoverItem">Item de descubrimiento</param>
public Guid AddDiscoverItem(DiscoverItem discoverItem)
{
discoverItem.ID = Guid.NewGuid();
_context.DiscoverItem.Add(discoverItem);
_context.SaveChanges();
return(discoverItem.ID);
}
private void OnItemClick(DiscoverItem clickedItem)
{
if (clickedItem != null)
{
NavigationService.Frame.SetListDataItemForNextConnectedAnimation(clickedItem);
NavigationService.Navigate(typeof(MoviesDetailViewModel).FullName, clickedItem.id);
}
}
/// <summary>
/// Procesa un item de descubrimiento
/// </summary>
/// <param name="itemIDstring">Identificador del item</param>
/// <returns></returns>
public bool ProcessItem(string itemIDstring)
{
Guid itemID = JsonConvert.DeserializeObject <Guid>(itemIDstring);
try
{
DiscoverItemBDService discoverItemBDService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <DiscoverItemBDService>();
ProcessDiscoverStateJobBDService processDiscoverStateJobBDService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <ProcessDiscoverStateJobBDService>();
CallCronApiService callCronApiService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <CallCronApiService>();
CallEtlApiService callEtlApiService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <CallEtlApiService>();
CallUrisFactoryApiService callUrisFactoryApiService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <CallUrisFactoryApiService>();
DiscoverItem discoverItem = discoverItemBDService.GetDiscoverItemById(itemID);
if (discoverItem != null)
{
//Aplicamos el proceso de descubrimiento
DiscoverResult resultado = Init(discoverItem, callEtlApiService, callUrisFactoryApiService);
Process(discoverItem, resultado,
discoverItemBDService,
callCronApiService,
callUrisFactoryApiService,
processDiscoverStateJobBDService
);
}
}
catch (Exception ex)
{
Logging.Error(ex);
//Se ha producido un error al aplicar el descubrimiento
//Modificamos los datos del DiscoverItem que ha fallado
DiscoverItemBDService discoverItemBDService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <DiscoverItemBDService>();
DiscoverItem discoverItemBBDD = discoverItemBDService.GetDiscoverItemById(itemID);
discoverItemBBDD.UpdateError($"{ex.Message}\n{ex.StackTrace}\n");
discoverItemBDService.ModifyDiscoverItem(discoverItemBBDD);
if (!string.IsNullOrEmpty(discoverItemBBDD.JobID))
{
//Si viene de una tarea actualizamos su estado de descubrimiento
ProcessDiscoverStateJobBDService processDiscoverStateJobBDService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <ProcessDiscoverStateJobBDService>();
ProcessDiscoverStateJob processDiscoverStateJob = processDiscoverStateJobBDService.GetProcessDiscoverStateJobByIdJob(discoverItemBBDD.JobID);
if (processDiscoverStateJob != null)
{
processDiscoverStateJob.State = "Error";
processDiscoverStateJobBDService.ModifyProcessDiscoverStateJob(processDiscoverStateJob);
}
else
{
processDiscoverStateJob = new ProcessDiscoverStateJob()
{
State = "Error", JobId = discoverItemBBDD.JobID
};
processDiscoverStateJobBDService.AddProcessDiscoverStateJob(processDiscoverStateJob);
}
}
}
return(true);
}
///<summary>
///Añade un item de descubrimiento
///</summary>
///<param name="discoverItem">Item de descubrimiento</param>
public Guid AddDiscoverItem(DiscoverItem discoverItem)
{
Guid discoveritemID = Guid.Empty;
discoveritemID = Guid.NewGuid();
discoverItem.ID = discoveritemID;
_discoverItems.Add(discoverItem);
return(discoveritemID);
}
public IActionResult RetryDiscover(string idJob, string IdDiscoverItem)
{
DiscoverItem item = _discoverItemService.GetDiscoverItemById(new Guid(IdDiscoverItem));
Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(item.Rdf));
FormFile file = new FormFile(stream, 0, stream.Length, "rdfFile", "rdf.xml");
//Lo reencolamos sin corregir para que lo vuelva a intentar
_callEDtlPublishService.CallDataPublish(file, idJob, false);
//Eliminamos el item
_discoverItemService.RemoveDiscoverItem(item.ID);
return(RedirectToAction("DetailsJob", "Job", new { id = idJob }));
}
///<summary>
///Elimina un discoverItem
///</summary>
///<param name="identifier">Identificador del item</param>
public bool RemoveDiscoverItem(Guid identifier)
{
try
{
DiscoverItem discoverItem = GetDiscoverItemById(identifier);
if (discoverItem != null)
{
_discoverItems.Remove(discoverItem);
}
return(true);
}
catch (Exception ex)
{
return(false);
}
}
public IActionResult dataDiscoverState(Guid identifier)
{
try
{
DiscoverItem item = _discoverItemService.GetDiscoverItemById(identifier);
if (item != null)
{
DiscoverStateResult discoverStateResult = new DiscoverStateResult()
{
ID = item.ID,
Status = (DiscoverStateResult.DiscoverItemStatus)Enum.Parse(typeof(DiscoverStateResult.DiscoverItemStatus), item.Status),
Rdf = item.Rdf,
DiscoverRdf = item.DiscoverRdf,
Error = item.Error,
DiscoverReport = item.DiscoverReport,
DissambiguationProblems = new List <DiscoverStateResult.DiscoverDissambiguation>()
};
foreach (DiscoverItem.DiscoverDissambiguation problem in item.DissambiguationProblems)
{
DiscoverStateResult.DiscoverDissambiguation discoverDissambiguation = new DiscoverStateResult.DiscoverDissambiguation()
{
IDOrigin = problem.IDOrigin,
DissambiguationCandiates = new List <DiscoverStateResult.DiscoverDissambiguation.DiscoverDissambiguationCandiate>()
};
foreach (DiscoverItem.DiscoverDissambiguation.DiscoverDissambiguationCandiate candidate in problem.DissambiguationCandiates)
{
discoverDissambiguation.DissambiguationCandiates.Add(new DiscoverStateResult.DiscoverDissambiguation.DiscoverDissambiguationCandiate()
{
IDCandidate = candidate.IDCandidate,
Score = candidate.Score
});
}
discoverStateResult.DissambiguationProblems.Add(discoverDissambiguation);
}
return(Ok(discoverStateResult));
}
return(NotFound());
}
catch (Exception ex)
{
return(Problem(ex.ToString()));
}
}
public IActionResult dataDiscover(IFormFile rdfFile)
{
try
{
XmlDocument rdf = SparqlUtility.GetRDFFromFile(rdfFile);
DiscoverItem discoverItem = new DiscoverItem()
{
Rdf = rdf.InnerXml, Publish = false, DissambiguationProcessed = false, Status = "Pending"
};
Guid addedID = _discoverItemService.AddDiscoverItem(discoverItem);
_amqpService.PublishMessage(addedID);
return(Ok(addedID));
}
catch (Exception ex)
{
return(Problem(ex.ToString()));
}
}
public IActionResult dataPublish(IFormFile rdfFile, string jobId, bool discoverProcessed)
{
try
{
string idDiscoverItem = null;
if (Request != null)
{
idDiscoverItem = Request.Query["idDiscoverItem"].ToString();
}
if (!string.IsNullOrEmpty(idDiscoverItem))
{
//Si viene el parametro 'idDiscoverItem' actualizamos un DiscoverItem ya existente.
XmlDocument rdf = SparqlUtility.GetRDFFromFile(rdfFile);
DiscoverItem discoverItem = _discoverItemService.GetDiscoverItemById(new Guid(idDiscoverItem));
discoverItem.DissambiguationProcessed = discoverProcessed;
discoverItem.Publish = true;
discoverItem.Status = "Pending";
discoverItem.DiscoverRdf = rdf.InnerXml;
_amqpService.PublishMessage(idDiscoverItem);
return(Ok());
}
else
{
XmlDocument rdf = SparqlUtility.GetRDFFromFile(rdfFile);
DiscoverItem discoverItem = new DiscoverItem()
{
JobID = jobId, Rdf = rdf.InnerXml, Publish = true, DissambiguationProcessed = discoverProcessed, Status = "Pending"
};
Guid addedID = _discoverItemService.AddDiscoverItem(discoverItem);
_amqpService.PublishMessage(addedID);
return(Ok());
}
}
catch (Exception ex)
{
return(Problem(ex.ToString()));
}
}
///<summary>
///Modifica una item de descubrimiento
///</summary>
///<param name="discoverItem">tem de descubrimiento a modificar con los datos nuevos</param>
public bool ModifyDiscoverItem(DiscoverItem discoverItem)
{
bool modified = false;
DiscoverItem discoverItemOriginal = GetDiscoverItemById(discoverItem.ID);
if (discoverItemOriginal != null)
{
discoverItemOriginal.Status = discoverItem.Status;
discoverItemOriginal.Rdf = discoverItem.Rdf;
discoverItemOriginal.DiscoverRdf = discoverItem.DiscoverRdf;
discoverItemOriginal.Error = discoverItem.Error;
discoverItemOriginal.JobID = discoverItem.JobID;
discoverItemOriginal.Publish = discoverItem.Publish;
discoverItemOriginal.DissambiguationProcessed = discoverItem.DissambiguationProcessed;
discoverItemOriginal.DiscoverReport = discoverItem.DiscoverReport;
discoverItemOriginal.DissambiguationProblems = discoverItem.DissambiguationProblems;
discoverItemOriginal.DiscardDissambiguations = discoverItem.DiscardDissambiguations;
discoverItemOriginal.LoadedEntities = discoverItem.LoadedEntities;
modified = true;
}
return(modified);
}
private async Task QueryTrending(DiscoverItem item)
{
CommonProvider common = new CommonProvider();
List <TrendingResult> result = new List <TrendingResult>();
if (pageIndex == 0)
{
if (TrendingList == null)
{
TrendingList = new ObservableCollection <TrendingResult>();
}
else
{
TrendingList.Clear();
}
}
if (TypeSelectedItem.HotType != HotTopicType.None)
{
result = await common.QueryDiscover(TypeSelectedItem.HotType, pageIndex + 1, 20, new List <TrendingResult>(TrendingList));
}
if (TypeSelectedItem.Type != TopicType.None)
{
int index = 0;
if (TrendingList.Count != 0)
{
index = TrendingList[TrendingList.Count - 1].RecommendedAt - 1;
}
result = await common.QueryDiscover(TypeSelectedItem.Type, index, 20, 20);
}
result.ForEach(x => TrendingList.Add(x));
pageIndex++;
}
public IActionResult ResolveDiscover(string idJob, string IdDiscoverItem, Dictionary <string, string> DissambiguationProblemsResolve)
{
DiscoverItem item = _discoverItemService.GetDiscoverItemById(new Guid(IdDiscoverItem));
//Cargamos el RDF
RohGraph dataGraph = new RohGraph();
dataGraph.LoadFromString(item.DiscoverRdf, new RdfXmlParser());
//Modificamos el RDF
TripleStore store = new TripleStore();
store.Add(dataGraph);
//Cambiamos candidato.Key por entityID
foreach (string uriOriginal in DissambiguationProblemsResolve.Keys)
{
if (!string.IsNullOrEmpty(DissambiguationProblemsResolve[uriOriginal]))
{
//En caso de que la resolución sea una URI de Unidata añadimos el SameAs
if (!string.IsNullOrEmpty(_unidataPrefix.GetUnidataDomain()) && DissambiguationProblemsResolve[uriOriginal].StartsWith(_unidataPrefix.GetUnidataDomain()))
{
IUriNode t_subject = dataGraph.CreateUriNode(UriFactory.Create(uriOriginal));
IUriNode t_predicate = dataGraph.CreateUriNode(UriFactory.Create("http://www.w3.org/2002/07/owl#sameAs"));
IUriNode t_object = dataGraph.CreateUriNode(UriFactory.Create(DissambiguationProblemsResolve[uriOriginal]));
dataGraph.Assert(new Triple(t_subject, t_predicate, t_object));
}
else
{
//En caso de que la resolución NO sea una URI de Unidata modificamos las URLs
SparqlUpdateParser parser = new SparqlUpdateParser();
//Actualizamos los sujetos
SparqlUpdateCommandSet updateSubject = parser.ParseFromString(@"DELETE { ?s ?p ?o. }
INSERT{<" + DissambiguationProblemsResolve[uriOriginal] + @"> ?p ?o.}
WHERE
{
?s ?p ?o. FILTER(?s = <" + uriOriginal + @">)
}");
//Actualizamos los objetos
SparqlUpdateCommandSet updateObject = parser.ParseFromString(@"DELETE { ?s ?p ?o. }
INSERT{?s ?p <" + DissambiguationProblemsResolve[uriOriginal] + @">.}
WHERE
{
?s ?p ?o. FILTER(?o = <" + uriOriginal + @">)
}");
LeviathanUpdateProcessor processor = new LeviathanUpdateProcessor(store);
processor.ProcessCommandSet(updateSubject);
processor.ProcessCommandSet(updateObject);
}
}
}
System.IO.StringWriter sw = new System.IO.StringWriter();
RdfXmlWriter rdfXmlWriter = new RdfXmlWriter();
rdfXmlWriter.Save(dataGraph, sw);
string rdfXml = sw.ToString();
Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(rdfXml));
FormFile file = new FormFile(stream, 0, stream.Length, "rdfFile", "rdf.xml");
//Actualizamos el item
Dictionary <string, List <string> > discards = new Dictionary <string, List <string> >();
foreach (DiscoverItem.DiscoverDissambiguation dissambiguation in item.DissambiguationProblems)
{
if (DissambiguationProblemsResolve.ContainsKey(dissambiguation.IDOrigin) && DissambiguationProblemsResolve[dissambiguation.IDOrigin] == null)
{
discards.Add(dissambiguation.IDOrigin, dissambiguation.DissambiguationCandiates.Select(x => x.IDCandidate).ToList());
}
}
item.UpdateDissambiguationDiscards(discards, rdfXml);
item.DiscoverRdf = rdfXml;
item.Status = "Pending";
_discoverItemService.ModifyDiscoverItem(item);
//Lo reencolamos corregido junto con su identificador
_callEDtlPublishService.CallDataPublish(file, idJob, false, IdDiscoverItem);
return(RedirectToAction("DetailsJob", "Job", new { id = idJob }));
}
//TODO mover a otro sitio
/// <summary>
/// Realiza el proceso completo de desubrimiento sobre un RDF
/// </summary>
/// <param name="pDiscoverItem">Item de descubrimiento</param>
/// <param name="pCallEtlApiService">Servicio para hacer llamadas a los métodos del controlador etl del API_CARGA </param>
/// <param name="pCallUrisFactoryApiService">Servicio para hacer llamadas a los métodos del Uris Factory</param>
/// <returns>DiscoverResult con los datos del descubrimiento</returns>
private DiscoverResult Init(DiscoverItem pDiscoverItem, CallEtlApiService pCallEtlApiService, CallUrisFactoryApiService pCallUrisFactoryApiService)
{
#region Cargamos configuraciones
ConfigSparql ConfigSparql = new ConfigSparql();
string SGI_SPARQLEndpoint = ConfigSparql.GetEndpoint();
string SGI_SPARQLGraph = ConfigSparql.GetGraph();
string SGI_SPARQLQueryParam = ConfigSparql.GetQueryParam();
string SGI_SPARQLUsername = ConfigSparql.GetUsername();
string SGI_SPARQLPassword = ConfigSparql.GetPassword();
string Unidata_SPARQLEndpoint = ConfigSparql.GetUnidataEndpoint();
string Unidata_SPARQLGraph = ConfigSparql.GetUnidataGraph();
string Unidata_SPARQLQueryParam = ConfigSparql.GetUnidataQueryParam();
string Unidata_SPARQLUsername = ConfigSparql.GetUnidataUsername();
string Unidata_SPARQLPassword = ConfigSparql.GetUnidataPassword();
ConfigService ConfigService = new ConfigService();
float MaxScore = ConfigService.GetMaxScore();
float MinScore = ConfigService.GetMinScore();
string UnidataDomain = ConfigService.GetUnidataDomain();
ConfigScopus ConfigScopus = new ConfigScopus();
string ScopusApiKey = ConfigScopus.GetScopusApiKey();
string ScopusUrl = ConfigScopus.GetScopusUrl();
ConfigCrossref ConfigCrossref = new ConfigCrossref();
string CrossrefUserAgent = ConfigCrossref.GetCrossrefUserAgent();
ConfigWOS ConfigWOS = new ConfigWOS();
string WOSAuthorization = ConfigWOS.GetWOSAuthorization();
#endregion
DiscoverUtility discoverUtility = new DiscoverUtility();
DateTime discoverInitTime = DateTime.Now;
//Cargamos la ontología
if (_ontologyGraph == null)
{
_ontologyGraph = pCallEtlApiService.CallGetOntology();
}
//Cargamos datos del RDF
RohGraph dataGraph = new RohGraph();
Dictionary <string, HashSet <string> > discardDissambiguations = new Dictionary <string, HashSet <string> >();
if (!string.IsNullOrEmpty(pDiscoverItem.DiscoverRdf))
{
//Si tenemos valor en DiscoverRdf, trabajamos con este RDF, ya que estamos reprocesando un rdf validado
dataGraph.LoadFromString(pDiscoverItem.DiscoverRdf, new RdfXmlParser());
if (pDiscoverItem.DiscardDissambiguations != null)
{
foreach (DiscoverItem.DiscardDissambiguation discardDissambiguation in pDiscoverItem.DiscardDissambiguations)
{
if (!discardDissambiguations.ContainsKey(discardDissambiguation.IDOrigin))
{
discardDissambiguations.Add(discardDissambiguation.IDOrigin, new HashSet <string>());
}
discardDissambiguations[discardDissambiguation.IDOrigin].UnionWith(discardDissambiguation.DiscardCandidates);
}
}
}
else
{
dataGraph.LoadFromString(pDiscoverItem.Rdf, new RdfXmlParser());
}
//Cargamos el razonador para inferir datos en la ontología
RohRdfsReasoner reasoner = new RohRdfsReasoner();
reasoner.Initialise(_ontologyGraph);
//Cargamos los datos con inferencia
RohGraph dataInferenceGraph = dataGraph.Clone();
reasoner.Apply(dataInferenceGraph);
//Datos para trabajar con la reconciliación
ReconciliationData reconciliationData = new ReconciliationData();
//Datos para trabajar con el descubrimiento de enlaces
DiscoverLinkData discoverLinkData = new DiscoverLinkData();
//Almacenamos las entidades con dudas acerca de su reonciliación
Dictionary <string, Dictionary <string, float> > reconciliationEntitiesProbability = new Dictionary <string, Dictionary <string, float> >();
//Cargamos la caché global
if (_discoverCacheGlobal == null)
{
_discoverCacheGlobal = new DiscoverCacheGlobal();
discoverUtility.LoadPersonWithName(_discoverCacheGlobal, SGI_SPARQLEndpoint, SGI_SPARQLGraph, SGI_SPARQLQueryParam, SGI_SPARQLUsername, SGI_SPARQLPassword);
discoverUtility.LoadEntitiesWithTitle(_discoverCacheGlobal, SGI_SPARQLEndpoint, SGI_SPARQLGraph, SGI_SPARQLQueryParam, SGI_SPARQLUsername, SGI_SPARQLPassword);
}
if (!pDiscoverItem.DissambiguationProcessed)
{
bool hasChanges = true;
//Cache del proceso de descubrimiento
DiscoverCache discoverCache = new DiscoverCache();
//Se realizarán este proceso iterativamente hasta que no haya ningún cambio en lo que a reconciliaciones se refiere
while (hasChanges)
{
hasChanges = false;
//Preparamos los datos para proceder con la reconciliazción
discoverUtility.PrepareData(dataGraph, reasoner, out dataInferenceGraph,
out Dictionary <string, HashSet <string> > entitiesRdfTypes,
out Dictionary <string, string> entitiesRdfType,
out Dictionary <string, List <DisambiguationData> > disambiguationDataRdf, false);
//Carga los scores de las personas
//Aquí se almacenarán los nombres de las personas del RDF, junto con los candidatos de la BBDD y su score
Dictionary <string, Dictionary <string, float> > namesScore = new Dictionary <string, Dictionary <string, float> >();
discoverUtility.LoadNamesScore(ref namesScore, dataInferenceGraph, discoverCache, _discoverCacheGlobal, MinScore, MaxScore);
//0.- Macamos como reconciliadas aquellas que ya estén cargadas en la BBDD con los mismos identificadores
List <string> entidadesCargadas = discoverUtility.LoadEntitiesDB(entitiesRdfType.Keys.ToList().Except(reconciliationData.reconciliatedEntityList.Keys.Union(reconciliationData.reconciliatedEntityList.Values)), SGI_SPARQLEndpoint, SGI_SPARQLGraph, SGI_SPARQLQueryParam, SGI_SPARQLUsername, SGI_SPARQLPassword).Keys.ToList();
foreach (string entitiID in entidadesCargadas)
{
reconciliationData.reconciliatedEntityList.Add(entitiID, entitiID);
reconciliationData.reconciliatedEntitiesWithSubject.Add(entitiID);
}
//1.- Realizamos reconciliación con los identificadores configurados (y el roh:identifier) y marcamos como reconciliadas las entidades seleccionadas para no intentar reconciliarlas posteriormente
discoverUtility.ReconciliateIDs(ref hasChanges, ref reconciliationData, entitiesRdfType, disambiguationDataRdf, discardDissambiguations, _ontologyGraph, ref dataGraph, discoverCache, SGI_SPARQLEndpoint, SGI_SPARQLQueryParam, SGI_SPARQLGraph, SGI_SPARQLUsername, SGI_SPARQLPassword);
//2.- Realizamos la reconciliación con los datos del Propio RDF
discoverUtility.ReconciliateRDF(ref hasChanges, ref reconciliationData, _ontologyGraph, ref dataGraph, reasoner, discardDissambiguations, discoverCache, _discoverCacheGlobal, MinScore, MaxScore);
//3.- Realizamos la reconciliación con los datos de la BBDD
discoverUtility.ReconciliateBBDD(ref hasChanges, ref reconciliationData, out reconciliationEntitiesProbability, _ontologyGraph, ref dataGraph, reasoner, namesScore, discardDissambiguations, discoverCache, _discoverCacheGlobal, MinScore, MaxScore, SGI_SPARQLEndpoint, SGI_SPARQLQueryParam, SGI_SPARQLGraph, SGI_SPARQLUsername, SGI_SPARQLPassword);
//4.- Realizamos la reconciliación con los datos de las integraciones externas
//TODO descomentar
//discoverUtility.ExternalIntegration(ref hasChanges, ref reconciliationData, ref discoverLinkData, ref reconciliationEntitiesProbability, ref dataGraph, reasoner, namesScore,entitiesWithTitle, ontologyGraph, out Dictionary<string, ReconciliationData.ReconciliationScore> entidadesReconciliadasConIntegracionExternaAux, discardDissambiguations, discoverCache,discoverCacheGlobal, ScopusApiKey, ScopusUrl, CrossrefUserAgent, WOSAuthorization, MinScore, MaxScore, SGI_SPARQLEndpoint, SGI_SPARQLQueryParam, SGI_SPARQLGraph, SGI_SPARQLUsername, SGI_SPARQLPassword,pCallUrisFactoryApiService);
//Eliminamos de las probabilidades aquellos que ya estén reconciliados
foreach (string key in reconciliationData.reconciliatedEntityList.Keys)
{
reconciliationEntitiesProbability.Remove(key);
}
}
//5.-Realizamos la detección de equivalencias con Unidata
//TODO descomentar cuando esté habilitaado Unidata
//TODO descomentar y revisar en unidata no tienen roh:identifier
//discoverUtility.EquivalenceDiscover(ontologyGraph, ref dataGraph, reasoner, discoverCache, ref reconciliationEntitiesProbability, discardDissambiguations, UnidataDomain, MinScore, MaxScore, Unidata_SPARQLEndpoint, Unidata_SPARQLQueryParam, Unidata_SPARQLGraph, Unidata_SPARQLUsername, Unidata_SPARQLPassword);
}
//TODO comrpobar cuando esté habilitaado Unidata
DateTime discoverEndTime = DateTime.Now;
DiscoverResult resultado = new DiscoverResult(dataGraph, dataInferenceGraph, _ontologyGraph, reconciliationData, reconciliationEntitiesProbability, discoverInitTime, discoverEndTime, discoverLinkData);
return(resultado);
}
/// <summary>
/// Procesa los resultados del descubrimiento
/// </summary>
/// <param name="pDiscoverItem">Objeto con los datos de com procesar el proeso de descubrimiento</param>
/// <param name="pDiscoverResult">Resultado de la aplicación del descubrimiento</param>
/// <param name="pDiscoverItemBDService">Clase para gestionar las operaciones de las tareas de descubrimiento</param>
/// <param name="pCallCronApiService">Servicio para hacer llamadas a los métodos del apiCron</param>
/// <param name="pCallUrisFactoryApiService">Servicio para hacer llamadas a los métodos del Uris Factory</param>
/// <param name="pProcessDiscoverStateJobBDService">Clase para gestionar los estados de descubrimiento de las tareas</param>
/// <returns></returns>
public void Process(DiscoverItem pDiscoverItem, DiscoverResult pDiscoverResult, DiscoverItemBDService pDiscoverItemBDService, CallCronApiService pCallCronApiService, CallUrisFactoryApiService pCallUrisFactoryApiService, ProcessDiscoverStateJobBDService pProcessDiscoverStateJobBDService)
{
#region Cargamos configuraciones
ConfigSparql ConfigSparql = new ConfigSparql();
string SGI_SPARQLEndpoint = ConfigSparql.GetEndpoint();
string SGI_SPARQLGraph = ConfigSparql.GetGraph();
string SGI_SPARQLQueryParam = ConfigSparql.GetQueryParam();
string SGI_SPARQLUsername = ConfigSparql.GetUsername();
string SGI_SPARQLPassword = ConfigSparql.GetPassword();
string Unidata_SPARQLEndpoint = ConfigSparql.GetUnidataEndpoint();
string Unidata_SPARQLGraph = ConfigSparql.GetUnidataGraph();
string Unidata_SPARQLQueryParam = ConfigSparql.GetUnidataQueryParam();
string Unidata_SPARQLUsername = ConfigSparql.GetUnidataUsername();
string Unidata_SPARQLPassword = ConfigSparql.GetUnidataPassword();
ConfigService ConfigService = new ConfigService();
string UnidataDomain = ConfigService.GetUnidataDomain();
string UnidataUriTransform = ConfigService.GetUnidataUriTransform();
#endregion
/*
* En función del resultado obtenido se realiza una de las siguientes acciones:
* Si para alguna entidad hay más de un candidato que supere el umbral máximo o hay alguna entidad que supere el umbral mínimo pero no alcance el máximo se agregará el RDF a una BBDD junto con todos los datos necesarios para que el administrador decida como proceder.
* Si no estamos en el punto anterior
* Se obtienen las entidades principales del RDF y se eliminan todos los triples que haya en la BBDD en los que aparezcan como sujeto u objeto.
* Se eliminan todos los triples cuyo sujeto y predicado estén en el RDF a cargar y estén marcados como monovaluados.
* Se vuelcan los triples a la BBDD.
*
*
*/
if (pDiscoverItem.Publish)
{
if (pDiscoverResult.discoveredEntitiesProbability.Count > 0)
{
//Hay dudas en la desambiguación, por lo que lo actualizamos en la BBDD con su estado correspondiente
pDiscoverItem.UpdateDissambiguationProblems(
pDiscoverResult.discoveredEntitiesProbability,
pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD.Values.Select(x => x.uri).Union(pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds.Values).Union(pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject).Union(pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration.Values.Select(x => x.uri)).ToList(),
pDiscoverResult.GetDataGraphRDF());
pDiscoverItemBDService.ModifyDiscoverItem(pDiscoverItem);
}
else
{
string urlDiscoverAgent = pCallUrisFactoryApiService.GetUri("Agent", "discover");
//Creamos los SameAs hacia unidata para las entidades que NO lo tengan hacia Unidata
//TODO descomentar cuando esté habilitaado Unidata
//pDiscoverResult.dataGraph = AsioPublication.CreateUnidataSameAs(pDiscoverResult.dataGraph, UnidataDomain, UnidataUriTransform);
//Publicamos en el SGI
AsioPublication asioPublication = new AsioPublication(SGI_SPARQLEndpoint, SGI_SPARQLQueryParam, SGI_SPARQLGraph, SGI_SPARQLUsername, SGI_SPARQLPassword);
asioPublication.PublishRDF(pDiscoverResult.dataGraph, pDiscoverResult.ontologyGraph, new KeyValuePair <string, string>(urlDiscoverAgent, "Algoritmos de descubrimiento"), pDiscoverResult.start, pDiscoverResult.end, pDiscoverResult.discoverLinkData, pCallUrisFactoryApiService);
//TODO Lógica nombres de personas
SparqlResultSet sparqlResultSet = (SparqlResultSet)pDiscoverResult.dataGraph.ExecuteQuery("select ?s ?name where {?s a <http://purl.org/roh/mirror/foaf#Person>. ?s <http://purl.org/roh/mirror/foaf#name> ?name}");
foreach (SparqlResult sparqlResult in sparqlResultSet.Results)
{
string s = sparqlResult["s"].ToString();
string nombre = sparqlResult["name"].ToString();
if (sparqlResult["name"] is ILiteralNode)
{
nombre = ((ILiteralNode)sparqlResult["name"]).Value;
}
_discoverCacheGlobal.PersonsNormalizedNames[s] = DiscoverUtility.NormalizeName(nombre);
}
//TODO Lógica titles
SparqlResultSet sparqlResultSetTitles = (SparqlResultSet)pDiscoverResult.dataGraph.ExecuteQuery("select * where {?s a ?rdftype. ?s <http://purl.org/roh#title> ?title}");
foreach (SparqlResult sparqlResult in sparqlResultSetTitles.Results)
{
string s = sparqlResult["s"].ToString();
string rdftype = sparqlResult["rdftype"].ToString();
string title = sparqlResult["title"].ToString();
if (sparqlResult["title"] is ILiteralNode)
{
title = ((ILiteralNode)sparqlResult["title"]).Value;
}
title = DiscoverUtility.NormalizeTitle(title);
if (!_discoverCacheGlobal.EntitiesNormalizedTitles.ContainsKey(rdftype))
{
_discoverCacheGlobal.EntitiesNormalizedTitles.Add(rdftype, new Dictionary <string, HashSet <string> >());
}
if (!_discoverCacheGlobal.EntitiesNormalizedTitles[rdftype].ContainsKey(title))
{
_discoverCacheGlobal.EntitiesNormalizedTitles[rdftype].Add(title, new HashSet <string>());
}
_discoverCacheGlobal.EntitiesNormalizedTitles[rdftype][title].Add(s);
}
//TODO descomentar cuando esté habilitaado Unidata
if (false)
{
//Publicamos en UNIDATA
AsioPublication asioPublicationUnidata = new AsioPublication(Unidata_SPARQLEndpoint, Unidata_SPARQLQueryParam, Unidata_SPARQLGraph, Unidata_SPARQLUsername, Unidata_SPARQLPassword);
// Prepara el grafo para su carga en Unidata, para ello coge las URIs de Unidata del SameAs y la aplica a los sujetos y los antiguos sujetos se agregan al SameAs
RohGraph unidataGraph = AsioPublication.TransformUrisToUnidata(pDiscoverResult.dataGraph, UnidataDomain, UnidataUriTransform);
//Eliminamos los triples de crisIdentifier ya que no hay que volcarlos en unidata
{
TripleStore store = new TripleStore();
store.Add(unidataGraph);
SparqlUpdateParser parser = new SparqlUpdateParser();
//Actualizamos los sujetos
SparqlUpdateCommandSet updateSubject = parser.ParseFromString(
@" DELETE { ?s ?p ?o. }
WHERE
{
?s ?p ?o. FILTER(?p =<" + mPropertySGIRohCrisIdentifier + @">)
}");
LeviathanUpdateProcessor processor = new LeviathanUpdateProcessor(store);
processor.ProcessCommandSet(updateSubject);
}
asioPublicationUnidata.PublishRDF(unidataGraph, pDiscoverResult.ontologyGraph, new KeyValuePair <string, string>(urlDiscoverAgent, "Algoritmos de descubrimiento"), pDiscoverResult.start, pDiscoverResult.end, pDiscoverResult.discoverLinkData, pCallUrisFactoryApiService);
}
//Lo marcamos como procesado en la BBDD y eliminamos sus metadatos
pDiscoverItem.UpdateProcessed();
pDiscoverItemBDService.ModifyDiscoverItem(pDiscoverItem);
}
//Actualizamos el estado de descubrimiento de la tarea si el estado encolado esta en estado Succeeded o Failed (ha finalizado)
//TODO cambiar por query a BBDD
string statusQueueJob = pCallCronApiService.GetJob(pDiscoverItem.JobID).State;
if ((statusQueueJob == "Failed" || statusQueueJob == "Succeeded"))
{
ProcessDiscoverStateJob processDiscoverStateJob = pProcessDiscoverStateJobBDService.GetProcessDiscoverStateJobByIdJob(pDiscoverItem.JobID);
string state;
//Actualizamos a error si existen items en estado error o con problemas de desambiguación
if (pDiscoverItemBDService.ExistsDiscoverItemsErrorOrDissambiguatinProblems(pDiscoverItem.JobID))
{
state = "Error";
}
else if (pDiscoverItemBDService.ExistsDiscoverItemsPending(pDiscoverItem.JobID))
{
//Actualizamos a 'Pending' si aún existen items pendientes
state = "Pending";
}
else
{
//Actualizamos a Success si no existen items en estado error ni con problemas de desambiguación y no hay ninguno pendiente
state = "Success";
}
if (processDiscoverStateJob != null)
{
processDiscoverStateJob.State = state;
pProcessDiscoverStateJobBDService.ModifyProcessDiscoverStateJob(processDiscoverStateJob);
}
else
{
processDiscoverStateJob = new ProcessDiscoverStateJob()
{
State = state, JobId = pDiscoverItem.JobID
};
pProcessDiscoverStateJobBDService.AddProcessDiscoverStateJob(processDiscoverStateJob);
}
}
}
else
{
//Actualizamos en BBDD
DiscoverItem discoverItemBD = pDiscoverItemBDService.GetDiscoverItemById(pDiscoverItem.ID);
//Reporte de descubrimiento
string discoverReport = "Time processed (seconds): " + pDiscoverResult.secondsProcessed + "\n";
if (pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject != null && pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject.Count > 0)
{
discoverReport += "Entities discover with the same uri: " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject.Count + "\n";
foreach (string uri in pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject)
{
discoverReport += "\t" + uri + "\n";
}
}
if (pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds != null && pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds.Count > 0)
{
discoverReport += "Entities discover with some common identifier: " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds.Count + "\n";
foreach (string uri in pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds.Keys)
{
discoverReport += "\t" + uri + " --> " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds[uri] + "\n";
}
}
if (pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD != null && pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD.Count > 0)
{
discoverReport += "Entities discover with reconciliation config: " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD.Count + "\n";
foreach (string uri in pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD.Keys)
{
discoverReport += "\t" + uri + " --> " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD[uri] + "\n";
}
}
if (pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration != null && pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration.Count > 0)
{
discoverReport += "Entities discover with external integrations config: " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration.Count + "\n";
foreach (string uri in pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration.Keys)
{
discoverReport += "\t" + uri + " --> " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration[uri] + "\n";
}
}
if (pDiscoverResult.discoverLinkData != null && pDiscoverResult.discoverLinkData.entitiesProperties != null && pDiscoverResult.discoverLinkData.entitiesProperties.Count > 0)
{
discoverReport += "Entities with identifiers obtained with External integration: " + pDiscoverResult.discoverLinkData.entitiesProperties.Count + "\n";
foreach (string uri in pDiscoverResult.discoverLinkData.entitiesProperties.Keys)
{
foreach (DiscoverLinkData.PropertyData property in pDiscoverResult.discoverLinkData.entitiesProperties[uri])
{
foreach (string value in property.valueProvenance.Keys)
{
foreach (string provenance in property.valueProvenance[value])
{
discoverReport += "\t" + uri + " - " + property.property + " - " + value + " --> " + provenance + "\n";
}
}
}
}
}
discoverItemBD.UpdateReport(pDiscoverResult.discoveredEntitiesProbability, pDiscoverResult.GetDataGraphRDF(), discoverReport);
pDiscoverItemBDService.ModifyDiscoverItem(discoverItemBD);
}
}
