public override void Cleanup() { base.Cleanup(); TypeSelectedItem = null; page = 0; lastPage = 0; }
///<summary> ///Elimina un discoverItem ///</summary> ///<param name="identifier">Identificador del item</param> public bool RemoveDiscoverItem(Guid identifier) { try { DiscoverItem discoverItem = GetDiscoverItemById(identifier); if (discoverItem != null) { if (discoverItem.DissambiguationProblems != null) { foreach (var dissambiguationProblem in discoverItem.DissambiguationProblems) { _context.Entry(dissambiguationProblem).State = EntityState.Deleted; if (dissambiguationProblem.DissambiguationCandiates != null) { foreach (var dissambiguationCandidate in dissambiguationProblem.DissambiguationCandiates) { _context.Entry(dissambiguationCandidate).State = EntityState.Deleted; } } } } _context.Entry(discoverItem).State = EntityState.Deleted; _context.SaveChanges(); } return(true); } catch (Exception ex) { return(false); } }
public void CheckInitialized() { if (TypeSelectedItem == null) { TypeSelectedItem = DisconverItems[0]; } }
/// <summary> /// Obtiene los detalles de un rdf /// </summary> /// <returns></returns> public IActionResult DetailsRdf(Guid itemId) { //Cargamos la ontología RohGraph ontologyGraph = new RohGraph(); ontologyGraph = _callEDtlPublishService.CallGetOntology(); SparqlResultSet sparqlResultSetNombresPropiedades = (SparqlResultSet)ontologyGraph.ExecuteQuery("select distinct ?entidad ?nombre where { ?entidad <http://www.w3.org/2000/01/rdf-schema#label> ?nombre. FILTER(lang(?nombre) = 'es')}"); //Guardamos todos los nombres de las propiedades en un diccionario Dictionary <string, string> communNamePropierties = new Dictionary <string, string>(); foreach (SparqlResult sparqlResult in sparqlResultSetNombresPropiedades.Results) { communNamePropierties.Add(sparqlResult["entidad"].ToString(), ((LiteralNode)(sparqlResult["nombre"])).Value); } //Cargamos los datos DiscoverItem discoveryGraph = _discoverItemService.GetDiscoverItemById(itemId); RohGraph dataGraph = new RohGraph(); dataGraph.LoadFromString(discoveryGraph.DiscoverRdf, new RdfXmlParser()); //Guardamos todas las entidades que no son blankNodes List <String> entities = new List <string>(); SparqlResultSet sparqlResultSetListaEntidadesNotBN = (SparqlResultSet)dataGraph.ExecuteQuery("select ?s count(?p) as ?num where { ?s ?p ?o. FILTER (!isBlank(?s)) }group by ?s order by desc(?num) "); foreach (SparqlResult sparqlResult in sparqlResultSetListaEntidadesNotBN.Results) { entities.Add(sparqlResult["s"].ToString()); } List <DiscoverRdfViewModel> model = new List <DiscoverRdfViewModel>(); //Guardamos todas las entidades List <String> allEntities = new List <string>(); SparqlResultSet sparqlResultSetEntidades = (SparqlResultSet)dataGraph.ExecuteQuery("select distinct ?s where { ?s ?p ?o }"); foreach (SparqlResult sparqlResult in sparqlResultSetEntidades.Results) { allEntities.Add(sparqlResult["s"].ToString()); } SparqlResultSet sparqlResultSet = (SparqlResultSet)dataGraph.ExecuteQuery("select ?s ?p ?o where { ?s ?p ?o }"); Dictionary <string, List <SparqlResult> > entitySparqlResult = new Dictionary <string, List <SparqlResult> >(); foreach (SparqlResult sparqlResult in sparqlResultSet.Results) { if (!entitySparqlResult.ContainsKey(sparqlResult["s"].ToString())) { entitySparqlResult.Add(sparqlResult["s"].ToString(), new List <SparqlResult>()); } entitySparqlResult[sparqlResult["s"].ToString()].Add(sparqlResult); } foreach (var idEntity in entities) { DiscoverRdfViewModel entidad = createDiscoverRdfViewModel(idEntity, entitySparqlResult, new List <string>(), allEntities, communNamePropierties, discoveryGraph.LoadedEntities); model.Add(entidad); } return(View(model)); }
///<summary> ///Añade un item de descubrimiento ///</summary> ///<param name="discoverItem">Item de descubrimiento</param> public Guid AddDiscoverItem(DiscoverItem discoverItem) { discoverItem.ID = Guid.NewGuid(); _context.DiscoverItem.Add(discoverItem); _context.SaveChanges(); return(discoverItem.ID); }
private void OnItemClick(DiscoverItem clickedItem) { if (clickedItem != null) { NavigationService.Frame.SetListDataItemForNextConnectedAnimation(clickedItem); NavigationService.Navigate(typeof(MoviesDetailViewModel).FullName, clickedItem.id); } }
/// <summary> /// Procesa un item de descubrimiento /// </summary> /// <param name="itemIDstring">Identificador del item</param> /// <returns></returns> public bool ProcessItem(string itemIDstring) { Guid itemID = JsonConvert.DeserializeObject <Guid>(itemIDstring); try { DiscoverItemBDService discoverItemBDService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <DiscoverItemBDService>(); ProcessDiscoverStateJobBDService processDiscoverStateJobBDService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <ProcessDiscoverStateJobBDService>(); CallCronApiService callCronApiService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <CallCronApiService>(); CallEtlApiService callEtlApiService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <CallEtlApiService>(); CallUrisFactoryApiService callUrisFactoryApiService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <CallUrisFactoryApiService>(); DiscoverItem discoverItem = discoverItemBDService.GetDiscoverItemById(itemID); if (discoverItem != null) { //Aplicamos el proceso de descubrimiento DiscoverResult resultado = Init(discoverItem, callEtlApiService, callUrisFactoryApiService); Process(discoverItem, resultado, discoverItemBDService, callCronApiService, callUrisFactoryApiService, processDiscoverStateJobBDService ); } } catch (Exception ex) { Logging.Error(ex); //Se ha producido un error al aplicar el descubrimiento //Modificamos los datos del DiscoverItem que ha fallado DiscoverItemBDService discoverItemBDService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <DiscoverItemBDService>(); DiscoverItem discoverItemBBDD = discoverItemBDService.GetDiscoverItemById(itemID); discoverItemBBDD.UpdateError($"{ex.Message}\n{ex.StackTrace}\n"); discoverItemBDService.ModifyDiscoverItem(discoverItemBBDD); if (!string.IsNullOrEmpty(discoverItemBBDD.JobID)) { //Si viene de una tarea actualizamos su estado de descubrimiento ProcessDiscoverStateJobBDService processDiscoverStateJobBDService = _serviceScopeFactory.CreateScope().ServiceProvider.GetRequiredService <ProcessDiscoverStateJobBDService>(); ProcessDiscoverStateJob processDiscoverStateJob = processDiscoverStateJobBDService.GetProcessDiscoverStateJobByIdJob(discoverItemBBDD.JobID); if (processDiscoverStateJob != null) { processDiscoverStateJob.State = "Error"; processDiscoverStateJobBDService.ModifyProcessDiscoverStateJob(processDiscoverStateJob); } else { processDiscoverStateJob = new ProcessDiscoverStateJob() { State = "Error", JobId = discoverItemBBDD.JobID }; processDiscoverStateJobBDService.AddProcessDiscoverStateJob(processDiscoverStateJob); } } } return(true); }
///<summary> ///Añade un item de descubrimiento ///</summary> ///<param name="discoverItem">Item de descubrimiento</param> public Guid AddDiscoverItem(DiscoverItem discoverItem) { Guid discoveritemID = Guid.Empty; discoveritemID = Guid.NewGuid(); discoverItem.ID = discoveritemID; _discoverItems.Add(discoverItem); return(discoveritemID); }
public IActionResult RetryDiscover(string idJob, string IdDiscoverItem) { DiscoverItem item = _discoverItemService.GetDiscoverItemById(new Guid(IdDiscoverItem)); Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(item.Rdf)); FormFile file = new FormFile(stream, 0, stream.Length, "rdfFile", "rdf.xml"); //Lo reencolamos sin corregir para que lo vuelva a intentar _callEDtlPublishService.CallDataPublish(file, idJob, false); //Eliminamos el item _discoverItemService.RemoveDiscoverItem(item.ID); return(RedirectToAction("DetailsJob", "Job", new { id = idJob })); }
///<summary> ///Elimina un discoverItem ///</summary> ///<param name="identifier">Identificador del item</param> public bool RemoveDiscoverItem(Guid identifier) { try { DiscoverItem discoverItem = GetDiscoverItemById(identifier); if (discoverItem != null) { _discoverItems.Remove(discoverItem); } return(true); } catch (Exception ex) { return(false); } }
public IActionResult dataDiscoverState(Guid identifier) { try { DiscoverItem item = _discoverItemService.GetDiscoverItemById(identifier); if (item != null) { DiscoverStateResult discoverStateResult = new DiscoverStateResult() { ID = item.ID, Status = (DiscoverStateResult.DiscoverItemStatus)Enum.Parse(typeof(DiscoverStateResult.DiscoverItemStatus), item.Status), Rdf = item.Rdf, DiscoverRdf = item.DiscoverRdf, Error = item.Error, DiscoverReport = item.DiscoverReport, DissambiguationProblems = new List <DiscoverStateResult.DiscoverDissambiguation>() }; foreach (DiscoverItem.DiscoverDissambiguation problem in item.DissambiguationProblems) { DiscoverStateResult.DiscoverDissambiguation discoverDissambiguation = new DiscoverStateResult.DiscoverDissambiguation() { IDOrigin = problem.IDOrigin, DissambiguationCandiates = new List <DiscoverStateResult.DiscoverDissambiguation.DiscoverDissambiguationCandiate>() }; foreach (DiscoverItem.DiscoverDissambiguation.DiscoverDissambiguationCandiate candidate in problem.DissambiguationCandiates) { discoverDissambiguation.DissambiguationCandiates.Add(new DiscoverStateResult.DiscoverDissambiguation.DiscoverDissambiguationCandiate() { IDCandidate = candidate.IDCandidate, Score = candidate.Score }); } discoverStateResult.DissambiguationProblems.Add(discoverDissambiguation); } return(Ok(discoverStateResult)); } return(NotFound()); } catch (Exception ex) { return(Problem(ex.ToString())); } }
public IActionResult dataDiscover(IFormFile rdfFile) { try { XmlDocument rdf = SparqlUtility.GetRDFFromFile(rdfFile); DiscoverItem discoverItem = new DiscoverItem() { Rdf = rdf.InnerXml, Publish = false, DissambiguationProcessed = false, Status = "Pending" }; Guid addedID = _discoverItemService.AddDiscoverItem(discoverItem); _amqpService.PublishMessage(addedID); return(Ok(addedID)); } catch (Exception ex) { return(Problem(ex.ToString())); } }
public IActionResult dataPublish(IFormFile rdfFile, string jobId, bool discoverProcessed) { try { string idDiscoverItem = null; if (Request != null) { idDiscoverItem = Request.Query["idDiscoverItem"].ToString(); } if (!string.IsNullOrEmpty(idDiscoverItem)) { //Si viene el parametro 'idDiscoverItem' actualizamos un DiscoverItem ya existente. XmlDocument rdf = SparqlUtility.GetRDFFromFile(rdfFile); DiscoverItem discoverItem = _discoverItemService.GetDiscoverItemById(new Guid(idDiscoverItem)); discoverItem.DissambiguationProcessed = discoverProcessed; discoverItem.Publish = true; discoverItem.Status = "Pending"; discoverItem.DiscoverRdf = rdf.InnerXml; _amqpService.PublishMessage(idDiscoverItem); return(Ok()); } else { XmlDocument rdf = SparqlUtility.GetRDFFromFile(rdfFile); DiscoverItem discoverItem = new DiscoverItem() { JobID = jobId, Rdf = rdf.InnerXml, Publish = true, DissambiguationProcessed = discoverProcessed, Status = "Pending" }; Guid addedID = _discoverItemService.AddDiscoverItem(discoverItem); _amqpService.PublishMessage(addedID); return(Ok()); } } catch (Exception ex) { return(Problem(ex.ToString())); } }
///<summary> ///Modifica una item de descubrimiento ///</summary> ///<param name="discoverItem">tem de descubrimiento a modificar con los datos nuevos</param> public bool ModifyDiscoverItem(DiscoverItem discoverItem) { bool modified = false; DiscoverItem discoverItemOriginal = GetDiscoverItemById(discoverItem.ID); if (discoverItemOriginal != null) { discoverItemOriginal.Status = discoverItem.Status; discoverItemOriginal.Rdf = discoverItem.Rdf; discoverItemOriginal.DiscoverRdf = discoverItem.DiscoverRdf; discoverItemOriginal.Error = discoverItem.Error; discoverItemOriginal.JobID = discoverItem.JobID; discoverItemOriginal.Publish = discoverItem.Publish; discoverItemOriginal.DissambiguationProcessed = discoverItem.DissambiguationProcessed; discoverItemOriginal.DiscoverReport = discoverItem.DiscoverReport; discoverItemOriginal.DissambiguationProblems = discoverItem.DissambiguationProblems; discoverItemOriginal.DiscardDissambiguations = discoverItem.DiscardDissambiguations; discoverItemOriginal.LoadedEntities = discoverItem.LoadedEntities; modified = true; } return(modified); }
private async Task QueryTrending(DiscoverItem item) { CommonProvider common = new CommonProvider(); List <TrendingResult> result = new List <TrendingResult>(); if (pageIndex == 0) { if (TrendingList == null) { TrendingList = new ObservableCollection <TrendingResult>(); } else { TrendingList.Clear(); } } if (TypeSelectedItem.HotType != HotTopicType.None) { result = await common.QueryDiscover(TypeSelectedItem.HotType, pageIndex + 1, 20, new List <TrendingResult>(TrendingList)); } if (TypeSelectedItem.Type != TopicType.None) { int index = 0; if (TrendingList.Count != 0) { index = TrendingList[TrendingList.Count - 1].RecommendedAt - 1; } result = await common.QueryDiscover(TypeSelectedItem.Type, index, 20, 20); } result.ForEach(x => TrendingList.Add(x)); pageIndex++; }
public IActionResult ResolveDiscover(string idJob, string IdDiscoverItem, Dictionary <string, string> DissambiguationProblemsResolve) { DiscoverItem item = _discoverItemService.GetDiscoverItemById(new Guid(IdDiscoverItem)); //Cargamos el RDF RohGraph dataGraph = new RohGraph(); dataGraph.LoadFromString(item.DiscoverRdf, new RdfXmlParser()); //Modificamos el RDF TripleStore store = new TripleStore(); store.Add(dataGraph); //Cambiamos candidato.Key por entityID foreach (string uriOriginal in DissambiguationProblemsResolve.Keys) { if (!string.IsNullOrEmpty(DissambiguationProblemsResolve[uriOriginal])) { //En caso de que la resolución sea una URI de Unidata añadimos el SameAs if (!string.IsNullOrEmpty(_unidataPrefix.GetUnidataDomain()) && DissambiguationProblemsResolve[uriOriginal].StartsWith(_unidataPrefix.GetUnidataDomain())) { IUriNode t_subject = dataGraph.CreateUriNode(UriFactory.Create(uriOriginal)); IUriNode t_predicate = dataGraph.CreateUriNode(UriFactory.Create("http://www.w3.org/2002/07/owl#sameAs")); IUriNode t_object = dataGraph.CreateUriNode(UriFactory.Create(DissambiguationProblemsResolve[uriOriginal])); dataGraph.Assert(new Triple(t_subject, t_predicate, t_object)); } else { //En caso de que la resolución NO sea una URI de Unidata modificamos las URLs SparqlUpdateParser parser = new SparqlUpdateParser(); //Actualizamos los sujetos SparqlUpdateCommandSet updateSubject = parser.ParseFromString(@"DELETE { ?s ?p ?o. } INSERT{<" + DissambiguationProblemsResolve[uriOriginal] + @"> ?p ?o.} WHERE { ?s ?p ?o. FILTER(?s = <" + uriOriginal + @">) }"); //Actualizamos los objetos SparqlUpdateCommandSet updateObject = parser.ParseFromString(@"DELETE { ?s ?p ?o. } INSERT{?s ?p <" + DissambiguationProblemsResolve[uriOriginal] + @">.} WHERE { ?s ?p ?o. FILTER(?o = <" + uriOriginal + @">) }"); LeviathanUpdateProcessor processor = new LeviathanUpdateProcessor(store); processor.ProcessCommandSet(updateSubject); processor.ProcessCommandSet(updateObject); } } } System.IO.StringWriter sw = new System.IO.StringWriter(); RdfXmlWriter rdfXmlWriter = new RdfXmlWriter(); rdfXmlWriter.Save(dataGraph, sw); string rdfXml = sw.ToString(); Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(rdfXml)); FormFile file = new FormFile(stream, 0, stream.Length, "rdfFile", "rdf.xml"); //Actualizamos el item Dictionary <string, List <string> > discards = new Dictionary <string, List <string> >(); foreach (DiscoverItem.DiscoverDissambiguation dissambiguation in item.DissambiguationProblems) { if (DissambiguationProblemsResolve.ContainsKey(dissambiguation.IDOrigin) && DissambiguationProblemsResolve[dissambiguation.IDOrigin] == null) { discards.Add(dissambiguation.IDOrigin, dissambiguation.DissambiguationCandiates.Select(x => x.IDCandidate).ToList()); } } item.UpdateDissambiguationDiscards(discards, rdfXml); item.DiscoverRdf = rdfXml; item.Status = "Pending"; _discoverItemService.ModifyDiscoverItem(item); //Lo reencolamos corregido junto con su identificador _callEDtlPublishService.CallDataPublish(file, idJob, false, IdDiscoverItem); return(RedirectToAction("DetailsJob", "Job", new { id = idJob })); }
//TODO mover a otro sitio /// <summary> /// Realiza el proceso completo de desubrimiento sobre un RDF /// </summary> /// <param name="pDiscoverItem">Item de descubrimiento</param> /// <param name="pCallEtlApiService">Servicio para hacer llamadas a los métodos del controlador etl del API_CARGA </param> /// <param name="pCallUrisFactoryApiService">Servicio para hacer llamadas a los métodos del Uris Factory</param> /// <returns>DiscoverResult con los datos del descubrimiento</returns> private DiscoverResult Init(DiscoverItem pDiscoverItem, CallEtlApiService pCallEtlApiService, CallUrisFactoryApiService pCallUrisFactoryApiService) { #region Cargamos configuraciones ConfigSparql ConfigSparql = new ConfigSparql(); string SGI_SPARQLEndpoint = ConfigSparql.GetEndpoint(); string SGI_SPARQLGraph = ConfigSparql.GetGraph(); string SGI_SPARQLQueryParam = ConfigSparql.GetQueryParam(); string SGI_SPARQLUsername = ConfigSparql.GetUsername(); string SGI_SPARQLPassword = ConfigSparql.GetPassword(); string Unidata_SPARQLEndpoint = ConfigSparql.GetUnidataEndpoint(); string Unidata_SPARQLGraph = ConfigSparql.GetUnidataGraph(); string Unidata_SPARQLQueryParam = ConfigSparql.GetUnidataQueryParam(); string Unidata_SPARQLUsername = ConfigSparql.GetUnidataUsername(); string Unidata_SPARQLPassword = ConfigSparql.GetUnidataPassword(); ConfigService ConfigService = new ConfigService(); float MaxScore = ConfigService.GetMaxScore(); float MinScore = ConfigService.GetMinScore(); string UnidataDomain = ConfigService.GetUnidataDomain(); ConfigScopus ConfigScopus = new ConfigScopus(); string ScopusApiKey = ConfigScopus.GetScopusApiKey(); string ScopusUrl = ConfigScopus.GetScopusUrl(); ConfigCrossref ConfigCrossref = new ConfigCrossref(); string CrossrefUserAgent = ConfigCrossref.GetCrossrefUserAgent(); ConfigWOS ConfigWOS = new ConfigWOS(); string WOSAuthorization = ConfigWOS.GetWOSAuthorization(); #endregion DiscoverUtility discoverUtility = new DiscoverUtility(); DateTime discoverInitTime = DateTime.Now; //Cargamos la ontología if (_ontologyGraph == null) { _ontologyGraph = pCallEtlApiService.CallGetOntology(); } //Cargamos datos del RDF RohGraph dataGraph = new RohGraph(); Dictionary <string, HashSet <string> > discardDissambiguations = new Dictionary <string, HashSet <string> >(); if (!string.IsNullOrEmpty(pDiscoverItem.DiscoverRdf)) { //Si tenemos valor en DiscoverRdf, trabajamos con este RDF, ya que estamos reprocesando un rdf validado dataGraph.LoadFromString(pDiscoverItem.DiscoverRdf, new RdfXmlParser()); if (pDiscoverItem.DiscardDissambiguations != null) { foreach (DiscoverItem.DiscardDissambiguation discardDissambiguation in pDiscoverItem.DiscardDissambiguations) { if (!discardDissambiguations.ContainsKey(discardDissambiguation.IDOrigin)) { discardDissambiguations.Add(discardDissambiguation.IDOrigin, new HashSet <string>()); } discardDissambiguations[discardDissambiguation.IDOrigin].UnionWith(discardDissambiguation.DiscardCandidates); } } } else { dataGraph.LoadFromString(pDiscoverItem.Rdf, new RdfXmlParser()); } //Cargamos el razonador para inferir datos en la ontología RohRdfsReasoner reasoner = new RohRdfsReasoner(); reasoner.Initialise(_ontologyGraph); //Cargamos los datos con inferencia RohGraph dataInferenceGraph = dataGraph.Clone(); reasoner.Apply(dataInferenceGraph); //Datos para trabajar con la reconciliación ReconciliationData reconciliationData = new ReconciliationData(); //Datos para trabajar con el descubrimiento de enlaces DiscoverLinkData discoverLinkData = new DiscoverLinkData(); //Almacenamos las entidades con dudas acerca de su reonciliación Dictionary <string, Dictionary <string, float> > reconciliationEntitiesProbability = new Dictionary <string, Dictionary <string, float> >(); //Cargamos la caché global if (_discoverCacheGlobal == null) { _discoverCacheGlobal = new DiscoverCacheGlobal(); discoverUtility.LoadPersonWithName(_discoverCacheGlobal, SGI_SPARQLEndpoint, SGI_SPARQLGraph, SGI_SPARQLQueryParam, SGI_SPARQLUsername, SGI_SPARQLPassword); discoverUtility.LoadEntitiesWithTitle(_discoverCacheGlobal, SGI_SPARQLEndpoint, SGI_SPARQLGraph, SGI_SPARQLQueryParam, SGI_SPARQLUsername, SGI_SPARQLPassword); } if (!pDiscoverItem.DissambiguationProcessed) { bool hasChanges = true; //Cache del proceso de descubrimiento DiscoverCache discoverCache = new DiscoverCache(); //Se realizarán este proceso iterativamente hasta que no haya ningún cambio en lo que a reconciliaciones se refiere while (hasChanges) { hasChanges = false; //Preparamos los datos para proceder con la reconciliazción discoverUtility.PrepareData(dataGraph, reasoner, out dataInferenceGraph, out Dictionary <string, HashSet <string> > entitiesRdfTypes, out Dictionary <string, string> entitiesRdfType, out Dictionary <string, List <DisambiguationData> > disambiguationDataRdf, false); //Carga los scores de las personas //Aquí se almacenarán los nombres de las personas del RDF, junto con los candidatos de la BBDD y su score Dictionary <string, Dictionary <string, float> > namesScore = new Dictionary <string, Dictionary <string, float> >(); discoverUtility.LoadNamesScore(ref namesScore, dataInferenceGraph, discoverCache, _discoverCacheGlobal, MinScore, MaxScore); //0.- Macamos como reconciliadas aquellas que ya estén cargadas en la BBDD con los mismos identificadores List <string> entidadesCargadas = discoverUtility.LoadEntitiesDB(entitiesRdfType.Keys.ToList().Except(reconciliationData.reconciliatedEntityList.Keys.Union(reconciliationData.reconciliatedEntityList.Values)), SGI_SPARQLEndpoint, SGI_SPARQLGraph, SGI_SPARQLQueryParam, SGI_SPARQLUsername, SGI_SPARQLPassword).Keys.ToList(); foreach (string entitiID in entidadesCargadas) { reconciliationData.reconciliatedEntityList.Add(entitiID, entitiID); reconciliationData.reconciliatedEntitiesWithSubject.Add(entitiID); } //1.- Realizamos reconciliación con los identificadores configurados (y el roh:identifier) y marcamos como reconciliadas las entidades seleccionadas para no intentar reconciliarlas posteriormente discoverUtility.ReconciliateIDs(ref hasChanges, ref reconciliationData, entitiesRdfType, disambiguationDataRdf, discardDissambiguations, _ontologyGraph, ref dataGraph, discoverCache, SGI_SPARQLEndpoint, SGI_SPARQLQueryParam, SGI_SPARQLGraph, SGI_SPARQLUsername, SGI_SPARQLPassword); //2.- Realizamos la reconciliación con los datos del Propio RDF discoverUtility.ReconciliateRDF(ref hasChanges, ref reconciliationData, _ontologyGraph, ref dataGraph, reasoner, discardDissambiguations, discoverCache, _discoverCacheGlobal, MinScore, MaxScore); //3.- Realizamos la reconciliación con los datos de la BBDD discoverUtility.ReconciliateBBDD(ref hasChanges, ref reconciliationData, out reconciliationEntitiesProbability, _ontologyGraph, ref dataGraph, reasoner, namesScore, discardDissambiguations, discoverCache, _discoverCacheGlobal, MinScore, MaxScore, SGI_SPARQLEndpoint, SGI_SPARQLQueryParam, SGI_SPARQLGraph, SGI_SPARQLUsername, SGI_SPARQLPassword); //4.- Realizamos la reconciliación con los datos de las integraciones externas //TODO descomentar //discoverUtility.ExternalIntegration(ref hasChanges, ref reconciliationData, ref discoverLinkData, ref reconciliationEntitiesProbability, ref dataGraph, reasoner, namesScore,entitiesWithTitle, ontologyGraph, out Dictionary<string, ReconciliationData.ReconciliationScore> entidadesReconciliadasConIntegracionExternaAux, discardDissambiguations, discoverCache,discoverCacheGlobal, ScopusApiKey, ScopusUrl, CrossrefUserAgent, WOSAuthorization, MinScore, MaxScore, SGI_SPARQLEndpoint, SGI_SPARQLQueryParam, SGI_SPARQLGraph, SGI_SPARQLUsername, SGI_SPARQLPassword,pCallUrisFactoryApiService); //Eliminamos de las probabilidades aquellos que ya estén reconciliados foreach (string key in reconciliationData.reconciliatedEntityList.Keys) { reconciliationEntitiesProbability.Remove(key); } } //5.-Realizamos la detección de equivalencias con Unidata //TODO descomentar cuando esté habilitaado Unidata //TODO descomentar y revisar en unidata no tienen roh:identifier //discoverUtility.EquivalenceDiscover(ontologyGraph, ref dataGraph, reasoner, discoverCache, ref reconciliationEntitiesProbability, discardDissambiguations, UnidataDomain, MinScore, MaxScore, Unidata_SPARQLEndpoint, Unidata_SPARQLQueryParam, Unidata_SPARQLGraph, Unidata_SPARQLUsername, Unidata_SPARQLPassword); } //TODO comrpobar cuando esté habilitaado Unidata DateTime discoverEndTime = DateTime.Now; DiscoverResult resultado = new DiscoverResult(dataGraph, dataInferenceGraph, _ontologyGraph, reconciliationData, reconciliationEntitiesProbability, discoverInitTime, discoverEndTime, discoverLinkData); return(resultado); }
/// <summary> /// Procesa los resultados del descubrimiento /// </summary> /// <param name="pDiscoverItem">Objeto con los datos de com procesar el proeso de descubrimiento</param> /// <param name="pDiscoverResult">Resultado de la aplicación del descubrimiento</param> /// <param name="pDiscoverItemBDService">Clase para gestionar las operaciones de las tareas de descubrimiento</param> /// <param name="pCallCronApiService">Servicio para hacer llamadas a los métodos del apiCron</param> /// <param name="pCallUrisFactoryApiService">Servicio para hacer llamadas a los métodos del Uris Factory</param> /// <param name="pProcessDiscoverStateJobBDService">Clase para gestionar los estados de descubrimiento de las tareas</param> /// <returns></returns> public void Process(DiscoverItem pDiscoverItem, DiscoverResult pDiscoverResult, DiscoverItemBDService pDiscoverItemBDService, CallCronApiService pCallCronApiService, CallUrisFactoryApiService pCallUrisFactoryApiService, ProcessDiscoverStateJobBDService pProcessDiscoverStateJobBDService) { #region Cargamos configuraciones ConfigSparql ConfigSparql = new ConfigSparql(); string SGI_SPARQLEndpoint = ConfigSparql.GetEndpoint(); string SGI_SPARQLGraph = ConfigSparql.GetGraph(); string SGI_SPARQLQueryParam = ConfigSparql.GetQueryParam(); string SGI_SPARQLUsername = ConfigSparql.GetUsername(); string SGI_SPARQLPassword = ConfigSparql.GetPassword(); string Unidata_SPARQLEndpoint = ConfigSparql.GetUnidataEndpoint(); string Unidata_SPARQLGraph = ConfigSparql.GetUnidataGraph(); string Unidata_SPARQLQueryParam = ConfigSparql.GetUnidataQueryParam(); string Unidata_SPARQLUsername = ConfigSparql.GetUnidataUsername(); string Unidata_SPARQLPassword = ConfigSparql.GetUnidataPassword(); ConfigService ConfigService = new ConfigService(); string UnidataDomain = ConfigService.GetUnidataDomain(); string UnidataUriTransform = ConfigService.GetUnidataUriTransform(); #endregion /* * En función del resultado obtenido se realiza una de las siguientes acciones: * Si para alguna entidad hay más de un candidato que supere el umbral máximo o hay alguna entidad que supere el umbral mínimo pero no alcance el máximo se agregará el RDF a una BBDD junto con todos los datos necesarios para que el administrador decida como proceder. * Si no estamos en el punto anterior * Se obtienen las entidades principales del RDF y se eliminan todos los triples que haya en la BBDD en los que aparezcan como sujeto u objeto. * Se eliminan todos los triples cuyo sujeto y predicado estén en el RDF a cargar y estén marcados como monovaluados. * Se vuelcan los triples a la BBDD. * * */ if (pDiscoverItem.Publish) { if (pDiscoverResult.discoveredEntitiesProbability.Count > 0) { //Hay dudas en la desambiguación, por lo que lo actualizamos en la BBDD con su estado correspondiente pDiscoverItem.UpdateDissambiguationProblems( pDiscoverResult.discoveredEntitiesProbability, pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD.Values.Select(x => x.uri).Union(pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds.Values).Union(pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject).Union(pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration.Values.Select(x => x.uri)).ToList(), pDiscoverResult.GetDataGraphRDF()); pDiscoverItemBDService.ModifyDiscoverItem(pDiscoverItem); } else { string urlDiscoverAgent = pCallUrisFactoryApiService.GetUri("Agent", "discover"); //Creamos los SameAs hacia unidata para las entidades que NO lo tengan hacia Unidata //TODO descomentar cuando esté habilitaado Unidata //pDiscoverResult.dataGraph = AsioPublication.CreateUnidataSameAs(pDiscoverResult.dataGraph, UnidataDomain, UnidataUriTransform); //Publicamos en el SGI AsioPublication asioPublication = new AsioPublication(SGI_SPARQLEndpoint, SGI_SPARQLQueryParam, SGI_SPARQLGraph, SGI_SPARQLUsername, SGI_SPARQLPassword); asioPublication.PublishRDF(pDiscoverResult.dataGraph, pDiscoverResult.ontologyGraph, new KeyValuePair <string, string>(urlDiscoverAgent, "Algoritmos de descubrimiento"), pDiscoverResult.start, pDiscoverResult.end, pDiscoverResult.discoverLinkData, pCallUrisFactoryApiService); //TODO Lógica nombres de personas SparqlResultSet sparqlResultSet = (SparqlResultSet)pDiscoverResult.dataGraph.ExecuteQuery("select ?s ?name where {?s a <http://purl.org/roh/mirror/foaf#Person>. ?s <http://purl.org/roh/mirror/foaf#name> ?name}"); foreach (SparqlResult sparqlResult in sparqlResultSet.Results) { string s = sparqlResult["s"].ToString(); string nombre = sparqlResult["name"].ToString(); if (sparqlResult["name"] is ILiteralNode) { nombre = ((ILiteralNode)sparqlResult["name"]).Value; } _discoverCacheGlobal.PersonsNormalizedNames[s] = DiscoverUtility.NormalizeName(nombre); } //TODO Lógica titles SparqlResultSet sparqlResultSetTitles = (SparqlResultSet)pDiscoverResult.dataGraph.ExecuteQuery("select * where {?s a ?rdftype. ?s <http://purl.org/roh#title> ?title}"); foreach (SparqlResult sparqlResult in sparqlResultSetTitles.Results) { string s = sparqlResult["s"].ToString(); string rdftype = sparqlResult["rdftype"].ToString(); string title = sparqlResult["title"].ToString(); if (sparqlResult["title"] is ILiteralNode) { title = ((ILiteralNode)sparqlResult["title"]).Value; } title = DiscoverUtility.NormalizeTitle(title); if (!_discoverCacheGlobal.EntitiesNormalizedTitles.ContainsKey(rdftype)) { _discoverCacheGlobal.EntitiesNormalizedTitles.Add(rdftype, new Dictionary <string, HashSet <string> >()); } if (!_discoverCacheGlobal.EntitiesNormalizedTitles[rdftype].ContainsKey(title)) { _discoverCacheGlobal.EntitiesNormalizedTitles[rdftype].Add(title, new HashSet <string>()); } _discoverCacheGlobal.EntitiesNormalizedTitles[rdftype][title].Add(s); } //TODO descomentar cuando esté habilitaado Unidata if (false) { //Publicamos en UNIDATA AsioPublication asioPublicationUnidata = new AsioPublication(Unidata_SPARQLEndpoint, Unidata_SPARQLQueryParam, Unidata_SPARQLGraph, Unidata_SPARQLUsername, Unidata_SPARQLPassword); // Prepara el grafo para su carga en Unidata, para ello coge las URIs de Unidata del SameAs y la aplica a los sujetos y los antiguos sujetos se agregan al SameAs RohGraph unidataGraph = AsioPublication.TransformUrisToUnidata(pDiscoverResult.dataGraph, UnidataDomain, UnidataUriTransform); //Eliminamos los triples de crisIdentifier ya que no hay que volcarlos en unidata { TripleStore store = new TripleStore(); store.Add(unidataGraph); SparqlUpdateParser parser = new SparqlUpdateParser(); //Actualizamos los sujetos SparqlUpdateCommandSet updateSubject = parser.ParseFromString( @" DELETE { ?s ?p ?o. } WHERE { ?s ?p ?o. FILTER(?p =<" + mPropertySGIRohCrisIdentifier + @">) }"); LeviathanUpdateProcessor processor = new LeviathanUpdateProcessor(store); processor.ProcessCommandSet(updateSubject); } asioPublicationUnidata.PublishRDF(unidataGraph, pDiscoverResult.ontologyGraph, new KeyValuePair <string, string>(urlDiscoverAgent, "Algoritmos de descubrimiento"), pDiscoverResult.start, pDiscoverResult.end, pDiscoverResult.discoverLinkData, pCallUrisFactoryApiService); } //Lo marcamos como procesado en la BBDD y eliminamos sus metadatos pDiscoverItem.UpdateProcessed(); pDiscoverItemBDService.ModifyDiscoverItem(pDiscoverItem); } //Actualizamos el estado de descubrimiento de la tarea si el estado encolado esta en estado Succeeded o Failed (ha finalizado) //TODO cambiar por query a BBDD string statusQueueJob = pCallCronApiService.GetJob(pDiscoverItem.JobID).State; if ((statusQueueJob == "Failed" || statusQueueJob == "Succeeded")) { ProcessDiscoverStateJob processDiscoverStateJob = pProcessDiscoverStateJobBDService.GetProcessDiscoverStateJobByIdJob(pDiscoverItem.JobID); string state; //Actualizamos a error si existen items en estado error o con problemas de desambiguación if (pDiscoverItemBDService.ExistsDiscoverItemsErrorOrDissambiguatinProblems(pDiscoverItem.JobID)) { state = "Error"; } else if (pDiscoverItemBDService.ExistsDiscoverItemsPending(pDiscoverItem.JobID)) { //Actualizamos a 'Pending' si aún existen items pendientes state = "Pending"; } else { //Actualizamos a Success si no existen items en estado error ni con problemas de desambiguación y no hay ninguno pendiente state = "Success"; } if (processDiscoverStateJob != null) { processDiscoverStateJob.State = state; pProcessDiscoverStateJobBDService.ModifyProcessDiscoverStateJob(processDiscoverStateJob); } else { processDiscoverStateJob = new ProcessDiscoverStateJob() { State = state, JobId = pDiscoverItem.JobID }; pProcessDiscoverStateJobBDService.AddProcessDiscoverStateJob(processDiscoverStateJob); } } } else { //Actualizamos en BBDD DiscoverItem discoverItemBD = pDiscoverItemBDService.GetDiscoverItemById(pDiscoverItem.ID); //Reporte de descubrimiento string discoverReport = "Time processed (seconds): " + pDiscoverResult.secondsProcessed + "\n"; if (pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject != null && pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject.Count > 0) { discoverReport += "Entities discover with the same uri: " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject.Count + "\n"; foreach (string uri in pDiscoverResult.reconciliationData.reconciliatedEntitiesWithSubject) { discoverReport += "\t" + uri + "\n"; } } if (pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds != null && pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds.Count > 0) { discoverReport += "Entities discover with some common identifier: " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds.Count + "\n"; foreach (string uri in pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds.Keys) { discoverReport += "\t" + uri + " --> " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithIds[uri] + "\n"; } } if (pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD != null && pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD.Count > 0) { discoverReport += "Entities discover with reconciliation config: " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD.Count + "\n"; foreach (string uri in pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD.Keys) { discoverReport += "\t" + uri + " --> " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithBBDD[uri] + "\n"; } } if (pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration != null && pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration.Count > 0) { discoverReport += "Entities discover with external integrations config: " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration.Count + "\n"; foreach (string uri in pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration.Keys) { discoverReport += "\t" + uri + " --> " + pDiscoverResult.reconciliationData.reconciliatedEntitiesWithExternalIntegration[uri] + "\n"; } } if (pDiscoverResult.discoverLinkData != null && pDiscoverResult.discoverLinkData.entitiesProperties != null && pDiscoverResult.discoverLinkData.entitiesProperties.Count > 0) { discoverReport += "Entities with identifiers obtained with External integration: " + pDiscoverResult.discoverLinkData.entitiesProperties.Count + "\n"; foreach (string uri in pDiscoverResult.discoverLinkData.entitiesProperties.Keys) { foreach (DiscoverLinkData.PropertyData property in pDiscoverResult.discoverLinkData.entitiesProperties[uri]) { foreach (string value in property.valueProvenance.Keys) { foreach (string provenance in property.valueProvenance[value]) { discoverReport += "\t" + uri + " - " + property.property + " - " + value + " --> " + provenance + "\n"; } } } } } discoverItemBD.UpdateReport(pDiscoverResult.discoveredEntitiesProbability, pDiscoverResult.GetDataGraphRDF(), discoverReport); pDiscoverItemBDService.ModifyDiscoverItem(discoverItemBD); } }