コード例 #1
0
        public async Task <ResourceWriteResultCTO> EditResource(Uri pidUri, ResourceRequestDTO resourceRequest)
        {
            _validationService.CheckInstantiableEntityType(resourceRequest);

            using (var lockService = _lockServiceFactory.CreateLockService())
            {
                await lockService.CreateLockAsync(pidUri.ToString());

                var resourcesCTO = GetResourcesByPidUri(pidUri);
                var id           = resourcesCTO.GetDraftOrPublishedVersion().Id;

                if (resourcesCTO.HasPublishedAndNoDraft)
                {
                    id = CreateNewResourceId();
                }

                var(validationResult, failed, validationFacade) =
                    await _resourcePreprocessService.ValidateAndPreProcessResource(id, resourceRequest, resourcesCTO,
                                                                                   ResourceCrudAction.Update);

                HandleValidationFailures(resourcesCTO.GetDraftOrPublishedVersion(), id, validationResult, failed,
                                         validationFacade);

                using (var transaction = _resourceRepository.CreateTransaction())
                {
                    // try deleting draft version and all inbound edges are changed to the new entry.
                    _resourceRepository.DeleteDraft(validationFacade.RequestResource.PidUri,
                                                    new Uri(validationFacade.RequestResource.Id));

                    // all inbound edges pointing to an entry of a pid uri(published entry) will be duplicated to the request id as well.
                    _resourceRepository.Relink(pidUri, new Uri(validationFacade.RequestResource.Id));

                    if (resourcesCTO.HasDraft)
                    {
                        _identifierService.DeleteAllUnpublishedIdentifiers(resourcesCTO.Draft);
                    }

                    _resourceRepository.Create(validationFacade.RequestResource, validationFacade.MetadataProperties);

                    CreateHasPidEntryDraftProperty(validationFacade.RequestResource.PidUri);

                    transaction.Commit();
                }

                // Check whether the correct entity type is specified -> throw exception
                return(new ResourceWriteResultCTO(validationFacade.RequestResource, validationResult));
            }
        }
コード例 #2
0
        /// <summary>
        /// Read resources from SQS que and Validate
        /// </summary>
        /// <returns></returns>
        private async Task AddUpdateResources()
        {
            _logger.LogInformation("BackgroundService: Running.... ");
            Stopwatch stpWatch = new Stopwatch();

            stpWatch.Start();
            //Check for msgs in a loop
            int msgcount, totalMsgCount = 0;

            try
            {
                do
                {
                    //Check msgs available in SQS
                    var msgs = await _awsSQSHelper.ReceiveResourceMessageAsync();

                    msgcount       = msgs.Count;
                    totalMsgCount += msgs.Count;

                    //get Instance graphs if there is msg to process

                    Uri resInstanceGraph   = null;
                    Uri draftInstanceGraph = null;
                    if (msgs.Count > 0)
                    {
                        _logger.LogInformation("BackgroundService: Found {count} messages from resource queue ", msgs.Count);
                        if (resInstanceGraph == null)
                        {
                            resInstanceGraph = _metadataService.GetInstanceGraph(PIDO.PidConcept);
                        }

                        if (draftInstanceGraph == null)
                        {
                            draftInstanceGraph = _metadataService.GetInstanceGraph("draft");
                        }

                        //List to collect ValidationFacade of each resource
                        List <BulkUploadResult> totalValidationResult = new List <BulkUploadResult>();

                        //Iterate on each msg which will contain list of resource
                        foreach (var msg in msgs)
                        {
                            ResourceRequestDTO resource;
                            // Try Get reources from the msg
                            try
                            {
                                resource = JsonConvert.DeserializeObject <ResourceRequestDTO>(msg.Body);
                            }
                            catch (System.Exception ex)
                            {
                                //Collect result
                                totalValidationResult.Add(new BulkUploadResult
                                {
                                    ActionDone         = "Error",
                                    ErrorMessage       = "Unable to Deserialize the resource",
                                    TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                    pidUri             = "",
                                    ResourceLabel      = ex.Message,
                                    ResourceDefinition = ""
                                });
                                // Delete msg from input Queue
                                if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
                                {
                                    _logger.LogInformation("BackgroundService: Could not delete meessage");
                                }
                                continue;
                            }

                            //Extract pidUri
                            var hasPid = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.hasPID, true);
                            Uri pidUri = null;
                            try
                            {
                                if (hasPid != null && ((COLID.Graph.TripleStore.DataModels.Base.Entity)hasPid).Id != string.Empty)
                                {
                                    pidUri = new Uri(((COLID.Graph.TripleStore.DataModels.Base.Entity)hasPid).Id);
                                }
                            }
                            catch
                            {
                                pidUri = null;
                            }
                            //Check SourceId
                            string srcId = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasSourceID, true);
                            if (srcId == null)
                            {
                                //Collect result
                                totalValidationResult.Add(new BulkUploadResult
                                {
                                    ActionDone         = "Error",
                                    ErrorMessage       = "SourceId not found.",
                                    TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                    pidUri             = pidUri == null ? "" : pidUri.ToString(),
                                    ResourceLabel      = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
                                    ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
                                    StateItems         = resource.StateItems
                                });
                                // Delete msg from input Queue
                                if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
                                {
                                    _logger.LogInformation("BackgroundService: Could not delete meessage");
                                }
                                continue;
                            }

                            //Check Entity Type
                            try
                            {
                                _validationService.CheckInstantiableEntityType(resource);
                            }
                            catch (System.Exception ex)
                            {
                                //Collect result
                                totalValidationResult.Add(new BulkUploadResult
                                {
                                    ActionDone         = "Error",
                                    ErrorMessage       = ex.Message,
                                    TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                    pidUri             = pidUri == null ? "" : pidUri.ToString(),
                                    SourceId           = srcId,
                                    ResourceLabel      = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
                                    ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
                                    StateItems         = resource.StateItems
                                });
                                // Delete msg from input Queue
                                if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
                                {
                                    _logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
                                }
                                continue;
                            }

                            ResourcesCTO resourcesCTO = new ResourcesCTO();
                            bool         pidUriExistsinTripleStore = false;

                            if (pidUri == null)
                            {
                                //Check Whether resource is already present in Neptune (using SourceId)
                                ISet <Uri> resourceInstanceGraphs = new HashSet <Uri>();
                                resourceInstanceGraphs.Add(resInstanceGraph);
                                resourceInstanceGraphs.Add(draftInstanceGraph);
                                pidUri = _resourceRepository.GetPidUriBySourceId(srcId, resourceInstanceGraphs);
                            }
                            else
                            {
                                try
                                {
                                    resourcesCTO = _resourceService.GetResourcesByPidUri(pidUri);
                                    pidUriExistsinTripleStore = true;
                                }
                                catch
                                {
                                    pidUriExistsinTripleStore = false;
                                }
                            }

                            //if Pid Uri is null then Add
                            if (pidUri == null || (pidUri != null && pidUriExistsinTripleStore == false))
                            {
                                try
                                {
                                    string newResourceId = CreateNewResourceId();
                                    _logger.LogInformation("BackgroundService: About to Validate New resource: {msg}", msg.Body);
                                    //Validate
                                    var(validationResult, failed, validationFacade) =
                                        await _resourcePreprocessService.ValidateAndPreProcessResource(newResourceId, resource, new ResourcesCTO(), ResourceCrudAction.Create);

                                    _logger.LogInformation("BackgroundService: Validation Complete for: {srcId} having status {stat}", srcId, failed.ToString());

                                    //Update pidUri in stateItem
                                    if (failed == false && resource.StateItems[0].ContainsKey("pid_uri"))
                                    {
                                        resource.StateItems[0]["pid_uri"] = validationFacade.RequestResource.PidUri.ToString();
                                    }

                                    //Create result data
                                    BulkUploadResult result = new BulkUploadResult
                                    {
                                        ActionDone         = failed ? "Error" : "Validated",
                                        ErrorMessage       = failed ? "Validation Failed while Adding the resource." : string.Empty,
                                        Results            = validationResult.Results,
                                        Triples            = validationResult.Triples.Replace(ColidEntryLifecycleStatus.Draft, ColidEntryLifecycleStatus.Published),
                                        InstanceGraph      = resInstanceGraph.ToString(),
                                        TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                        pidUri             = validationFacade.RequestResource.PidUri.ToString(),
                                        ResourceId         = newResourceId,
                                        SourceId           = srcId,
                                        ResourceLabel      = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
                                        ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
                                        StateItems         = resource.StateItems
                                    };

                                    //Get distribution
                                    if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.Distribution))
                                    {
                                        List <dynamic> distList = resource.Properties[Graph.Metadata.Constants.Resource.Distribution];
                                        foreach (dynamic dist in distList)
                                        {
                                            string EndPointPidUri = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties[Graph.Metadata.Constants.Resource.hasPID][0].Id;
                                            string EndPointUrl    = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.DistributionEndpoints.HasNetworkAddress, true);
                                            result.DistributionEndPoint.Add(EndPointPidUri, EndPointUrl);
                                        }
                                    }

                                    //Collect Result
                                    totalValidationResult.Add(result);

                                    //if validation passed then Update Nginx Proxy info for the resource DynamoDB
                                    if (!failed)
                                    {
                                        _proxyConfigService.AddUpdateNginxConfigRepository(resource);
                                    }
                                }
                                catch (System.Exception ex)
                                {
                                    //Collect result
                                    totalValidationResult.Add(new BulkUploadResult
                                    {
                                        ActionDone         = "Error",
                                        ErrorMessage       = ex.Message,
                                        TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                        pidUri             = pidUri == null ? "" : pidUri.ToString(),
                                        SourceId           = srcId,
                                        ResourceLabel      = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
                                        ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true)
                                    });

                                    // Delete msg from input Queue
                                    if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
                                    {
                                        _logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
                                    }
                                    continue;
                                }
                            }
                            else //Update Resource
                            {
                                try
                                {
                                    //var resourcesCTO = _resourceService.GetResourcesByPidUri(pidUri);   // Draft und Published resource getrennt behandeln.
                                    var id = resourcesCTO.GetDraftOrPublishedVersion().Id; // Draft und Published resource getrennt behandeln.

                                    //Update resource with PidUri
                                    if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.hasPID))
                                    {
                                        ((COLID.Graph.TripleStore.DataModels.Base.Entity)resource.Properties[Graph.Metadata.Constants.Resource.hasPID][0]).Id = pidUri.ToString();
                                    }

                                    _logger.LogInformation("BackgroundService: About to Validate Existing resource: {msg}", msg.Body);
                                    var(validationResult, failed, validationFacade) =
                                        await _resourcePreprocessService.ValidateAndPreProcessResource(id, resource, resourcesCTO, ResourceCrudAction.Publish, false, null);

                                    _logger.LogInformation("BackgroundService: Validation Complete for: {srcId} having status {stat}", srcId, failed.ToString());

                                    // The validation failed, if the results are cricital errors.
                                    if (failed)
                                    {
                                        //Collect result
                                        totalValidationResult.Add(new BulkUploadResult
                                        {
                                            ActionDone         = "Error",
                                            ErrorMessage       = "Validation Failed while updating the resource.",
                                            Results            = validationResult.Results,
                                            TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                            pidUri             = validationFacade.RequestResource.PidUri.ToString(),
                                            ResourceId         = id,
                                            SourceId           = srcId,
                                            ResourceLabel      = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
                                            ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
                                            StateItems         = resource.StateItems
                                        });
                                        // Delete msg from input Queue
                                        if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
                                        {
                                            _logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
                                        }
                                        continue;
                                    }

                                    if (resourcesCTO.HasPublished && (!_resourceService.ResourceHasChanged(resourcesCTO.Published, validationFacade.RequestResource)))
                                    {
                                        //Collect result
                                        totalValidationResult.Add(new BulkUploadResult
                                        {
                                            ActionDone         = "Error",
                                            ErrorMessage       = "No changes found in this resource.",
                                            TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                            pidUri             = validationFacade.RequestResource.PidUri.ToString(),
                                            ResourceId         = id,
                                            SourceId           = srcId,
                                            ResourceLabel      = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
                                            ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
                                            StateItems         = resource.StateItems
                                        });
                                        // Delete msg from input Queue
                                        if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
                                        {
                                            _logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
                                        }
                                        continue;
                                    }

                                    var resourcetoCreate = _resourceService.SetHasLaterVersionResourceId(validationFacade.RequestResource);
                                    using (var transaction = _resourceRepository.CreateTransaction())
                                    {
                                        // try deleting draft version and all inbound edges are changed to the new entry.
                                        _resourceRepository.DeleteDraft(validationFacade.RequestResource.PidUri,
                                                                        new Uri(validationFacade.RequestResource.Id), draftInstanceGraph);

                                        if (resourcesCTO.HasDraft)
                                        {
                                            _identifierService.DeleteAllUnpublishedIdentifiers(resourcesCTO.Draft);
                                        }

                                        if (resourcesCTO.HasPublished)
                                        {
                                            // Try to delete published and all inbound edges are changed to the new entry.
                                            _resourceRepository.DeletePublished(validationFacade.RequestResource.PidUri,
                                                                                new Uri(validationFacade.RequestResource.Id), resInstanceGraph);
                                        }

                                        //Add existing revision to the new resource
                                        if (resourcesCTO.Published != null)
                                        {
                                            var existingRevisions = resourcesCTO.Published.Properties.TryGetValue(COLID.Graph.Metadata.Constants.Resource.HasRevision, out List <dynamic> revisionValues) ? revisionValues : null;
                                            if (existingRevisions != null)
                                            {
                                                resourcetoCreate.Properties.Add(COLID.Graph.Metadata.Constants.Resource.HasRevision, existingRevisions);
                                            }
                                        }

                                        //Add Published
                                        _resourceRepository.Create(resourcetoCreate, validationFacade.MetadataProperties, resInstanceGraph);

                                        //Update revision
                                        if (resourcesCTO.Published == null)
                                        {
                                            await _revisionService.InitializeResourceInAdditionalsGraph(resourcetoCreate, validationFacade.MetadataProperties);
                                        }
                                        else
                                        {
                                            Graph.Metadata.DataModels.Resources.Resource updatedResource = await _revisionService.AddAdditionalsAndRemovals(resourcesCTO.Published, validationFacade.RequestResource);
                                        }

                                        _logger.LogInformation("BackgroundService: Commit - {sparqlQuery}", transaction.GetSparqlString());

                                        transaction.Commit();
                                        //Index resource
                                        //_indexingService.IndexPublishedResource(pidUri, updatedResource, validationFacade.ResourcesCTO);
                                    }
                                    _logger.LogInformation("BackgroundService: Resource updated having sourceId {sourceId} ", srcId);
                                    //Create result data
                                    BulkUploadResult result = new BulkUploadResult
                                    {
                                        ActionDone         = "Updated",
                                        TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                        pidUri             = pidUri == null ? "" : pidUri.ToString(),
                                        ResourceId         = id,
                                        SourceId           = srcId,
                                        ResourceLabel      = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
                                        ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
                                        StateItems         = resource.StateItems
                                    };

                                    //Get distribution
                                    if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.Distribution))
                                    {
                                        List <dynamic> distList = resource.Properties[Graph.Metadata.Constants.Resource.Distribution];
                                        foreach (dynamic dist in distList)
                                        {
                                            string EndPointPidUri = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties[Graph.Metadata.Constants.Resource.hasPID][0].Id;
                                            string EndPointUrl    = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.DistributionEndpoints.HasNetworkAddress, true);
                                            result.DistributionEndPoint.Add(EndPointPidUri, EndPointUrl);
                                        }
                                    }

                                    //Collect Result
                                    totalValidationResult.Add(result);

                                    //Update Nginx Proxy info for the resource DynamoDB
                                    _proxyConfigService.AddUpdateNginxConfigRepository(resource);
                                }
                                catch (System.Exception ex)
                                {
                                    _logger.LogInformation("BackgroundService:  Error while updating - {msg} ", ex.Message);
                                    //Collect result
                                    totalValidationResult.Add(new BulkUploadResult
                                    {
                                        ActionDone         = "Error",
                                        ErrorMessage       = ex.Message,
                                        TimeTaken          = stpWatch.ElapsedMilliseconds.ToString(),
                                        pidUri             = pidUri == null ? "" : pidUri.ToString(),
                                        SourceId           = srcId,
                                        ResourceLabel      = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
                                        ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
                                        StateItems         = resource.StateItems
                                    });

                                    // Delete msg from input Queue
                                    if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
                                    {
                                        _logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
                                    }
                                    continue;
                                }
                            }

                            // Delete msg from input Queue
                            if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
                            {
                                _logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
                            }
                        }

                        //Send msg to output queue
                        try
                        {
                            await _awsSQSHelper.SendResourceMessageAsync(totalValidationResult);
                        }
                        catch (System.Exception ex)
                        {
                            _logger.LogInformation("BackgroundService: Could not send meessage to output queue {msg} ", ex.Message);
                        }
                    }
                } while (msgcount > 0);
            }
            catch (System.Exception ex)
            {
                _logger.LogError("BackgroundService: " + ex.InnerException == null ? ex.Message : ex.InnerException.Message);
            }

            stpWatch.Stop();
            if (totalMsgCount > 0)
            {
                //_logger.LogInformation("BackgroundService: Processed {totMsgCount} messages in {milsec} Milliseconds.", totalMsgCount, stpWatch.ElapsedMilliseconds);
            }
        }