public void IndexUnlinkedResource(Uri pidUri, ResourcesCTO resource, Uri unlinkedPidUri, ResourcesCTO unlinkedListResource)
{
var resourceIndex = new ResourceIndexingDTO(ResourceCrudAction.Unlinking, pidUri, resource.GetDraftOrPublishedVersion(), resource);
IndexResource(resourceIndex);
IndexLinkedResource(unlinkedPidUri, unlinkedListResource.GetDraftOrPublishedVersion(), unlinkedListResource);
}
public async Task <Tuple <ValidationResult, bool, EntityValidationFacade> > ValidateAndPreProcessResource(string resourceId, ResourceRequestDTO resourceRequestDTO,
ResourcesCTO resourcesCTO, ResourceCrudAction resourceCrudAction, bool nestedValidation = false, string consumerGroup = null, bool changeResourceType = false)
{
var requestResource = _mapper.Map <Resource>(resourceRequestDTO);
requestResource.Id = string.IsNullOrWhiteSpace(resourceId) ? CreateNewResourceId() : resourceId;
string entityType = requestResource.Properties.GetValueOrNull(Graph.Metadata.Constants.RDF.Type, true).ToString();
var metadata = _metadataService.GetMetadataForEntityType(entityType);
// If it is a nested validation (example distribution endpoint), the consumer group of the parent must be included in the process.
var actualConsumerGroup = nestedValidation ? consumerGroup : requestResource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasConsumerGroup, true);
var validationFacade = new EntityValidationFacade(resourceCrudAction, requestResource, resourcesCTO, resourceRequestDTO.HasPreviousVersion, metadata, actualConsumerGroup);
if (changeResourceType)
{
// dirty solution for changing resource type. could be refactored in the future
var validationRes = await _validationService.ValidateEntity(requestResource, metadata).ConfigureAwait(true);
if (validationRes.Results.Count == 1 && validationRes.Results[0].Path == Graph.Metadata.Constants.Resource.BaseUri)
{
validationFacade.ResourceCrudAction = ResourceCrudAction.Create;
}
}
// Remove passed properties for several properties and replace it with repo-resource properties afterwards
RemoveProperty(Graph.Metadata.Constants.Resource.HasLaterVersion, requestResource);
//RemoveProperty(Graph.Metadata.Constants.Resource.HasHistoricVersion, requestResource);
RemoveProperty(Graph.Metadata.Constants.Resource.ChangeRequester, requestResource);
if (resourceCrudAction != ResourceCrudAction.Create)
{
UpdatePropertyFromRepositoryResource(Graph.Metadata.Constants.Resource.HasLaterVersion, validationFacade);
//validationFacade.RequestResource.Properties.AddOrUpdate(Graph.Metadata.Constants.Resource.MetadataGraphConfiguration, new List<dynamic>() { _metadataConfigService.GetLatestConfiguration().Id });
}
if (!nestedValidation) // so only new resources get this property, no distribution endpoints
{
RemoveProperty(Graph.Metadata.Constants.Resource.MetadataGraphConfiguration, requestResource);
validationFacade.RequestResource.Properties.AddOrUpdate(Graph.Metadata.Constants.Resource.MetadataGraphConfiguration, new List <dynamic>()
{
_metadataConfigService.GetLatestConfiguration().Id
});
}
// Each property have to be valiated and in same cases transformed
var keys = requestResource.Properties.Keys.ToList();
foreach (var key in keys)
{
var property = new KeyValuePair <string, List <dynamic> >(key, requestResource.Properties[key]);
if (changeResourceType && (property.Key == Graph.Metadata.Constants.RDF.Type))
{
continue;
}
_entityPropertyValidator.Validate(key, validationFacade);
await ValidateEndpoint(property, validationFacade);
}
// The following processes may only be executed for the main entry, so that the function already ends here with nested validations.
if (nestedValidation)
{
var nestedValidationResult = new ValidationResult()
{
Results = validationFacade.ValidationResults
};
var failedValidation = !nestedValidationResult.Conforms &&
nestedValidationResult.Severity != ValidationResultSeverity.Info;
return(new Tuple <ValidationResult, bool, EntityValidationFacade>(nestedValidationResult, failedValidation, validationFacade));
}
var validationResult = await _validationService.ValidateEntity(requestResource, metadata).ConfigureAwait(true);
validationResult.Results = validationResult.Results.Select(r =>
{
r.ResultSeverity = IsWarningSeverity(r, resourceCrudAction) ? ValidationResultSeverity.Warning : r.ResultSeverity;
return(r);
}).ToList();
string validationResourceId = validationFacade.ResourceCrudAction == ResourceCrudAction.Create ? null : resourcesCTO.GetDraftOrPublishedVersion().Id;
var duplicateResults = _identifierValidationService.CheckDuplicates(requestResource, validationResourceId, resourceRequestDTO.HasPreviousVersion);
if (changeResourceType)
{
duplicateResults = duplicateResults.ToList().FindAll(r => r.Path != Graph.Metadata.Constants.Resource.hasPID);
}
// Check whether forbidden properties are contained in the entity.
var forbiddenPropertiesResults = _validationService.CheckForbiddenProperties(requestResource);
// TODO: Concat or AddRange check
validationResult.Results = validationResult.Results.Concat(validationFacade.ValidationResults).Concat(duplicateResults).Concat(forbiddenPropertiesResults).OrderBy(t => t.ResultSeverity).ToList();
var failed = ProcessFailed(validationResult, resourceCrudAction);
// dirty solution for changing resource type (see also above)
validationFacade.ResourceCrudAction = changeResourceType ? ResourceCrudAction.Update : validationFacade.ResourceCrudAction;
if (failed)
{
// Reset the lifecycle Status to the correct value
if (resourceCrudAction == ResourceCrudAction.Update)
{
requestResource.Properties.AddOrUpdate(Graph.Metadata.Constants.Resource.HasEntryLifecycleStatus, new List <dynamic>()
{
resourcesCTO.HasDraft?Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.Draft: Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.Published
});
}
}
else
{
if (resourceCrudAction == ResourceCrudAction.Update && resourcesCTO.HasPublished)
{
requestResource.PublishedVersion = resourcesCTO.Published.Id;
}
}
return(new Tuple <ValidationResult, bool, EntityValidationFacade>(validationResult, failed, validationFacade));
}
/// <summary>
/// Read resources from SQS que and Validate
/// </summary>
/// <returns></returns>
private async Task AddUpdateResources()
{
_logger.LogInformation("BackgroundService: Running.... ");
Stopwatch stpWatch = new Stopwatch();
stpWatch.Start();
//Check for msgs in a loop
int msgcount, totalMsgCount = 0;
try
{
do
{
//Check msgs available in SQS
var msgs = await _awsSQSHelper.ReceiveResourceMessageAsync();
msgcount = msgs.Count;
totalMsgCount += msgs.Count;
//get Instance graphs if there is msg to process
Uri resInstanceGraph = null;
Uri draftInstanceGraph = null;
if (msgs.Count > 0)
{
_logger.LogInformation("BackgroundService: Found {count} messages from resource queue ", msgs.Count);
if (resInstanceGraph == null)
{
resInstanceGraph = _metadataService.GetInstanceGraph(PIDO.PidConcept);
}
if (draftInstanceGraph == null)
{
draftInstanceGraph = _metadataService.GetInstanceGraph("draft");
}
//List to collect ValidationFacade of each resource
List <BulkUploadResult> totalValidationResult = new List <BulkUploadResult>();
//Iterate on each msg which will contain list of resource
foreach (var msg in msgs)
{
ResourceRequestDTO resource;
// Try Get reources from the msg
try
{
resource = JsonConvert.DeserializeObject <ResourceRequestDTO>(msg.Body);
}
catch (System.Exception ex)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = "Unable to Deserialize the resource",
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = "",
ResourceLabel = ex.Message,
ResourceDefinition = ""
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage");
}
continue;
}
//Extract pidUri
var hasPid = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.hasPID, true);
Uri pidUri = null;
try
{
if (hasPid != null && ((COLID.Graph.TripleStore.DataModels.Base.Entity)hasPid).Id != string.Empty)
{
pidUri = new Uri(((COLID.Graph.TripleStore.DataModels.Base.Entity)hasPid).Id);
}
}
catch
{
pidUri = null;
}
//Check SourceId
string srcId = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasSourceID, true);
if (srcId == null)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = "SourceId not found.",
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage");
}
continue;
}
//Check Entity Type
try
{
_validationService.CheckInstantiableEntityType(resource);
}
catch (System.Exception ex)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = ex.Message,
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
ResourcesCTO resourcesCTO = new ResourcesCTO();
bool pidUriExistsinTripleStore = false;
if (pidUri == null)
{
//Check Whether resource is already present in Neptune (using SourceId)
ISet <Uri> resourceInstanceGraphs = new HashSet <Uri>();
resourceInstanceGraphs.Add(resInstanceGraph);
resourceInstanceGraphs.Add(draftInstanceGraph);
pidUri = _resourceRepository.GetPidUriBySourceId(srcId, resourceInstanceGraphs);
}
else
{
try
{
resourcesCTO = _resourceService.GetResourcesByPidUri(pidUri);
pidUriExistsinTripleStore = true;
}
catch
{
pidUriExistsinTripleStore = false;
}
}
//if Pid Uri is null then Add
if (pidUri == null || (pidUri != null && pidUriExistsinTripleStore == false))
{
try
{
string newResourceId = CreateNewResourceId();
_logger.LogInformation("BackgroundService: About to Validate New resource: {msg}", msg.Body);
//Validate
var(validationResult, failed, validationFacade) =
await _resourcePreprocessService.ValidateAndPreProcessResource(newResourceId, resource, new ResourcesCTO(), ResourceCrudAction.Create);
_logger.LogInformation("BackgroundService: Validation Complete for: {srcId} having status {stat}", srcId, failed.ToString());
//Update pidUri in stateItem
if (failed == false && resource.StateItems[0].ContainsKey("pid_uri"))
{
resource.StateItems[0]["pid_uri"] = validationFacade.RequestResource.PidUri.ToString();
}
//Create result data
BulkUploadResult result = new BulkUploadResult
{
ActionDone = failed ? "Error" : "Validated",
ErrorMessage = failed ? "Validation Failed while Adding the resource." : string.Empty,
Results = validationResult.Results,
Triples = validationResult.Triples.Replace(ColidEntryLifecycleStatus.Draft, ColidEntryLifecycleStatus.Published),
InstanceGraph = resInstanceGraph.ToString(),
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = validationFacade.RequestResource.PidUri.ToString(),
ResourceId = newResourceId,
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
};
//Get distribution
if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.Distribution))
{
List <dynamic> distList = resource.Properties[Graph.Metadata.Constants.Resource.Distribution];
foreach (dynamic dist in distList)
{
string EndPointPidUri = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties[Graph.Metadata.Constants.Resource.hasPID][0].Id;
string EndPointUrl = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.DistributionEndpoints.HasNetworkAddress, true);
result.DistributionEndPoint.Add(EndPointPidUri, EndPointUrl);
}
}
//Collect Result
totalValidationResult.Add(result);
//if validation passed then Update Nginx Proxy info for the resource DynamoDB
if (!failed)
{
_proxyConfigService.AddUpdateNginxConfigRepository(resource);
}
}
catch (System.Exception ex)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = ex.Message,
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true)
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
}
else //Update Resource
{
try
{
//var resourcesCTO = _resourceService.GetResourcesByPidUri(pidUri); // Draft und Published resource getrennt behandeln.
var id = resourcesCTO.GetDraftOrPublishedVersion().Id; // Draft und Published resource getrennt behandeln.
//Update resource with PidUri
if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.hasPID))
{
((COLID.Graph.TripleStore.DataModels.Base.Entity)resource.Properties[Graph.Metadata.Constants.Resource.hasPID][0]).Id = pidUri.ToString();
}
_logger.LogInformation("BackgroundService: About to Validate Existing resource: {msg}", msg.Body);
var(validationResult, failed, validationFacade) =
await _resourcePreprocessService.ValidateAndPreProcessResource(id, resource, resourcesCTO, ResourceCrudAction.Publish, false, null);
_logger.LogInformation("BackgroundService: Validation Complete for: {srcId} having status {stat}", srcId, failed.ToString());
// The validation failed, if the results are cricital errors.
if (failed)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = "Validation Failed while updating the resource.",
Results = validationResult.Results,
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = validationFacade.RequestResource.PidUri.ToString(),
ResourceId = id,
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
if (resourcesCTO.HasPublished && (!_resourceService.ResourceHasChanged(resourcesCTO.Published, validationFacade.RequestResource)))
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = "No changes found in this resource.",
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = validationFacade.RequestResource.PidUri.ToString(),
ResourceId = id,
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
var resourcetoCreate = _resourceService.SetHasLaterVersionResourceId(validationFacade.RequestResource);
using (var transaction = _resourceRepository.CreateTransaction())
{
// try deleting draft version and all inbound edges are changed to the new entry.
_resourceRepository.DeleteDraft(validationFacade.RequestResource.PidUri,
new Uri(validationFacade.RequestResource.Id), draftInstanceGraph);
if (resourcesCTO.HasDraft)
{
_identifierService.DeleteAllUnpublishedIdentifiers(resourcesCTO.Draft);
}
if (resourcesCTO.HasPublished)
{
// Try to delete published and all inbound edges are changed to the new entry.
_resourceRepository.DeletePublished(validationFacade.RequestResource.PidUri,
new Uri(validationFacade.RequestResource.Id), resInstanceGraph);
}
//Add existing revision to the new resource
if (resourcesCTO.Published != null)
{
var existingRevisions = resourcesCTO.Published.Properties.TryGetValue(COLID.Graph.Metadata.Constants.Resource.HasRevision, out List <dynamic> revisionValues) ? revisionValues : null;
if (existingRevisions != null)
{
resourcetoCreate.Properties.Add(COLID.Graph.Metadata.Constants.Resource.HasRevision, existingRevisions);
}
}
//Add Published
_resourceRepository.Create(resourcetoCreate, validationFacade.MetadataProperties, resInstanceGraph);
//Update revision
if (resourcesCTO.Published == null)
{
await _revisionService.InitializeResourceInAdditionalsGraph(resourcetoCreate, validationFacade.MetadataProperties);
}
else
{
Graph.Metadata.DataModels.Resources.Resource updatedResource = await _revisionService.AddAdditionalsAndRemovals(resourcesCTO.Published, validationFacade.RequestResource);
}
_logger.LogInformation("BackgroundService: Commit - {sparqlQuery}", transaction.GetSparqlString());
transaction.Commit();
//Index resource
//_indexingService.IndexPublishedResource(pidUri, updatedResource, validationFacade.ResourcesCTO);
}
_logger.LogInformation("BackgroundService: Resource updated having sourceId {sourceId} ", srcId);
//Create result data
BulkUploadResult result = new BulkUploadResult
{
ActionDone = "Updated",
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
ResourceId = id,
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
};
//Get distribution
if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.Distribution))
{
List <dynamic> distList = resource.Properties[Graph.Metadata.Constants.Resource.Distribution];
foreach (dynamic dist in distList)
{
string EndPointPidUri = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties[Graph.Metadata.Constants.Resource.hasPID][0].Id;
string EndPointUrl = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.DistributionEndpoints.HasNetworkAddress, true);
result.DistributionEndPoint.Add(EndPointPidUri, EndPointUrl);
}
}
//Collect Result
totalValidationResult.Add(result);
//Update Nginx Proxy info for the resource DynamoDB
_proxyConfigService.AddUpdateNginxConfigRepository(resource);
}
catch (System.Exception ex)
{
_logger.LogInformation("BackgroundService: Error while updating - {msg} ", ex.Message);
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = ex.Message,
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
}
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
}
//Send msg to output queue
try
{
await _awsSQSHelper.SendResourceMessageAsync(totalValidationResult);
}
catch (System.Exception ex)
{
_logger.LogInformation("BackgroundService: Could not send meessage to output queue {msg} ", ex.Message);
}
}
} while (msgcount > 0);
}
catch (System.Exception ex)
{
_logger.LogError("BackgroundService: " + ex.InnerException == null ? ex.Message : ex.InnerException.Message);
}
stpWatch.Stop();
if (totalMsgCount > 0)
{
//_logger.LogInformation("BackgroundService: Processed {totMsgCount} messages in {milsec} Milliseconds.", totalMsgCount, stpWatch.ElapsedMilliseconds);
}
}
