public void IndexLinkedResource(Uri pidUri, Entity resource, ResourcesCTO repoResources)
{
// TODO: Repos resources
var resourceIndex = new ResourceIndexingDTO(ResourceCrudAction.Linking, pidUri, resource, repoResources);
IndexResource(resourceIndex);
}
public async void CreateResource_Success_WithPreviousVersion()
{
// Arrange
var requestBuilder = new ResourceBuilder()
.GenerateSampleData();
var request = requestBuilder.BuildRequestDto();
var resource = requestBuilder.Build();
request.HasPreviousVersion = Graph.Metadata.Constants.Entity.IdPrefix + Guid.NewGuid();
var resourceCto = new ResourcesCTO(null, null);
var validationFacade = new EntityValidationFacade(ResourceCrudAction.Create, resource, resourceCto, request.HasPreviousVersion, _metadata, null);
var preProcessServiceResult = Task.FromResult(new Tuple <ValidationResult, bool, EntityValidationFacade>(null, false, validationFacade));
_mockPreProcessService.Setup(t => t.ValidateAndPreProcessResource(It.IsAny <string>(), request, It.IsAny <ResourcesCTO>(), ResourceCrudAction.Create, It.IsAny <bool>(), It.IsAny <string>())).Returns(preProcessServiceResult);
// Act
var result = await _service.CreateResource(request);
// Assert
Assert.NotNull(result);
_mockResourceRepo.Verify(t => t.Create(validationFacade.RequestResource, It.IsAny <IList <MetadataProperty> >()), Times.Once);
_mockResourceLinkingService.Verify(t => t.LinkResourceIntoList(validationFacade.RequestResource.PidUri, It.IsAny <Uri>()), Times.Once);
}
public void IndexUnlinkedResource(Uri pidUri, ResourcesCTO resource, Uri unlinkedPidUri, ResourcesCTO unlinkedListResource)
{
var resourceIndex = new ResourceIndexingDTO(ResourceCrudAction.Unlinking, pidUri, resource.GetDraftOrPublishedVersion(), resource);
IndexResource(resourceIndex);
IndexLinkedResource(unlinkedPidUri, unlinkedListResource.GetDraftOrPublishedVersion(), unlinkedListResource);
}
public void IndexNewResource(Uri pidUri, Resource resource, IList <VersionOverviewCTO> resourceVersions)
{
var repoResource = new ResourcesCTO(null, null, resourceVersions);
var resourceIndex = new ResourceIndexingDTO(ResourceCrudAction.Create, pidUri, resource, repoResource);
IndexResource(resourceIndex);
}
/// <summary>
/// There is an id for the endpoint, but a new id must be assigned to the endpoint if a draft is created from a published entry.
/// However, the process must be performed as an update to ensure that certain properties such as identifiers are inherited from the published entry.
/// This does not apply to attachments.
/// </summary>
/// <param name="key">The property key of the sub entity.</param>
/// <param name="subEntity">The sub entity.</param>
/// <param name="mainResource">The resource, where the sub entity belongs to.</param>
/// <returns>The entity id of the nested entity. Empty, if the subproperty should be saved as a copy with a new id.</returns>
private string GetNestedEntityId(string key, Entity subEntity, ResourcesCTO mainResource)
{
/* if(key != Graph.Metadata.Constants.AttachmentConstants.HasAttachment && mainResource.HasPublishedAndNoDraft)
* {
* return string.Empty;
* }*/
return(subEntity.Id);
}
public async void PublishResource_Should_ThrowError_Exception_AlreadyPublished()
{
// Arrange
var resource = new ResourceBuilder().GenerateSampleData().Build();
var resourceCto = new ResourcesCTO(null, resource);
_mockResourceRepo.Setup(s => s.GetResourcesByPidUri(resource.PidUri, It.IsAny <IList <string> >())).Returns(resourceCto);
// Act
await Assert.ThrowsAsync <BusinessException>(() => _service.PublishResource(resource.PidUri));
}
public EntityValidationFacade(ResourceCrudAction resourceCrucAction,
Resource requestResource,
ResourcesCTO resourcesCTO,
string previousVersion,
IList <MetadataProperty> metadataProperties,
string consumerGroup)
{
ResourceCrudAction = resourceCrucAction;
RequestResource = requestResource;
ResourcesCTO = resourcesCTO;
PreviousVersion = previousVersion;
MetadataProperties = metadataProperties;
ConsumerGroup = consumerGroup;
ValidationResults = new List <ValidationResultProperty>();
}
public async void EditResource_Success()
{
// Arrange
var requestBuilder = new ResourceBuilder()
.GenerateSampleData();
var request = requestBuilder.BuildRequestDto();
var resource = requestBuilder.Build();
var publishedId = Graph.Metadata.Constants.Entity.IdPrefix + Guid.NewGuid();
resource.Id = publishedId;
var resourceCto = new ResourcesCTO(null, resource);
_mockResourceRepo.Setup(s => s.GetResourcesByPidUri(resource.PidUri, It.IsAny <IList <string> >())).Returns(resourceCto);
var validationFacade = new EntityValidationFacade(ResourceCrudAction.Create, resource, resourceCto, request.HasPreviousVersion, _metadata, null);
var validationResult = new ValidationResult();
var preProcessServiceResult = new Tuple <ValidationResult, bool, EntityValidationFacade>(validationResult, false, validationFacade);
var newResourceId = string.Empty;
_mockPreProcessService.Setup(t => t.ValidateAndPreProcessResource(It.IsAny <string>(), request, It.IsAny <ResourcesCTO>(), ResourceCrudAction.Update, It.IsAny <bool>(), It.IsAny <string>()))
.ReturnsAsync(preProcessServiceResult)
.Callback <string, ResourceRequestDTO, ResourcesCTO, ResourceCrudAction, bool, string>((a, b, c, d, e, f) =>
{
newResourceId = a;
validationResult.Results.Add(new ValidationResultProperty(a, "some path", "some value", "some message", ValidationResultSeverity.Warning));
});
// Act
var result = await _service.EditResource(resource.PidUri, request);
// Assert
Assert.All(result.ValidationResult.Results, t =>
{
Assert.Equal(t.Node, newResourceId);
});
_mockResourceRepo.Verify(s => s.DeleteDraft(result.Resource.PidUri, new Uri(result.Resource.Id)), Times.Once);
_mockResourceRepo.Verify(s => s.Create(resource, _metadata), Times.Once);
_mockResourceRepo.Verify(s => s.CreateLinkingProperty(resource.PidUri, new Uri(
Graph.Metadata.Constants.Resource.HasPidEntryDraft),
Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.Published,
Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.Draft), Times.Once);
_mockIdentifierService.Verify(s => s.DeleteAllUnpublishedIdentifiers(It.IsAny <Entity>()), Times.Never);
}
public void HandleOverwriteProperties_Success()
{
// Arrange
var propertyKey = Graph.Metadata.Constants.RDF.Type;
var requestResource = CreateResourceWithType(Graph.Metadata.Constants.Resource.Type.GenericDataset);
var repoResource = CreateResourceWithType(Graph.Metadata.Constants.Resource.Type.Ontology);
var resourcesCTO = new ResourcesCTO(repoResource, repoResource, new List <VersionOverviewCTO>());
var entityValidationFacade = new EntityValidationFacade(ResourceCrudAction.Update, requestResource, resourcesCTO, string.Empty, _metadata, string.Empty);
// Act
_validator.Validate(propertyKey, entityValidationFacade);
// Assert
var resourceType = entityValidationFacade.RequestResource.Properties[Graph.Metadata.Constants.RDF.Type];
Assert.All(resourceType, t => Assert.Equal(Graph.Metadata.Constants.Resource.Type.Ontology, t));
}
public void InternalHasValidationResult_Success_ReplaceAuthorWithEmailOnUpdate()
{
// Arrange
_mockUserInfoService.Setup(mock => mock.GetRoles()).Returns(new List <string>());
Resource newResource = CreateResourceWithAuthor("*****@*****.**");
Resource repoResource = CreateResourceWithAuthor("*****@*****.**");
ResourcesCTO repoResourceCTO = new ResourcesCTO()
{
Draft = repoResource, Published = repoResource
};
EntityValidationFacade validationFacade = new EntityValidationFacade(ResourceCrudAction.Update, newResource, repoResourceCTO, null, _metadata, null);
// Act
_validator.HasValidationResult(validationFacade, GetAuthorProperty(newResource));
// Assert
AssertValidationFacade(validationFacade, "*****@*****.**");
}
public void InternalHasValidationResult_OverwritePropertyWithRepoDateTime()
{
// Arrange
var repoDate = DateTime.UtcNow.AddDays(-1);
var repoResource = CreateResource(repoDate);
var resourceCto = new ResourcesCTO(repoResource, null);
var resource = CreateResource(DateTime.UtcNow);
EntityValidationFacade validationFacade = new EntityValidationFacade(ResourceCrudAction.Update, resource, resourceCto, null, _metadata, null);
// Act
_validator.HasValidationResult(validationFacade, GetDateTimeProperty(resource));
// Assert
Assert.Contains(Graph.Metadata.Constants.Resource.DateCreated, validationFacade.RequestResource.Properties);
var dateCreatedList = GetDateTimeProperty(validationFacade.RequestResource).Value;
Assert.All(dateCreatedList, t =>
{
Assert.Equal(repoDate.ToString("o"), t);
});
}
public async void EditResource__Should_ThrowError_ResourceValidation()
{
// Arrange
var requestBuilder = new ResourceBuilder()
.GenerateSampleData();
var request = requestBuilder.BuildRequestDto();
var resource = requestBuilder.Build();
var publishedId = Graph.Metadata.Constants.Entity.IdPrefix + Guid.NewGuid();
resource.Id = publishedId;
var resourceCto = new ResourcesCTO(null, resource);
_mockResourceRepo.Setup(s => s.GetResourcesByPidUri(resource.PidUri, It.IsAny <IList <string> >())).Returns(resourceCto);
var validationFacade = new EntityValidationFacade(ResourceCrudAction.Create, resource, resourceCto, request.HasPreviousVersion, _metadata, null);
var validationResult = new ValidationResult();
var preProcessServiceResult = new Tuple <ValidationResult, bool, EntityValidationFacade>(validationResult, true, validationFacade);
var newResourceId = string.Empty;
_mockPreProcessService.Setup(t => t.ValidateAndPreProcessResource(It.IsAny <string>(), request, It.IsAny <ResourcesCTO>(), ResourceCrudAction.Update, It.IsAny <bool>(), It.IsAny <string>()))
.ReturnsAsync(preProcessServiceResult)
.Callback <string, ResourceRequestDTO, ResourcesCTO, ResourceCrudAction, bool, string>((a, b, c, d, e, f) =>
{
newResourceId = a;
validationResult.Results.Add(new ValidationResultProperty(a, "some path", "some value", "some message", ValidationResultSeverity.Warning));
});
// Act
var result = await Assert.ThrowsAsync <ResourceValidationException>(() => _service.EditResource(resource.PidUri, request));
// Assert
_mockResourceRepo.Verify(s => s.CreateTransaction(), Times.Never);
Assert.All(result.ValidationResult.Results, t => Assert.Equal(t.Node, publishedId));
}
public void InternalHasValidationResult_OverwritePropertyWithNewDateTime()
{
// Arrange
var resourceDateCreated = DateTime.UtcNow.AddDays(-1);
var resource = CreateResource(resourceDateCreated);
var resourceCto = new ResourcesCTO(resource, null, new List <VersionOverviewCTO>());
var new_resource = CreateResource(DateTime.UtcNow);
EntityValidationFacade validationFacade = new EntityValidationFacade(ResourceCrudAction.Update, new_resource, resourceCto, null, _metadata, null);
// Act
_validator.HasValidationResult(validationFacade, GetDateTimeProperty(new_resource));
// Assert
Assert.Contains(Graph.Metadata.Constants.Resource.DateCreated, validationFacade.RequestResource.Properties);
var dateCreatedList = GetDateTimeProperty(validationFacade.RequestResource).Value;
Assert.All(dateCreatedList, t =>
{
DateTime dateCreated = Convert.ToDateTime(t);
Assert.Equal(1, dateCreated.CompareTo(resourceDateCreated));
});
}
public ResourcesCTO GetResourcesByPidUri(Uri pidUri, IList <string> resourceTypes)
{
SparqlParameterizedString parameterizedString = new SparqlParameterizedString
{
CommandText =
@"SELECT DISTINCT ?subject ?object ?predicate ?object_ ?publishedVersion ?objectPidUri ?inbound ?inboundPredicate ?inboundPidUri
@fromResourceNamedGraph
WHERE { {
?subject @hasPid @pidUri.
BIND(?subject as ?object).
?subject ?predicate ?object_.
OPTIONAL { ?publishedVersion @hasPidEntryDraft ?subject }
OPTIONAL { ?object_ @hasPid ?objectPidUri. }
} UNION {
?subject @hasPid @pidUri.
?subject (rdf:| !rdf:)+ ?object.
?object rdf:type ?objectType.
FILTER (?objectType NOT IN ( @resourceTypes ) )
?object ?predicate ?object_.
}
UNION {
?subject @hasPid @pidUri.
?object ?inboundPredicate ?subject.
?object @hasPid ?inboundPidUri.
BIND(@true as ?inbound).
Filter NOT EXISTS { ?draftResource @hasPidEntryDraft ?object}
}
}"
};
parameterizedString.SetPlainLiteral("fromResourceNamedGraph", _metadataGraphConfigurationRepository.GetGraphs(QueryGraphs).JoinAsFromNamedGraphs());
parameterizedString.SetUri("hasPid", new Uri(Graph.Metadata.Constants.EnterpriseCore.PidUri));
parameterizedString.SetUri("pidUri", pidUri);
parameterizedString.SetUri("hasPidEntryDraft", new Uri(Graph.Metadata.Constants.Resource.HasPidEntryDraft));
parameterizedString.SetPlainLiteral("resourceTypes", resourceTypes.JoinAsGraphsList());
parameterizedString.SetLiteral("true", Graph.Metadata.Constants.Boolean.True);
var resources = BuildResourceFromQuery(parameterizedString, pidUri);
var resourceDraft = resources.FirstOrDefault(r =>
r.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasEntryLifecycleStatus, true) == Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.Draft); // firstResource
var resourcePublished = resources.FirstOrDefault(r =>
{
var lifecycleStatus =
r.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasEntryLifecycleStatus, true);
return(lifecycleStatus == Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.Published ||
lifecycleStatus == Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.MarkedForDeletion);
});
IList <VersionOverviewCTO> versions = new List <VersionOverviewCTO>();
if (resourceDraft != null || resourcePublished != null)
{
versions = resourceDraft == null ? resourcePublished.Versions : resourceDraft.Versions;
}
ResourcesCTO resourcesCTO = new ResourcesCTO(resourceDraft, resourcePublished, versions);
return(resourcesCTO);
}
/// <summary>
/// Validates all linked entities that are not of type permanent identifier.
/// </summary>
/// <param name="property"></param>
/// <param name="validationFacade"></param>
/// <returns></returns>
private async Task ValidateEndpoint(KeyValuePair <string, List <dynamic> > property, EntityValidationFacade validationFacade)
{
var key = property.Key;
// TODO: Check for types permanent identifier
if (key != Graph.Metadata.Constants.EnterpriseCore.PidUri && key != Graph.Metadata.Constants.Resource.BaseUri)
{
var newPropertyValue = new List <dynamic>();
var tasks = await Task.WhenAll(validationFacade.RequestResource.Properties[key].Select(async propertyValue =>
{
if (!DynamicExtension.IsType <Entity>(propertyValue, out Entity parsedValue))
{
return(propertyValue);
}
var subEntityCrudAction = ResourceCrudAction.Create;
Entity repoEntity = null;
// Only if something is updating, we can check if there is a related distribution endpoint
if (validationFacade.ResourceCrudAction != ResourceCrudAction.Create)
{
var repoResource = validationFacade.ResourcesCTO.GetDraftOrPublishedVersion();
// Try to get the distribution endpoint
foreach (var repoProperty in repoResource.Properties)
{
foreach (var repoPropertyValue in repoProperty.Value)
{
if (!DynamicExtension.IsType <Entity>(repoPropertyValue, out Entity parsedRepoValue) ||
parsedRepoValue.Id != parsedValue.Id)
{
continue;
}
repoEntity = parsedRepoValue;
subEntityCrudAction = ResourceCrudAction.Update;
}
}
}
var entityRequest = new ResourceRequestDTO()
{
Properties = parsedValue.Properties
};
var entitiesCTO = new ResourcesCTO()
{
Draft = repoEntity, Published = repoEntity
};
var entityId = GetNestedEntityId(key, parsedValue, validationFacade.ResourcesCTO);
// The consumer group of the parent must be included in the process.
Tuple <ValidationResult, bool, EntityValidationFacade> subResult = await ValidateAndPreProcessResource(entityId, entityRequest, entitiesCTO, subEntityCrudAction, true, validationFacade.ConsumerGroup);
// Add validationResults to resource results
validationFacade.ValidationResults.AddRange(subResult.Item1.Results);
return(subResult.Item3.RequestResource);
})).ConfigureAwait(true);
validationFacade.RequestResource.Properties[key] = tasks.ToList();
}
}
public void IndexUpdatedResource(Uri pidUri, Entity resource, ResourcesCTO repoResources)
{
var resourceIndex = new ResourceIndexingDTO(ResourceCrudAction.Update, pidUri, resource, repoResources);
IndexResource(resourceIndex);
}
public async Task <Tuple <ValidationResult, bool, EntityValidationFacade> > ValidateAndPreProcessResource(string resourceId, ResourceRequestDTO resourceRequestDTO,
ResourcesCTO resourcesCTO, ResourceCrudAction resourceCrudAction, bool nestedValidation = false, string consumerGroup = null, bool changeResourceType = false)
{
var requestResource = _mapper.Map <Resource>(resourceRequestDTO);
requestResource.Id = string.IsNullOrWhiteSpace(resourceId) ? CreateNewResourceId() : resourceId;
string entityType = requestResource.Properties.GetValueOrNull(Graph.Metadata.Constants.RDF.Type, true).ToString();
var metadata = _metadataService.GetMetadataForEntityType(entityType);
// If it is a nested validation (example distribution endpoint), the consumer group of the parent must be included in the process.
var actualConsumerGroup = nestedValidation ? consumerGroup : requestResource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasConsumerGroup, true);
var validationFacade = new EntityValidationFacade(resourceCrudAction, requestResource, resourcesCTO, resourceRequestDTO.HasPreviousVersion, metadata, actualConsumerGroup);
if (changeResourceType)
{
// dirty solution for changing resource type. could be refactored in the future
var validationRes = await _validationService.ValidateEntity(requestResource, metadata).ConfigureAwait(true);
if (validationRes.Results.Count == 1 && validationRes.Results[0].Path == Graph.Metadata.Constants.Resource.BaseUri)
{
validationFacade.ResourceCrudAction = ResourceCrudAction.Create;
}
}
// Remove passed properties for several properties and replace it with repo-resource properties afterwards
RemoveProperty(Graph.Metadata.Constants.Resource.HasLaterVersion, requestResource);
//RemoveProperty(Graph.Metadata.Constants.Resource.HasHistoricVersion, requestResource);
RemoveProperty(Graph.Metadata.Constants.Resource.ChangeRequester, requestResource);
if (resourceCrudAction != ResourceCrudAction.Create)
{
UpdatePropertyFromRepositoryResource(Graph.Metadata.Constants.Resource.HasLaterVersion, validationFacade);
//validationFacade.RequestResource.Properties.AddOrUpdate(Graph.Metadata.Constants.Resource.MetadataGraphConfiguration, new List<dynamic>() { _metadataConfigService.GetLatestConfiguration().Id });
}
if (!nestedValidation) // so only new resources get this property, no distribution endpoints
{
RemoveProperty(Graph.Metadata.Constants.Resource.MetadataGraphConfiguration, requestResource);
validationFacade.RequestResource.Properties.AddOrUpdate(Graph.Metadata.Constants.Resource.MetadataGraphConfiguration, new List <dynamic>()
{
_metadataConfigService.GetLatestConfiguration().Id
});
}
// Each property have to be valiated and in same cases transformed
var keys = requestResource.Properties.Keys.ToList();
foreach (var key in keys)
{
var property = new KeyValuePair <string, List <dynamic> >(key, requestResource.Properties[key]);
if (changeResourceType && (property.Key == Graph.Metadata.Constants.RDF.Type))
{
continue;
}
_entityPropertyValidator.Validate(key, validationFacade);
await ValidateEndpoint(property, validationFacade);
}
// The following processes may only be executed for the main entry, so that the function already ends here with nested validations.
if (nestedValidation)
{
var nestedValidationResult = new ValidationResult()
{
Results = validationFacade.ValidationResults
};
var failedValidation = !nestedValidationResult.Conforms &&
nestedValidationResult.Severity != ValidationResultSeverity.Info;
return(new Tuple <ValidationResult, bool, EntityValidationFacade>(nestedValidationResult, failedValidation, validationFacade));
}
var validationResult = await _validationService.ValidateEntity(requestResource, metadata).ConfigureAwait(true);
validationResult.Results = validationResult.Results.Select(r =>
{
r.ResultSeverity = IsWarningSeverity(r, resourceCrudAction) ? ValidationResultSeverity.Warning : r.ResultSeverity;
return(r);
}).ToList();
string validationResourceId = validationFacade.ResourceCrudAction == ResourceCrudAction.Create ? null : resourcesCTO.GetDraftOrPublishedVersion().Id;
var duplicateResults = _identifierValidationService.CheckDuplicates(requestResource, validationResourceId, resourceRequestDTO.HasPreviousVersion);
if (changeResourceType)
{
duplicateResults = duplicateResults.ToList().FindAll(r => r.Path != Graph.Metadata.Constants.Resource.hasPID);
}
// Check whether forbidden properties are contained in the entity.
var forbiddenPropertiesResults = _validationService.CheckForbiddenProperties(requestResource);
// TODO: Concat or AddRange check
validationResult.Results = validationResult.Results.Concat(validationFacade.ValidationResults).Concat(duplicateResults).Concat(forbiddenPropertiesResults).OrderBy(t => t.ResultSeverity).ToList();
var failed = ProcessFailed(validationResult, resourceCrudAction);
// dirty solution for changing resource type (see also above)
validationFacade.ResourceCrudAction = changeResourceType ? ResourceCrudAction.Update : validationFacade.ResourceCrudAction;
if (failed)
{
// Reset the lifecycle Status to the correct value
if (resourceCrudAction == ResourceCrudAction.Update)
{
requestResource.Properties.AddOrUpdate(Graph.Metadata.Constants.Resource.HasEntryLifecycleStatus, new List <dynamic>()
{
resourcesCTO.HasDraft?Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.Draft: Graph.Metadata.Constants.Resource.ColidEntryLifecycleStatus.Published
});
}
}
else
{
if (resourceCrudAction == ResourceCrudAction.Update && resourcesCTO.HasPublished)
{
requestResource.PublishedVersion = resourcesCTO.Published.Id;
}
}
return(new Tuple <ValidationResult, bool, EntityValidationFacade>(validationResult, failed, validationFacade));
}
public void IndexMarkedForDeletionResource(Uri pidUri, Entity resource, ResourcesCTO repoResources)
{
var resourceIndex = new ResourceIndexingDTO(ResourceCrudAction.MarkedForDeletion, pidUri, resource, repoResources);
IndexResource(resourceIndex);
}
/// <summary>
/// Read resources from SQS que and Validate
/// </summary>
/// <returns></returns>
private async Task AddUpdateResources()
{
_logger.LogInformation("BackgroundService: Running.... ");
Stopwatch stpWatch = new Stopwatch();
stpWatch.Start();
//Check for msgs in a loop
int msgcount, totalMsgCount = 0;
try
{
do
{
//Check msgs available in SQS
var msgs = await _awsSQSHelper.ReceiveResourceMessageAsync();
msgcount = msgs.Count;
totalMsgCount += msgs.Count;
//get Instance graphs if there is msg to process
Uri resInstanceGraph = null;
Uri draftInstanceGraph = null;
if (msgs.Count > 0)
{
_logger.LogInformation("BackgroundService: Found {count} messages from resource queue ", msgs.Count);
if (resInstanceGraph == null)
{
resInstanceGraph = _metadataService.GetInstanceGraph(PIDO.PidConcept);
}
if (draftInstanceGraph == null)
{
draftInstanceGraph = _metadataService.GetInstanceGraph("draft");
}
//List to collect ValidationFacade of each resource
List <BulkUploadResult> totalValidationResult = new List <BulkUploadResult>();
//Iterate on each msg which will contain list of resource
foreach (var msg in msgs)
{
ResourceRequestDTO resource;
// Try Get reources from the msg
try
{
resource = JsonConvert.DeserializeObject <ResourceRequestDTO>(msg.Body);
}
catch (System.Exception ex)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = "Unable to Deserialize the resource",
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = "",
ResourceLabel = ex.Message,
ResourceDefinition = ""
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage");
}
continue;
}
//Extract pidUri
var hasPid = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.hasPID, true);
Uri pidUri = null;
try
{
if (hasPid != null && ((COLID.Graph.TripleStore.DataModels.Base.Entity)hasPid).Id != string.Empty)
{
pidUri = new Uri(((COLID.Graph.TripleStore.DataModels.Base.Entity)hasPid).Id);
}
}
catch
{
pidUri = null;
}
//Check SourceId
string srcId = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasSourceID, true);
if (srcId == null)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = "SourceId not found.",
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage");
}
continue;
}
//Check Entity Type
try
{
_validationService.CheckInstantiableEntityType(resource);
}
catch (System.Exception ex)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = ex.Message,
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
ResourcesCTO resourcesCTO = new ResourcesCTO();
bool pidUriExistsinTripleStore = false;
if (pidUri == null)
{
//Check Whether resource is already present in Neptune (using SourceId)
ISet <Uri> resourceInstanceGraphs = new HashSet <Uri>();
resourceInstanceGraphs.Add(resInstanceGraph);
resourceInstanceGraphs.Add(draftInstanceGraph);
pidUri = _resourceRepository.GetPidUriBySourceId(srcId, resourceInstanceGraphs);
}
else
{
try
{
resourcesCTO = _resourceService.GetResourcesByPidUri(pidUri);
pidUriExistsinTripleStore = true;
}
catch
{
pidUriExistsinTripleStore = false;
}
}
//if Pid Uri is null then Add
if (pidUri == null || (pidUri != null && pidUriExistsinTripleStore == false))
{
try
{
string newResourceId = CreateNewResourceId();
_logger.LogInformation("BackgroundService: About to Validate New resource: {msg}", msg.Body);
//Validate
var(validationResult, failed, validationFacade) =
await _resourcePreprocessService.ValidateAndPreProcessResource(newResourceId, resource, new ResourcesCTO(), ResourceCrudAction.Create);
_logger.LogInformation("BackgroundService: Validation Complete for: {srcId} having status {stat}", srcId, failed.ToString());
//Update pidUri in stateItem
if (failed == false && resource.StateItems[0].ContainsKey("pid_uri"))
{
resource.StateItems[0]["pid_uri"] = validationFacade.RequestResource.PidUri.ToString();
}
//Create result data
BulkUploadResult result = new BulkUploadResult
{
ActionDone = failed ? "Error" : "Validated",
ErrorMessage = failed ? "Validation Failed while Adding the resource." : string.Empty,
Results = validationResult.Results,
Triples = validationResult.Triples.Replace(ColidEntryLifecycleStatus.Draft, ColidEntryLifecycleStatus.Published),
InstanceGraph = resInstanceGraph.ToString(),
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = validationFacade.RequestResource.PidUri.ToString(),
ResourceId = newResourceId,
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
};
//Get distribution
if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.Distribution))
{
List <dynamic> distList = resource.Properties[Graph.Metadata.Constants.Resource.Distribution];
foreach (dynamic dist in distList)
{
string EndPointPidUri = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties[Graph.Metadata.Constants.Resource.hasPID][0].Id;
string EndPointUrl = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.DistributionEndpoints.HasNetworkAddress, true);
result.DistributionEndPoint.Add(EndPointPidUri, EndPointUrl);
}
}
//Collect Result
totalValidationResult.Add(result);
//if validation passed then Update Nginx Proxy info for the resource DynamoDB
if (!failed)
{
_proxyConfigService.AddUpdateNginxConfigRepository(resource);
}
}
catch (System.Exception ex)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = ex.Message,
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true)
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
}
else //Update Resource
{
try
{
//var resourcesCTO = _resourceService.GetResourcesByPidUri(pidUri); // Draft und Published resource getrennt behandeln.
var id = resourcesCTO.GetDraftOrPublishedVersion().Id; // Draft und Published resource getrennt behandeln.
//Update resource with PidUri
if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.hasPID))
{
((COLID.Graph.TripleStore.DataModels.Base.Entity)resource.Properties[Graph.Metadata.Constants.Resource.hasPID][0]).Id = pidUri.ToString();
}
_logger.LogInformation("BackgroundService: About to Validate Existing resource: {msg}", msg.Body);
var(validationResult, failed, validationFacade) =
await _resourcePreprocessService.ValidateAndPreProcessResource(id, resource, resourcesCTO, ResourceCrudAction.Publish, false, null);
_logger.LogInformation("BackgroundService: Validation Complete for: {srcId} having status {stat}", srcId, failed.ToString());
// The validation failed, if the results are cricital errors.
if (failed)
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = "Validation Failed while updating the resource.",
Results = validationResult.Results,
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = validationFacade.RequestResource.PidUri.ToString(),
ResourceId = id,
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
if (resourcesCTO.HasPublished && (!_resourceService.ResourceHasChanged(resourcesCTO.Published, validationFacade.RequestResource)))
{
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = "No changes found in this resource.",
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = validationFacade.RequestResource.PidUri.ToString(),
ResourceId = id,
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
var resourcetoCreate = _resourceService.SetHasLaterVersionResourceId(validationFacade.RequestResource);
using (var transaction = _resourceRepository.CreateTransaction())
{
// try deleting draft version and all inbound edges are changed to the new entry.
_resourceRepository.DeleteDraft(validationFacade.RequestResource.PidUri,
new Uri(validationFacade.RequestResource.Id), draftInstanceGraph);
if (resourcesCTO.HasDraft)
{
_identifierService.DeleteAllUnpublishedIdentifiers(resourcesCTO.Draft);
}
if (resourcesCTO.HasPublished)
{
// Try to delete published and all inbound edges are changed to the new entry.
_resourceRepository.DeletePublished(validationFacade.RequestResource.PidUri,
new Uri(validationFacade.RequestResource.Id), resInstanceGraph);
}
//Add existing revision to the new resource
if (resourcesCTO.Published != null)
{
var existingRevisions = resourcesCTO.Published.Properties.TryGetValue(COLID.Graph.Metadata.Constants.Resource.HasRevision, out List <dynamic> revisionValues) ? revisionValues : null;
if (existingRevisions != null)
{
resourcetoCreate.Properties.Add(COLID.Graph.Metadata.Constants.Resource.HasRevision, existingRevisions);
}
}
//Add Published
_resourceRepository.Create(resourcetoCreate, validationFacade.MetadataProperties, resInstanceGraph);
//Update revision
if (resourcesCTO.Published == null)
{
await _revisionService.InitializeResourceInAdditionalsGraph(resourcetoCreate, validationFacade.MetadataProperties);
}
else
{
Graph.Metadata.DataModels.Resources.Resource updatedResource = await _revisionService.AddAdditionalsAndRemovals(resourcesCTO.Published, validationFacade.RequestResource);
}
_logger.LogInformation("BackgroundService: Commit - {sparqlQuery}", transaction.GetSparqlString());
transaction.Commit();
//Index resource
//_indexingService.IndexPublishedResource(pidUri, updatedResource, validationFacade.ResourcesCTO);
}
_logger.LogInformation("BackgroundService: Resource updated having sourceId {sourceId} ", srcId);
//Create result data
BulkUploadResult result = new BulkUploadResult
{
ActionDone = "Updated",
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
ResourceId = id,
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
};
//Get distribution
if (resource.Properties.ContainsKey(Graph.Metadata.Constants.Resource.Distribution))
{
List <dynamic> distList = resource.Properties[Graph.Metadata.Constants.Resource.Distribution];
foreach (dynamic dist in distList)
{
string EndPointPidUri = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties[Graph.Metadata.Constants.Resource.hasPID][0].Id;
string EndPointUrl = ((COLID.Graph.TripleStore.DataModels.Base.EntityBase)dist).Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.DistributionEndpoints.HasNetworkAddress, true);
result.DistributionEndPoint.Add(EndPointPidUri, EndPointUrl);
}
}
//Collect Result
totalValidationResult.Add(result);
//Update Nginx Proxy info for the resource DynamoDB
_proxyConfigService.AddUpdateNginxConfigRepository(resource);
}
catch (System.Exception ex)
{
_logger.LogInformation("BackgroundService: Error while updating - {msg} ", ex.Message);
//Collect result
totalValidationResult.Add(new BulkUploadResult
{
ActionDone = "Error",
ErrorMessage = ex.Message,
TimeTaken = stpWatch.ElapsedMilliseconds.ToString(),
pidUri = pidUri == null ? "" : pidUri.ToString(),
SourceId = srcId,
ResourceLabel = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasLabel, true),
ResourceDefinition = resource.Properties.GetValueOrNull(Graph.Metadata.Constants.Resource.HasResourceDefintion, true),
StateItems = resource.StateItems
});
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
continue;
}
}
// Delete msg from input Queue
if (DeleteMessageFromSQS(msg.ReceiptHandle).Result == false)
{
_logger.LogInformation("BackgroundService: Could not delete meessage for sourceId {sourceId} ", srcId);
}
}
//Send msg to output queue
try
{
await _awsSQSHelper.SendResourceMessageAsync(totalValidationResult);
}
catch (System.Exception ex)
{
_logger.LogInformation("BackgroundService: Could not send meessage to output queue {msg} ", ex.Message);
}
}
} while (msgcount > 0);
}
catch (System.Exception ex)
{
_logger.LogError("BackgroundService: " + ex.InnerException == null ? ex.Message : ex.InnerException.Message);
}
stpWatch.Stop();
if (totalMsgCount > 0)
{
//_logger.LogInformation("BackgroundService: Processed {totMsgCount} messages in {milsec} Milliseconds.", totalMsgCount, stpWatch.ElapsedMilliseconds);
}
}
