public void SaveImportedRecord()
{
var userId = m_mockDataManager.GetOrCreateUser();
var externalRepositoryId = m_mockDataManager.CreateExternalRepository("", new List <FilteringExpressionContract>());
var historyId = m_mockDataManager.CreateImportHistory();
var info = new RepositoryImportProgressInfo(externalRepositoryId, "Test");
m_importedProjectManager.SaveImportedProject(m_importedRecord, userId, externalRepositoryId, m_mockDataManager.GetOrCreateBookType(),
new List <int>(), historyId, info);
Assert.AreEqual(false, info.IsCompleted);
Assert.AreEqual(0, info.FailedProjectsCount);
Assert.AreEqual(1, info.ProcessedProjectsCount);
Assert.AreEqual(null, info.FaultedMessage);
Assert.AreNotEqual(null, m_importedRecord.ProjectId);
var project = m_metadataRepository.GetAdditionalProjectMetadata(m_importedRecord.ProjectId, true, true, false, true, true, true,
false);
var snapshot = m_projectRepository.GetLatestSnapshot(m_importedRecord.ProjectId);
var importedRecordMetadata = m_importedRecordMetadataRepository.InvokeUnitOfWork(x => x.GetImportedRecordMetadataBySnapshot(snapshot.Id));
var importedProjectMetadata = m_importedProjectMetadataRepository.InvokeUnitOfWork(x => x.GetImportedProjectMetadata(m_importedRecord.ExternalId));
Assert.AreNotEqual(null, importedProjectMetadata.Id);
Assert.AreEqual(m_importedRecord.ExternalId, importedProjectMetadata.ExternalId);
Assert.AreEqual(project.Id, importedProjectMetadata.Project.Id);
Assert.AreEqual(historyId, importedRecordMetadata.LastUpdate.Id);
Assert.AreEqual(null, importedRecordMetadata.LastUpdateMessage);
Assert.AreEqual(importedProjectMetadata.Id, importedRecordMetadata.ImportedProjectMetadata.Id);
}
public virtual ActionBlock <ImportedRecord> BuildNullTargetBlock(RepositoryImportProgressInfo progressInfo,
ExecutionDataflowBlockOptions executionOptions)
{
return(new ActionBlock <ImportedRecord>(
importedRecord => { progressInfo.IncrementProcessedProjectsCount(); }, executionOptions
));
}
public ImportPipeline Build(ImportPipelineBuilder importPipelineBuilder, ExternalRepositoryDetailContract externalRepository,
int importHistoryId, RepositoryImportProgressInfo progressInfo, CancellationToken cancellationToken)
{
var executionOptions = new ExecutionDataflowBlockOptions {
CancellationToken = cancellationToken
};
var bufferBlock = new BufferBlock <object>(executionOptions);
var responseParserBlock =
importPipelineBuilder.BuildResponseParserBlock(externalRepository.ExternalRepositoryType.Name, executionOptions);
var filterBlock = importPipelineBuilder.BuildFilterBlock(externalRepository.Id, externalRepository.BibliographicFormat.Name,
executionOptions);
var projectParserBlock =
importPipelineBuilder.BuildProjectParserBlock(externalRepository.BibliographicFormat.Name, executionOptions);
var nullTargetBlock = importPipelineBuilder.BuildNullTargetBlock(progressInfo, executionOptions);
var saveBlock = importPipelineBuilder.BuildSaveBlock(m_importManager.UserId, importHistoryId, externalRepository.Id,
progressInfo, executionOptions);
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
bufferBlock.LinkTo(responseParserBlock, linkOptions);
responseParserBlock.LinkTo(filterBlock, linkOptions);
filterBlock.LinkTo(projectParserBlock, linkOptions, importedRecord => importedRecord.IsSuitable);
filterBlock.LinkTo(nullTargetBlock, linkOptions);
projectParserBlock.LinkTo(saveBlock, linkOptions);
return(new ImportPipeline {
BufferBlock = bufferBlock, LastBlock = saveBlock
});
}
public void SaveImportedProject(ImportedRecord importedRecord, int userId, int externalRepositoryId, int bookTypeId,
IList <int> roleIds, int importHistoryId, RepositoryImportProgressInfo progressInfo)
{
try
{
if (importedRecord.IsFailed)
{
progressInfo.IncrementFailedProjectsCount();
}
else
{
new SaveImportedDataWork(m_projectRepository, m_metadataRepository, m_catalogValueRepository,
m_personRepository, m_permissionRepository, importedRecord, userId, externalRepositoryId, bookTypeId,
roleIds).Execute();
}
}
catch (DataException e)
{
importedRecord.IsFailed = true;
importedRecord.FaultedMessage = e.Message;
progressInfo.IncrementFailedProjectsCount();
if (m_logger.IsErrorEnabled())
{
m_logger.LogError(e, e.Message);
}
}
try
{
progressInfo.IncrementProcessedProjectsCount();
m_importedRecordMetadataManager.CreateImportedRecordMetadata(importedRecord, importHistoryId);
}
catch (DataException e)
{
if (!importedRecord.IsFailed)
{
importedRecord.IsFailed = true;
importedRecord.FaultedMessage = e.Message;
progressInfo.IncrementFailedProjectsCount();
}
if (m_logger.IsErrorEnabled())
{
m_logger.LogError(e, e.Message);
}
}
}
public virtual ActionBlock <ImportedRecord> BuildSaveBlock(int userId, int importHistoryId, int externalRepositoryId,
RepositoryImportProgressInfo progressInfo, ExecutionDataflowBlockOptions executionOptions)
{
var bookTypeId = m_projectRepository.InvokeUnitOfWork(x => x.GetBookTypeByEnum(BookTypeEnum.BibliographicalItem)).Id;
var roleIds = m_permissionsProvider.GetRoleIdsByPermissionName($"{VokabularPermissionNames.AutoImport}{(int)BookTypeEnum.BibliographicalItem}") ??
new List <int>();
return(new ActionBlock <ImportedRecord>(importedRecord =>
{
using (var scope = m_serviceProvider.CreateScope())
{
var importedProjectManager = scope.ServiceProvider.GetRequiredService <ImportedProjectManager>();
importedProjectManager.SaveImportedProject(importedRecord, userId, externalRepositoryId, bookTypeId,
roleIds, importHistoryId, progressInfo);
}
},
executionOptions
));
}
public void SaveFailedImportedRecord()
{
var userId = m_mockDataManager.GetOrCreateUser();
var externalRepositoryId = m_mockDataManager.CreateExternalRepository("", new List <FilteringExpressionContract>());
var info = new RepositoryImportProgressInfo(externalRepositoryId, "Test");
m_importedProjectManager.SaveImportedProject(m_importedRecord, userId, externalRepositoryId, m_mockDataManager.GetOrCreateBookType(),
new List <int>(), m_mockDataManager.CreateImportHistory(), info);
m_importedRecord.IsNew = false;
m_importedRecord.IsFailed = true;
m_importedRecord.ImportedProject = null;
m_importedRecord.FaultedMessage = "Message";
var historyId = m_mockDataManager.CreateImportHistory();
m_importedProjectManager.SaveImportedProject(m_importedRecord, userId, externalRepositoryId, m_mockDataManager.GetOrCreateBookType(),
new List <int>(), historyId, info);
Assert.AreEqual(false, info.IsCompleted);
Assert.AreEqual(1, info.FailedProjectsCount);
Assert.AreEqual(2, info.ProcessedProjectsCount);
Assert.AreEqual(null, info.FaultedMessage);
var importedProjectMetadata = m_importedProjectMetadataRepository.InvokeUnitOfWork(x => x.GetImportedProjectMetadata(m_importedRecord.ExternalId));
Assert.AreNotEqual(null, importedProjectMetadata.Id);
Assert.AreEqual(m_importedRecord.ExternalId, importedProjectMetadata.ExternalId);
Assert.AreEqual(true, m_importedRecord.ImportedProjectMetadataId.HasValue);
var importedRecordMetadata = m_importedRecordMetadataRepository.InvokeUnitOfWork(x => x.GetImportedRecordMetadataByImportedProjectMetadata(m_importedRecord.ImportedProjectMetadataId.Value));
var metadata = importedRecordMetadata.FirstOrDefault(x => x.Snapshot == null);
Assert.AreNotEqual(null, metadata);
Assert.AreEqual(historyId, metadata.LastUpdate.Id);
Assert.AreEqual(m_importedRecord.FaultedMessage, metadata.LastUpdateMessage);
Assert.AreEqual(importedProjectMetadata.Id, metadata.ImportedProjectMetadata.Id);
}
public virtual async Task ImportFromResource(string configuration, ITargetBlock <object> buffer, RepositoryImportProgressInfo progressInfo,
DateTime?lastImport = null, CancellationToken cancellationToken = default(CancellationToken))
{
var oaiPmhResource = JsonConvert.DeserializeObject <OaiPmhRepositoryConfiguration>(configuration);
using (var client = m_oaiPmhCommunicationFactory.CreateClient(oaiPmhResource.Url))
{
var records = await client.GetRecordsListAsync(oaiPmhResource.DataFormat, oaiPmhResource.SetName, lastImport);
var resumptionToken = records.resumptionToken;
progressInfo.TotalProjectsCount = Convert.ToInt32(resumptionToken.completeListSize);
foreach (var recordType in records.record)
{
buffer.Post(recordType);
}
while (resumptionToken.Value != null && !cancellationToken.IsCancellationRequested)
{
records = await client.GetRecordsListWithRetryAsync(resumptionToken.Value);
resumptionToken = records.resumptionToken;
foreach (var recordType in records.record)
{
buffer.Post(recordType);
}
}
}
}
public async Task ImportAsync(int externalRepositoryId, CancellationToken cancellationToken)
{
var externalRepository = m_externalRepositoryManager.GetExternalRepository(externalRepositoryId);
var progressInfo = new RepositoryImportProgressInfo(externalRepositoryId, externalRepository.Name);
m_importManager.ActualProgress.TryAdd(externalRepositoryId, progressInfo);
var latestImportHistory = m_importHistoryManager.GetLatestSuccessfulImportHistory(externalRepositoryId);
var importHistoryId = m_importHistoryManager.CreateImportHistory(externalRepositoryId, m_importManager.UserId);
ImportPipeline importPipeline = null;
try
{
m_projectImportManagers.TryGetValue(externalRepository.ExternalRepositoryType.Name, out var importManager);
if (importManager == null)
{
throw new ImportFailedException(MainServiceErrorCode.RepositoryImportManagerNotFound,
$"The import manager was not found for repository type {externalRepository.ExternalRepositoryType.Name}.",
externalRepository.ExternalRepositoryType.Name);
}
importPipeline = m_importPipelineDirector.Build(m_importPipelineBuilder, externalRepository, importHistoryId, progressInfo, cancellationToken);
await importManager.ImportFromResource(externalRepository.Configuration, importPipeline.BufferBlock, progressInfo,
latestImportHistory?.Date, cancellationToken);
importPipeline.BufferBlock.Complete();
importPipeline.LastBlock.Completion.Wait(cancellationToken);
}
catch (OperationCanceledException)
{
progressInfo.FaultedMessage = MainServiceErrorCode.RepositoryImportCancelled;
progressInfo.FaultedMessageParams = new object[] { externalRepository.Name };
}
catch (AggregateException e)
{
var errorMessages = new StringBuilder();
var exceptions = e.InnerExceptions;
for (var i = 1; i <= exceptions.Count; i++)
{
errorMessages.AppendLine($"Error #{i}: {exceptions[i].Message}");
}
if (m_logger.IsErrorEnabled())
{
m_logger.LogError(e, errorMessages.ToString());
}
progressInfo.FaultedMessage = MainServiceErrorCode.RepositoryImportFailed;
progressInfo.FaultedMessageParams = new object[] { externalRepository.Name };
}
catch (ImportFailedException e)
{
if (m_logger.IsErrorEnabled())
{
m_logger.LogError(e, e.Message);
}
progressInfo.FaultedMessage = e.Code;
progressInfo.FaultedMessageParams = e.CodeParams;
}
catch (Exception e) // Catch every exception otherwise, ProjectImportBackgroundService will be stopped
{
if (m_logger.IsErrorEnabled())
{
m_logger.LogError(e, e.Message);
}
progressInfo.FaultedMessage = MainServiceErrorCode.RepositoryImportFailed;
progressInfo.FaultedMessageParams = new object[] { externalRepository.Name };
}
finally
{
var importHistory = m_importHistoryManager.GetImportHistory(importHistoryId);
m_importManager.CancellationTokens.TryGetValue(externalRepositoryId, out var cancellationTokenSource);
if (cancellationTokenSource != null && cancellationTokenSource.IsCancellationRequested)
{
progressInfo.FaultedMessage = MainServiceErrorCode.RepositoryImportCancelled;
progressInfo.FaultedMessageParams = new object[] { externalRepository.Name };
importHistory.Message = progressInfo.FaultedMessage;
importHistory.Status = ImportStatusEnum.Failed;
}
else if (!string.IsNullOrEmpty(progressInfo.FaultedMessage))
{
importHistory.Message = progressInfo.FaultedMessage;
importHistory.Status = ImportStatusEnum.Failed;
m_importManager.CancelTask(externalRepositoryId);
}
else if (progressInfo.FailedProjectsCount > 0)
{
importHistory.Status = ImportStatusEnum.CompletedWithWarnings;
}
else
{
importHistory.Status = ImportStatusEnum.Completed;
}
progressInfo.IsCompleted = true;
m_importHistoryManager.UpdateImportHistory(importHistory);
if (!string.IsNullOrEmpty(progressInfo.FaultedMessage))
{
importPipeline?.LastBlock.Completion.Wait(cancellationToken);
}
}
}
