public void SaveImportedRecord() { var userId = m_mockDataManager.GetOrCreateUser(); var externalRepositoryId = m_mockDataManager.CreateExternalRepository("", new List <FilteringExpressionContract>()); var historyId = m_mockDataManager.CreateImportHistory(); var info = new RepositoryImportProgressInfo(externalRepositoryId, "Test"); m_importedProjectManager.SaveImportedProject(m_importedRecord, userId, externalRepositoryId, m_mockDataManager.GetOrCreateBookType(), new List <int>(), historyId, info); Assert.AreEqual(false, info.IsCompleted); Assert.AreEqual(0, info.FailedProjectsCount); Assert.AreEqual(1, info.ProcessedProjectsCount); Assert.AreEqual(null, info.FaultedMessage); Assert.AreNotEqual(null, m_importedRecord.ProjectId); var project = m_metadataRepository.GetAdditionalProjectMetadata(m_importedRecord.ProjectId, true, true, false, true, true, true, false); var snapshot = m_projectRepository.GetLatestSnapshot(m_importedRecord.ProjectId); var importedRecordMetadata = m_importedRecordMetadataRepository.InvokeUnitOfWork(x => x.GetImportedRecordMetadataBySnapshot(snapshot.Id)); var importedProjectMetadata = m_importedProjectMetadataRepository.InvokeUnitOfWork(x => x.GetImportedProjectMetadata(m_importedRecord.ExternalId)); Assert.AreNotEqual(null, importedProjectMetadata.Id); Assert.AreEqual(m_importedRecord.ExternalId, importedProjectMetadata.ExternalId); Assert.AreEqual(project.Id, importedProjectMetadata.Project.Id); Assert.AreEqual(historyId, importedRecordMetadata.LastUpdate.Id); Assert.AreEqual(null, importedRecordMetadata.LastUpdateMessage); Assert.AreEqual(importedProjectMetadata.Id, importedRecordMetadata.ImportedProjectMetadata.Id); }
public virtual ActionBlock <ImportedRecord> BuildNullTargetBlock(RepositoryImportProgressInfo progressInfo, ExecutionDataflowBlockOptions executionOptions) { return(new ActionBlock <ImportedRecord>( importedRecord => { progressInfo.IncrementProcessedProjectsCount(); }, executionOptions )); }
public ImportPipeline Build(ImportPipelineBuilder importPipelineBuilder, ExternalRepositoryDetailContract externalRepository, int importHistoryId, RepositoryImportProgressInfo progressInfo, CancellationToken cancellationToken) { var executionOptions = new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken }; var bufferBlock = new BufferBlock <object>(executionOptions); var responseParserBlock = importPipelineBuilder.BuildResponseParserBlock(externalRepository.ExternalRepositoryType.Name, executionOptions); var filterBlock = importPipelineBuilder.BuildFilterBlock(externalRepository.Id, externalRepository.BibliographicFormat.Name, executionOptions); var projectParserBlock = importPipelineBuilder.BuildProjectParserBlock(externalRepository.BibliographicFormat.Name, executionOptions); var nullTargetBlock = importPipelineBuilder.BuildNullTargetBlock(progressInfo, executionOptions); var saveBlock = importPipelineBuilder.BuildSaveBlock(m_importManager.UserId, importHistoryId, externalRepository.Id, progressInfo, executionOptions); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; bufferBlock.LinkTo(responseParserBlock, linkOptions); responseParserBlock.LinkTo(filterBlock, linkOptions); filterBlock.LinkTo(projectParserBlock, linkOptions, importedRecord => importedRecord.IsSuitable); filterBlock.LinkTo(nullTargetBlock, linkOptions); projectParserBlock.LinkTo(saveBlock, linkOptions); return(new ImportPipeline { BufferBlock = bufferBlock, LastBlock = saveBlock }); }
public void SaveImportedProject(ImportedRecord importedRecord, int userId, int externalRepositoryId, int bookTypeId, IList <int> roleIds, int importHistoryId, RepositoryImportProgressInfo progressInfo) { try { if (importedRecord.IsFailed) { progressInfo.IncrementFailedProjectsCount(); } else { new SaveImportedDataWork(m_projectRepository, m_metadataRepository, m_catalogValueRepository, m_personRepository, m_permissionRepository, importedRecord, userId, externalRepositoryId, bookTypeId, roleIds).Execute(); } } catch (DataException e) { importedRecord.IsFailed = true; importedRecord.FaultedMessage = e.Message; progressInfo.IncrementFailedProjectsCount(); if (m_logger.IsErrorEnabled()) { m_logger.LogError(e, e.Message); } } try { progressInfo.IncrementProcessedProjectsCount(); m_importedRecordMetadataManager.CreateImportedRecordMetadata(importedRecord, importHistoryId); } catch (DataException e) { if (!importedRecord.IsFailed) { importedRecord.IsFailed = true; importedRecord.FaultedMessage = e.Message; progressInfo.IncrementFailedProjectsCount(); } if (m_logger.IsErrorEnabled()) { m_logger.LogError(e, e.Message); } } }
public virtual ActionBlock <ImportedRecord> BuildSaveBlock(int userId, int importHistoryId, int externalRepositoryId, RepositoryImportProgressInfo progressInfo, ExecutionDataflowBlockOptions executionOptions) { var bookTypeId = m_projectRepository.InvokeUnitOfWork(x => x.GetBookTypeByEnum(BookTypeEnum.BibliographicalItem)).Id; var roleIds = m_permissionsProvider.GetRoleIdsByPermissionName($"{VokabularPermissionNames.AutoImport}{(int)BookTypeEnum.BibliographicalItem}") ?? new List <int>(); return(new ActionBlock <ImportedRecord>(importedRecord => { using (var scope = m_serviceProvider.CreateScope()) { var importedProjectManager = scope.ServiceProvider.GetRequiredService <ImportedProjectManager>(); importedProjectManager.SaveImportedProject(importedRecord, userId, externalRepositoryId, bookTypeId, roleIds, importHistoryId, progressInfo); } }, executionOptions )); }
public void SaveFailedImportedRecord() { var userId = m_mockDataManager.GetOrCreateUser(); var externalRepositoryId = m_mockDataManager.CreateExternalRepository("", new List <FilteringExpressionContract>()); var info = new RepositoryImportProgressInfo(externalRepositoryId, "Test"); m_importedProjectManager.SaveImportedProject(m_importedRecord, userId, externalRepositoryId, m_mockDataManager.GetOrCreateBookType(), new List <int>(), m_mockDataManager.CreateImportHistory(), info); m_importedRecord.IsNew = false; m_importedRecord.IsFailed = true; m_importedRecord.ImportedProject = null; m_importedRecord.FaultedMessage = "Message"; var historyId = m_mockDataManager.CreateImportHistory(); m_importedProjectManager.SaveImportedProject(m_importedRecord, userId, externalRepositoryId, m_mockDataManager.GetOrCreateBookType(), new List <int>(), historyId, info); Assert.AreEqual(false, info.IsCompleted); Assert.AreEqual(1, info.FailedProjectsCount); Assert.AreEqual(2, info.ProcessedProjectsCount); Assert.AreEqual(null, info.FaultedMessage); var importedProjectMetadata = m_importedProjectMetadataRepository.InvokeUnitOfWork(x => x.GetImportedProjectMetadata(m_importedRecord.ExternalId)); Assert.AreNotEqual(null, importedProjectMetadata.Id); Assert.AreEqual(m_importedRecord.ExternalId, importedProjectMetadata.ExternalId); Assert.AreEqual(true, m_importedRecord.ImportedProjectMetadataId.HasValue); var importedRecordMetadata = m_importedRecordMetadataRepository.InvokeUnitOfWork(x => x.GetImportedRecordMetadataByImportedProjectMetadata(m_importedRecord.ImportedProjectMetadataId.Value)); var metadata = importedRecordMetadata.FirstOrDefault(x => x.Snapshot == null); Assert.AreNotEqual(null, metadata); Assert.AreEqual(historyId, metadata.LastUpdate.Id); Assert.AreEqual(m_importedRecord.FaultedMessage, metadata.LastUpdateMessage); Assert.AreEqual(importedProjectMetadata.Id, metadata.ImportedProjectMetadata.Id); }
public virtual async Task ImportFromResource(string configuration, ITargetBlock <object> buffer, RepositoryImportProgressInfo progressInfo, DateTime?lastImport = null, CancellationToken cancellationToken = default(CancellationToken)) { var oaiPmhResource = JsonConvert.DeserializeObject <OaiPmhRepositoryConfiguration>(configuration); using (var client = m_oaiPmhCommunicationFactory.CreateClient(oaiPmhResource.Url)) { var records = await client.GetRecordsListAsync(oaiPmhResource.DataFormat, oaiPmhResource.SetName, lastImport); var resumptionToken = records.resumptionToken; progressInfo.TotalProjectsCount = Convert.ToInt32(resumptionToken.completeListSize); foreach (var recordType in records.record) { buffer.Post(recordType); } while (resumptionToken.Value != null && !cancellationToken.IsCancellationRequested) { records = await client.GetRecordsListWithRetryAsync(resumptionToken.Value); resumptionToken = records.resumptionToken; foreach (var recordType in records.record) { buffer.Post(recordType); } } } }
public async Task ImportAsync(int externalRepositoryId, CancellationToken cancellationToken) { var externalRepository = m_externalRepositoryManager.GetExternalRepository(externalRepositoryId); var progressInfo = new RepositoryImportProgressInfo(externalRepositoryId, externalRepository.Name); m_importManager.ActualProgress.TryAdd(externalRepositoryId, progressInfo); var latestImportHistory = m_importHistoryManager.GetLatestSuccessfulImportHistory(externalRepositoryId); var importHistoryId = m_importHistoryManager.CreateImportHistory(externalRepositoryId, m_importManager.UserId); ImportPipeline importPipeline = null; try { m_projectImportManagers.TryGetValue(externalRepository.ExternalRepositoryType.Name, out var importManager); if (importManager == null) { throw new ImportFailedException(MainServiceErrorCode.RepositoryImportManagerNotFound, $"The import manager was not found for repository type {externalRepository.ExternalRepositoryType.Name}.", externalRepository.ExternalRepositoryType.Name); } importPipeline = m_importPipelineDirector.Build(m_importPipelineBuilder, externalRepository, importHistoryId, progressInfo, cancellationToken); await importManager.ImportFromResource(externalRepository.Configuration, importPipeline.BufferBlock, progressInfo, latestImportHistory?.Date, cancellationToken); importPipeline.BufferBlock.Complete(); importPipeline.LastBlock.Completion.Wait(cancellationToken); } catch (OperationCanceledException) { progressInfo.FaultedMessage = MainServiceErrorCode.RepositoryImportCancelled; progressInfo.FaultedMessageParams = new object[] { externalRepository.Name }; } catch (AggregateException e) { var errorMessages = new StringBuilder(); var exceptions = e.InnerExceptions; for (var i = 1; i <= exceptions.Count; i++) { errorMessages.AppendLine($"Error #{i}: {exceptions[i].Message}"); } if (m_logger.IsErrorEnabled()) { m_logger.LogError(e, errorMessages.ToString()); } progressInfo.FaultedMessage = MainServiceErrorCode.RepositoryImportFailed; progressInfo.FaultedMessageParams = new object[] { externalRepository.Name }; } catch (ImportFailedException e) { if (m_logger.IsErrorEnabled()) { m_logger.LogError(e, e.Message); } progressInfo.FaultedMessage = e.Code; progressInfo.FaultedMessageParams = e.CodeParams; } catch (Exception e) // Catch every exception otherwise, ProjectImportBackgroundService will be stopped { if (m_logger.IsErrorEnabled()) { m_logger.LogError(e, e.Message); } progressInfo.FaultedMessage = MainServiceErrorCode.RepositoryImportFailed; progressInfo.FaultedMessageParams = new object[] { externalRepository.Name }; } finally { var importHistory = m_importHistoryManager.GetImportHistory(importHistoryId); m_importManager.CancellationTokens.TryGetValue(externalRepositoryId, out var cancellationTokenSource); if (cancellationTokenSource != null && cancellationTokenSource.IsCancellationRequested) { progressInfo.FaultedMessage = MainServiceErrorCode.RepositoryImportCancelled; progressInfo.FaultedMessageParams = new object[] { externalRepository.Name }; importHistory.Message = progressInfo.FaultedMessage; importHistory.Status = ImportStatusEnum.Failed; } else if (!string.IsNullOrEmpty(progressInfo.FaultedMessage)) { importHistory.Message = progressInfo.FaultedMessage; importHistory.Status = ImportStatusEnum.Failed; m_importManager.CancelTask(externalRepositoryId); } else if (progressInfo.FailedProjectsCount > 0) { importHistory.Status = ImportStatusEnum.CompletedWithWarnings; } else { importHistory.Status = ImportStatusEnum.Completed; } progressInfo.IsCompleted = true; m_importHistoryManager.UpdateImportHistory(importHistory); if (!string.IsNullOrEmpty(progressInfo.FaultedMessage)) { importPipeline?.LastBlock.Completion.Wait(cancellationToken); } } }