public async Task If_GetPackage_from_repository_is_successfull_preprocessing_of_asset_is_started() { // Arrange var archiveRecordId = "654"; var packageId = "666"; var downloadResult = new RepositoryPackageResult { Valid = true, Success = true, PackageDetails = new RepositoryPackage { ArchiveRecordId = archiveRecordId, PackageFileName = "someZipFile.zip" } }; repositoryManager.Setup(e => e.GetPackage(packageId, archiveRecordId, It.IsAny <int>())).ReturnsAsync(downloadResult); // Act await InputQueueSendEndpoint.Send <IDownloadPackage>(new { PackageId = packageId, ArchiveRecordId = archiveRecordId, CallerId = "someCaller", RetentionCategory = CacheRetentionCategory.UsageCopyPublic }); // Wait for the results await downloadPackageTask; var context = await prepareForTransformationAssetTask; // Assert context.Message.RepositoryPackage.ArchiveRecordId.Should().Be(archiveRecordId); context.Message.CallerId = "someCaller"; context.Message.AssetType = AssetType.Gebrauchskopie; // Download from DIR is always UsageCopy context.Message.RepositoryPackage.PackageFileName.Should().Be("someZipFile.zip"); context.Message.RetentionCategory.Should().Be(CacheRetentionCategory.UsageCopyPublic); }
public async Task If_Package_not_valid_Sync_process_is_set_to_failed() { // Arrange var ar = new ArchiveRecord { ArchiveRecordId = "344" }; var mutationId = 999; var errMsg = "Some other error message"; var appendResult = new RepositoryPackageResult { Valid = false, Success = true, ErrorMessage = errMsg }; repositoryManager.Setup(e => e.AppendPackageToArchiveRecord(It.IsAny <ArchiveRecord>(), It.IsAny <long>(), It.IsAny <int>())) .ReturnsAsync(appendResult); // Act await InputQueueSendEndpoint.Send <IArchiveRecordAppendPackage>(new { ArchiveRecord = ar, MutationId = mutationId }); // Wait for the results await appendPackageTask; var context = await archiveRecordUpdatedTask; // Assert context.Message.ActionSuccessful.Should().Be(false); context.Message.MutationId.Should().Be(mutationId); context.Message.ErrorMessage.Should().Be(errMsg); }
public async Task If_Package_is_valid_extract_fulltext_is_initiated() { // Arrange var ar = new ArchiveRecord { ArchiveRecordId = "478" }; var mutationId = 777; var errMsg = string.Empty; var appendResult = new RepositoryPackageResult { Valid = true, Success = true, ErrorMessage = errMsg, PackageDetails = new RepositoryPackage { PackageFileName = "need a file name.whatever" } }; repositoryManager.Setup(e => e.AppendPackageToArchiveRecord(It.IsAny <ArchiveRecord>(), It.IsAny <long>(), It.IsAny <int>())) .ReturnsAsync(appendResult); // Act await InputQueueSendEndpoint.Send <IArchiveRecordAppendPackage>(new { ArchiveRecord = ar, MutationId = mutationId }); // Wait for the results await appendPackageTask; var context = await extractFulltextTask; // Assert context.Message.ArchiveRecord.ArchiveRecordId.Should().Be(ar.ArchiveRecordId); context.Message.MutationId.Should().Be(mutationId); }
public async Task If_Package_is_valid_preprocessing_of_asset_is_initiated() { try { // Arrange var ar = new ArchiveRecord { ArchiveRecordId = "478" }; var mutationId = 777; var errMsg = string.Empty; var appendResult = new RepositoryPackageResult { Valid = true, Success = true, ErrorMessage = errMsg, PackageDetails = new RepositoryPackage { PackageFileName = "need a file name.whatever" } }; repositoryManager.Setup(e => e.AppendPackageToArchiveRecord(It.IsAny <ArchiveRecord>(), It.IsAny <long>(), It.IsAny <int>())) .ReturnsAsync(appendResult); var appendPackageConsumer = harness.Consumer(() => new AppendPackageConsumer(repositoryManager.Object)); harness.Consumer(() => archiveRecordAppendPackageConsumer.Object); await harness.Start(); // Act await harness.InputQueueSendEndpoint.Send <IArchiveRecordAppendPackage>(new { ArchiveRecord = ar, MutationId = mutationId }); // Wait for the results Assert.That(await harness.Consumed.Any <IArchiveRecordAppendPackage>()); Assert.That(await appendPackageConsumer.Consumed.Any <IArchiveRecordAppendPackage>()); Assert.That(await harness.Sent.Any <PrepareForRecognitionMessage>()); var context = harness.Sent.Select <PrepareForRecognitionMessage>().First().Context; // Assert context.Message.ArchiveRecord.ArchiveRecordId.Should().Be(ar.ArchiveRecordId); context.Message.MutationId.Should().Be(mutationId); } finally { await harness.Stop(); } }
public async Task If_AppendPackage_failed_Sync_process_is_set_to_failed() { // Arrange try { var ar = new ArchiveRecord { ArchiveRecordId = "345" }; var mutationId = 666; var errMsg = "Some error message"; var appendResult = new RepositoryPackageResult { Valid = false, Success = false, ErrorMessage = errMsg }; repositoryManager.Setup(e => e.AppendPackageToArchiveRecord(It.IsAny <ArchiveRecord>(), It.IsAny <long>(), It.IsAny <int>())) .ReturnsAsync(appendResult); var appendPackageConsumer = harness.Consumer(() => new AppendPackageConsumer(repositoryManager.Object)); harness.Consumer(() => archiveRecordAppendPackageConsumer.Object); await harness.Start(); // Act await harness.InputQueueSendEndpoint.Send <IArchiveRecordAppendPackage>(new { ArchiveRecord = ar, MutationId = mutationId }); // Wait for the results Assert.That(await harness.Consumed.Any <IArchiveRecordAppendPackage>()); Assert.That(await appendPackageConsumer.Consumed.Any <IArchiveRecordAppendPackage>()); Assert.That(await harness.Published.Any <IArchiveRecordUpdated>()); var context = harness.Published.Select <IArchiveRecordUpdated>().First().Context; // Assert context.Message.ActionSuccessful.Should().Be(false); context.Message.MutationId.Should().Be(mutationId); context.Message.ErrorMessage.Should().Be(errMsg); } finally { await harness.Stop(); } }
public async Task If_GetPackage_from_repository_is_not_valid_AssetReady_returns_failure() { // Arrange var archiveRecordId = "651"; var packageId = "646"; var downloadResult = new RepositoryPackageResult { Valid = false, Success = true, PackageDetails = new RepositoryPackage { ArchiveRecordId = archiveRecordId, PackageFileName = "someZipFile.zip" } }; repositoryManager.Setup(e => e.GetPackage(packageId, archiveRecordId, It.IsAny <int>())).ReturnsAsync(downloadResult); // Act await InputQueueSendEndpoint.Send <IDownloadPackage>(new { PackageId = packageId, ArchiveRecordId = archiveRecordId, CallerId = "someCaller", RetentionCategory = CacheRetentionCategory.UsageCopyPublic }); // Wait for the results await downloadPackageTask; var context = await assetReadyUpdatedTask; // Assert context.Message.ArchiveRecordId.Should().Be(archiveRecordId); context.Message.CallerId = "someCaller"; context.Message.AssetType = AssetType.Gebrauchskopie; // Download from DIR is always UsageCopy context.Message.Valid.Should().Be(false); }
internal async Task <RepositoryPackageResult> GetPackageInternal(string packageId, string archiveRecordId, bool createMetadataXml, List <string> fileTypesToIgnore, int primaerdatenAuftragId) { Debug.Assert(fileTypesToIgnore != null, "fileTypesToIgnore must not be null"); // Init the return value var retVal = new RepositoryPackageResult { Success = false, Valid = false, PackageDetails = new RepositoryPackage { ArchiveRecordId = archiveRecordId } }; var currentStatus = AufbereitungsStatusEnum.AuftragGestartet; try { var allIgnoredFiles = new List <RepositoryFile>(); var rootFolder = repositoryDataAccess.GetRepositoryRoot(packageId); if (rootFolder != null) { var tempRootFolder = GetTempRootFolder(); var zipFileName = GetZipFileName(tempRootFolder); try { var watch = Stopwatch.StartNew(); Log.Information("Fetching the metadata for package with id {packageId}", packageId); // Get the metadata about the packages retVal.PackageDetails.PackageId = packageId; retVal.PackageDetails.Folders = repositoryDataAccess.GetFolders(rootFolder.Id); retVal.PackageDetails.Files = repositoryDataAccess.GetFiles(rootFolder.Id, ignoredFilenameRegex, out var ignored); allIgnoredFiles.AddRange(ignored); // Get the sub folders of the root folders foreach (var folder in retVal.PackageDetails.Folders) { GetFolderContent(folder, allIgnoredFiles); } if (allIgnoredFiles.Count > 0) { Log.Information("We have found {fileCount} files to ignore. These are: {files}", allIgnoredFiles.Count, JsonConvert.SerializeObject(allIgnoredFiles)); } // Ensure valid file names and prevent too long paths and file names packageValidator.EnsureValidPhysicalFileAndFolderNames(retVal.PackageDetails, Path.Combine(tempRootFolder, contentFolderName)); // Now create a folder and file structure on disk matching the metadata Log.Information("Creating package structure on disk for package with id {packageId}", packageId); LogFreeDiskSpace(packageId); CreatePackageOnDisk(tempRootFolder, retVal.PackageDetails, fileTypesToIgnore); LogFreeDiskSpace(packageId); // Create the metadata.xml if (createMetadataXml) { handler.CreateMetadataXml(Path.Combine(tempRootFolder, headerFolderName), retVal.PackageDetails, allIgnoredFiles); } // Get some information about the package var numberOfFilesInZipFile = Directory.GetFiles(Path.Combine(tempRootFolder, contentFolderName), "*.*", SearchOption.AllDirectories).Length; var sizeInBytes = GetSizeInBytesFromMetadata(retVal.PackageDetails, false); var sizeInBytesOnDisk = Directory.GetFiles(tempRootFolder, "*.*", SearchOption.AllDirectories) .Select(f => new FileInfo(f).Length).Sum(); var numberOfFilesInMetadata = GetFileCountFromMetadata(retVal.PackageDetails, false); // Get all files according to DIR metadata var numberOfFilesInMetadataRespectingIgnored = GetFileCountFromMetadata(retVal.PackageDetails, true); // Get all files not counting the ignored ones Log.Information("Package with id {packageId} has size {SizeInBytes:n0} bytes", packageId, sizeInBytes); currentStatus = AufbereitungsStatusEnum.PrimaerdatenExtrahiert; await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus); // Make a zip file ZipFile.CreateFromDirectory(tempRootFolder, zipFileName); Log.Information("ZipFile created for package with id {packageId}", packageId); LogFreeDiskSpace(packageId); currentStatus = AufbereitungsStatusEnum.ZipDateiErzeugt; await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus); // Check if package is valid. // Number of files must correspond. In case of just getting the files for OCR, the createMetadataXml is false and number of files is counted differently var isValidPackage = numberOfFilesInMetadata == numberOfFilesInZipFile || !createMetadataXml && numberOfFilesInZipFile == numberOfFilesInMetadataRespectingIgnored; var fi = new FileInfo(zipFileName); if (isValidPackage) { // Copy the zip file to the final destination. // Depending on the setting either by sftp or a simple file copy if (Settings.Default.UseSFTP) { CopyBySftp(fi); } else { MoveFileToDestination(fi); } } // Delete zip file. If it was moved it is already gone, so we check if (fi.Exists) { fi.Delete(); } // Construct the result retVal.PackageDetails.FileCount = numberOfFilesInMetadata; retVal.PackageDetails.SizeInBytes = sizeInBytes; retVal.PackageDetails.PackageFileName = fi.Name; // Adjust the download time with an estimated download speed retVal.PackageDetails.RepositoryExtractionDuration = watch.ElapsedTicks + GetProcessingTimeOfIgnoredFilesInTicks(sizeInBytes - sizeInBytesOnDisk); retVal.Success = true; retVal.Valid = isValidPackage; currentStatus = AufbereitungsStatusEnum.PaketTransferiert; await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus); if (!retVal.Valid) { var metadata = JsonConvert.SerializeObject(retVal.PackageDetails); Log.Error( "Have {numberOfFilesInZipFile} files in package, but should be {numberOfFilesInMetadata} files according to metadata. Metadata is {metadata}", numberOfFilesInZipFile, numberOfFilesInMetadata, metadata); } } catch (Exception ex) { Log.Error(ex, "Unknown error while creating the package with id {packageId}.", packageId); LogFreeDiskSpace(packageId); // Do we have a stry zip file? (out of space exception while zipping...) if (File.Exists(zipFileName)) { Log.Information("Found remains of zip file. Deleting zip file {zipFileName} for package {packageId}.", zipFileName, packageId); File.Delete(zipFileName); Log.Information("Deleted zip file {zipFileName} for package {packageId}.", zipFileName, packageId); } retVal.ErrorMessage = $"Unknown error: {ex.Message}."; while (ex.InnerException != null) { retVal.ErrorMessage += Environment.NewLine + ex.InnerException.Message; ex = ex.InnerException; } } finally { // Delete the temp files Directory.Delete(tempRootFolder, true); Log.Information("Deleted temp files for package {packageId}", packageId); } } else { Log.Warning("Could not find package with id {packageId} in the repository", packageId); retVal.ErrorMessage = $"Could not find package with id {packageId} in the repository"; } } catch (Exception ex) { Log.Error(ex, "Failed to get package with id {packageId} from repository", packageId); LogFreeDiskSpace(packageId); retVal.ErrorMessage = "Failed to get package from repository"; } // Bei einem Fehlerfall melden wir den letzten Status erneut, diesmal mit ErrorText an die Priorisierungsengine if (!retVal.Success) { await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus, retVal.ErrorMessage); } return(retVal); }