public async Task If_GetPackage_from_repository_is_successfull_preprocessing_of_asset_is_started()
{
// Arrange
var archiveRecordId = "654";
var packageId = "666";
var downloadResult = new RepositoryPackageResult
{
Valid = true, Success = true, PackageDetails = new RepositoryPackage
{
ArchiveRecordId = archiveRecordId, PackageFileName = "someZipFile.zip"
}
};
repositoryManager.Setup(e => e.GetPackage(packageId, archiveRecordId, It.IsAny <int>())).ReturnsAsync(downloadResult);
// Act
await InputQueueSendEndpoint.Send <IDownloadPackage>(new
{
PackageId = packageId,
ArchiveRecordId = archiveRecordId,
CallerId = "someCaller",
RetentionCategory = CacheRetentionCategory.UsageCopyPublic
});
// Wait for the results
await downloadPackageTask;
var context = await prepareForTransformationAssetTask;
// Assert
context.Message.RepositoryPackage.ArchiveRecordId.Should().Be(archiveRecordId);
context.Message.CallerId = "someCaller";
context.Message.AssetType = AssetType.Gebrauchskopie; // Download from DIR is always UsageCopy
context.Message.RepositoryPackage.PackageFileName.Should().Be("someZipFile.zip");
context.Message.RetentionCategory.Should().Be(CacheRetentionCategory.UsageCopyPublic);
}
public async Task If_Package_not_valid_Sync_process_is_set_to_failed()
{
// Arrange
var ar = new ArchiveRecord {
ArchiveRecordId = "344"
};
var mutationId = 999;
var errMsg = "Some other error message";
var appendResult = new RepositoryPackageResult {
Valid = false, Success = true, ErrorMessage = errMsg
};
repositoryManager.Setup(e => e.AppendPackageToArchiveRecord(It.IsAny <ArchiveRecord>(), It.IsAny <long>(), It.IsAny <int>()))
.ReturnsAsync(appendResult);
// Act
await InputQueueSendEndpoint.Send <IArchiveRecordAppendPackage>(new
{
ArchiveRecord = ar,
MutationId = mutationId
});
// Wait for the results
await appendPackageTask;
var context = await archiveRecordUpdatedTask;
// Assert
context.Message.ActionSuccessful.Should().Be(false);
context.Message.MutationId.Should().Be(mutationId);
context.Message.ErrorMessage.Should().Be(errMsg);
}
public async Task If_Package_is_valid_extract_fulltext_is_initiated()
{
// Arrange
var ar = new ArchiveRecord {
ArchiveRecordId = "478"
};
var mutationId = 777;
var errMsg = string.Empty;
var appendResult = new RepositoryPackageResult
{
Valid = true, Success = true, ErrorMessage = errMsg, PackageDetails = new RepositoryPackage
{
PackageFileName = "need a file name.whatever"
}
};
repositoryManager.Setup(e => e.AppendPackageToArchiveRecord(It.IsAny <ArchiveRecord>(), It.IsAny <long>(), It.IsAny <int>()))
.ReturnsAsync(appendResult);
// Act
await InputQueueSendEndpoint.Send <IArchiveRecordAppendPackage>(new
{
ArchiveRecord = ar,
MutationId = mutationId
});
// Wait for the results
await appendPackageTask;
var context = await extractFulltextTask;
// Assert
context.Message.ArchiveRecord.ArchiveRecordId.Should().Be(ar.ArchiveRecordId);
context.Message.MutationId.Should().Be(mutationId);
}
public async Task If_Package_is_valid_preprocessing_of_asset_is_initiated()
{
try
{
// Arrange
var ar = new ArchiveRecord {
ArchiveRecordId = "478"
};
var mutationId = 777;
var errMsg = string.Empty;
var appendResult = new RepositoryPackageResult
{
Valid = true, Success = true, ErrorMessage = errMsg, PackageDetails = new RepositoryPackage
{
PackageFileName = "need a file name.whatever"
}
};
repositoryManager.Setup(e => e.AppendPackageToArchiveRecord(It.IsAny <ArchiveRecord>(), It.IsAny <long>(), It.IsAny <int>()))
.ReturnsAsync(appendResult);
var appendPackageConsumer = harness.Consumer(() => new AppendPackageConsumer(repositoryManager.Object));
harness.Consumer(() => archiveRecordAppendPackageConsumer.Object);
await harness.Start();
// Act
await harness.InputQueueSendEndpoint.Send <IArchiveRecordAppendPackage>(new
{
ArchiveRecord = ar,
MutationId = mutationId
});
// Wait for the results
Assert.That(await harness.Consumed.Any <IArchiveRecordAppendPackage>());
Assert.That(await appendPackageConsumer.Consumed.Any <IArchiveRecordAppendPackage>());
Assert.That(await harness.Sent.Any <PrepareForRecognitionMessage>());
var context = harness.Sent.Select <PrepareForRecognitionMessage>().First().Context;
// Assert
context.Message.ArchiveRecord.ArchiveRecordId.Should().Be(ar.ArchiveRecordId);
context.Message.MutationId.Should().Be(mutationId);
}
finally
{
await harness.Stop();
}
}
public async Task If_AppendPackage_failed_Sync_process_is_set_to_failed()
{
// Arrange
try
{
var ar = new ArchiveRecord {
ArchiveRecordId = "345"
};
var mutationId = 666;
var errMsg = "Some error message";
var appendResult = new RepositoryPackageResult {
Valid = false, Success = false, ErrorMessage = errMsg
};
repositoryManager.Setup(e => e.AppendPackageToArchiveRecord(It.IsAny <ArchiveRecord>(), It.IsAny <long>(), It.IsAny <int>()))
.ReturnsAsync(appendResult);
var appendPackageConsumer = harness.Consumer(() => new AppendPackageConsumer(repositoryManager.Object));
harness.Consumer(() => archiveRecordAppendPackageConsumer.Object);
await harness.Start();
// Act
await harness.InputQueueSendEndpoint.Send <IArchiveRecordAppendPackage>(new
{
ArchiveRecord = ar,
MutationId = mutationId
});
// Wait for the results
Assert.That(await harness.Consumed.Any <IArchiveRecordAppendPackage>());
Assert.That(await appendPackageConsumer.Consumed.Any <IArchiveRecordAppendPackage>());
Assert.That(await harness.Published.Any <IArchiveRecordUpdated>());
var context = harness.Published.Select <IArchiveRecordUpdated>().First().Context;
// Assert
context.Message.ActionSuccessful.Should().Be(false);
context.Message.MutationId.Should().Be(mutationId);
context.Message.ErrorMessage.Should().Be(errMsg);
}
finally
{
await harness.Stop();
}
}
public async Task If_GetPackage_from_repository_is_not_valid_AssetReady_returns_failure()
{
// Arrange
var archiveRecordId = "651";
var packageId = "646";
var downloadResult = new RepositoryPackageResult
{
Valid = false,
Success = true,
PackageDetails = new RepositoryPackage
{
ArchiveRecordId = archiveRecordId,
PackageFileName = "someZipFile.zip"
}
};
repositoryManager.Setup(e => e.GetPackage(packageId, archiveRecordId, It.IsAny <int>())).ReturnsAsync(downloadResult);
// Act
await InputQueueSendEndpoint.Send <IDownloadPackage>(new
{
PackageId = packageId,
ArchiveRecordId = archiveRecordId,
CallerId = "someCaller",
RetentionCategory = CacheRetentionCategory.UsageCopyPublic
});
// Wait for the results
await downloadPackageTask;
var context = await assetReadyUpdatedTask;
// Assert
context.Message.ArchiveRecordId.Should().Be(archiveRecordId);
context.Message.CallerId = "someCaller";
context.Message.AssetType = AssetType.Gebrauchskopie; // Download from DIR is always UsageCopy
context.Message.Valid.Should().Be(false);
}
internal async Task <RepositoryPackageResult> GetPackageInternal(string packageId, string archiveRecordId, bool createMetadataXml,
List <string> fileTypesToIgnore, int primaerdatenAuftragId)
{
Debug.Assert(fileTypesToIgnore != null, "fileTypesToIgnore must not be null");
// Init the return value
var retVal = new RepositoryPackageResult
{
Success = false,
Valid = false,
PackageDetails = new RepositoryPackage {
ArchiveRecordId = archiveRecordId
}
};
var currentStatus = AufbereitungsStatusEnum.AuftragGestartet;
try
{
var allIgnoredFiles = new List <RepositoryFile>();
var rootFolder = repositoryDataAccess.GetRepositoryRoot(packageId);
if (rootFolder != null)
{
var tempRootFolder = GetTempRootFolder();
var zipFileName = GetZipFileName(tempRootFolder);
try
{
var watch = Stopwatch.StartNew();
Log.Information("Fetching the metadata for package with id {packageId}", packageId);
// Get the metadata about the packages
retVal.PackageDetails.PackageId = packageId;
retVal.PackageDetails.Folders = repositoryDataAccess.GetFolders(rootFolder.Id);
retVal.PackageDetails.Files = repositoryDataAccess.GetFiles(rootFolder.Id, ignoredFilenameRegex, out var ignored);
allIgnoredFiles.AddRange(ignored);
// Get the sub folders of the root folders
foreach (var folder in retVal.PackageDetails.Folders)
{
GetFolderContent(folder, allIgnoredFiles);
}
if (allIgnoredFiles.Count > 0)
{
Log.Information("We have found {fileCount} files to ignore. These are: {files}", allIgnoredFiles.Count,
JsonConvert.SerializeObject(allIgnoredFiles));
}
// Ensure valid file names and prevent too long paths and file names
packageValidator.EnsureValidPhysicalFileAndFolderNames(retVal.PackageDetails,
Path.Combine(tempRootFolder, contentFolderName));
// Now create a folder and file structure on disk matching the metadata
Log.Information("Creating package structure on disk for package with id {packageId}", packageId);
LogFreeDiskSpace(packageId);
CreatePackageOnDisk(tempRootFolder, retVal.PackageDetails, fileTypesToIgnore);
LogFreeDiskSpace(packageId);
// Create the metadata.xml
if (createMetadataXml)
{
handler.CreateMetadataXml(Path.Combine(tempRootFolder, headerFolderName), retVal.PackageDetails, allIgnoredFiles);
}
// Get some information about the package
var numberOfFilesInZipFile =
Directory.GetFiles(Path.Combine(tempRootFolder, contentFolderName), "*.*", SearchOption.AllDirectories).Length;
var sizeInBytes = GetSizeInBytesFromMetadata(retVal.PackageDetails, false);
var sizeInBytesOnDisk = Directory.GetFiles(tempRootFolder, "*.*", SearchOption.AllDirectories)
.Select(f => new FileInfo(f).Length).Sum();
var numberOfFilesInMetadata =
GetFileCountFromMetadata(retVal.PackageDetails, false); // Get all files according to DIR metadata
var numberOfFilesInMetadataRespectingIgnored =
GetFileCountFromMetadata(retVal.PackageDetails, true); // Get all files not counting the ignored ones
Log.Information("Package with id {packageId} has size {SizeInBytes:n0} bytes", packageId, sizeInBytes);
currentStatus = AufbereitungsStatusEnum.PrimaerdatenExtrahiert;
await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus);
// Make a zip file
ZipFile.CreateFromDirectory(tempRootFolder, zipFileName);
Log.Information("ZipFile created for package with id {packageId}", packageId);
LogFreeDiskSpace(packageId);
currentStatus = AufbereitungsStatusEnum.ZipDateiErzeugt;
await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus);
// Check if package is valid.
// Number of files must correspond. In case of just getting the files for OCR, the createMetadataXml is false and number of files is counted differently
var isValidPackage = numberOfFilesInMetadata == numberOfFilesInZipFile ||
!createMetadataXml && numberOfFilesInZipFile == numberOfFilesInMetadataRespectingIgnored;
var fi = new FileInfo(zipFileName);
if (isValidPackage)
{
// Copy the zip file to the final destination.
// Depending on the setting either by sftp or a simple file copy
if (Settings.Default.UseSFTP)
{
CopyBySftp(fi);
}
else
{
MoveFileToDestination(fi);
}
}
// Delete zip file. If it was moved it is already gone, so we check
if (fi.Exists)
{
fi.Delete();
}
// Construct the result
retVal.PackageDetails.FileCount = numberOfFilesInMetadata;
retVal.PackageDetails.SizeInBytes = sizeInBytes;
retVal.PackageDetails.PackageFileName = fi.Name;
// Adjust the download time with an estimated download speed
retVal.PackageDetails.RepositoryExtractionDuration =
watch.ElapsedTicks + GetProcessingTimeOfIgnoredFilesInTicks(sizeInBytes - sizeInBytesOnDisk);
retVal.Success = true;
retVal.Valid = isValidPackage;
currentStatus = AufbereitungsStatusEnum.PaketTransferiert;
await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus);
if (!retVal.Valid)
{
var metadata = JsonConvert.SerializeObject(retVal.PackageDetails);
Log.Error(
"Have {numberOfFilesInZipFile} files in package, but should be {numberOfFilesInMetadata} files according to metadata. Metadata is {metadata}",
numberOfFilesInZipFile,
numberOfFilesInMetadata, metadata);
}
}
catch (Exception ex)
{
Log.Error(ex, "Unknown error while creating the package with id {packageId}.", packageId);
LogFreeDiskSpace(packageId);
// Do we have a stry zip file? (out of space exception while zipping...)
if (File.Exists(zipFileName))
{
Log.Information("Found remains of zip file. Deleting zip file {zipFileName} for package {packageId}.", zipFileName,
packageId);
File.Delete(zipFileName);
Log.Information("Deleted zip file {zipFileName} for package {packageId}.", zipFileName, packageId);
}
retVal.ErrorMessage = $"Unknown error: {ex.Message}.";
while (ex.InnerException != null)
{
retVal.ErrorMessage += Environment.NewLine + ex.InnerException.Message;
ex = ex.InnerException;
}
}
finally
{
// Delete the temp files
Directory.Delete(tempRootFolder, true);
Log.Information("Deleted temp files for package {packageId}", packageId);
}
}
else
{
Log.Warning("Could not find package with id {packageId} in the repository", packageId);
retVal.ErrorMessage = $"Could not find package with id {packageId} in the repository";
}
}
catch (Exception ex)
{
Log.Error(ex, "Failed to get package with id {packageId} from repository", packageId);
LogFreeDiskSpace(packageId);
retVal.ErrorMessage = "Failed to get package from repository";
}
// Bei einem Fehlerfall melden wir den letzten Status erneut, diesmal mit ErrorText an die Priorisierungsengine
if (!retVal.Success)
{
await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus, retVal.ErrorMessage);
}
return(retVal);
}
