internal async Task <RepositoryPackageResult> GetPackageInternal(string packageId, string archiveRecordId, bool createMetadataXml,
List <string> fileTypesToIgnore, int primaerdatenAuftragId)
{
Debug.Assert(fileTypesToIgnore != null, "fileTypesToIgnore must not be null");
// Init the return value
var retVal = new RepositoryPackageResult
{
Success = false,
Valid = false,
PackageDetails = new RepositoryPackage {
ArchiveRecordId = archiveRecordId
}
};
var currentStatus = AufbereitungsStatusEnum.AuftragGestartet;
try
{
var allIgnoredFiles = new List <RepositoryFile>();
var rootFolder = repositoryDataAccess.GetRepositoryRoot(packageId);
if (rootFolder != null)
{
var tempRootFolder = GetTempRootFolder();
var zipFileName = GetZipFileName(tempRootFolder);
try
{
var watch = Stopwatch.StartNew();
Log.Information("Fetching the metadata for package with id {packageId}", packageId);
// Get the metadata about the packages
retVal.PackageDetails.PackageId = packageId;
retVal.PackageDetails.Folders = repositoryDataAccess.GetFolders(rootFolder.Id);
retVal.PackageDetails.Files = repositoryDataAccess.GetFiles(rootFolder.Id, ignoredFilenameRegex, out var ignored);
allIgnoredFiles.AddRange(ignored);
// Get the sub folders of the root folders
foreach (var folder in retVal.PackageDetails.Folders)
{
GetFolderContent(folder, allIgnoredFiles);
}
if (allIgnoredFiles.Count > 0)
{
Log.Information("We have found {fileCount} files to ignore. These are: {files}", allIgnoredFiles.Count,
JsonConvert.SerializeObject(allIgnoredFiles));
}
// Ensure valid file names and prevent too long paths and file names
packageValidator.EnsureValidPhysicalFileAndFolderNames(retVal.PackageDetails,
Path.Combine(tempRootFolder, contentFolderName));
// Now create a folder and file structure on disk matching the metadata
Log.Information("Creating package structure on disk for package with id {packageId}", packageId);
LogFreeDiskSpace(packageId);
CreatePackageOnDisk(tempRootFolder, retVal.PackageDetails, fileTypesToIgnore);
LogFreeDiskSpace(packageId);
// Create the metadata.xml
if (createMetadataXml)
{
handler.CreateMetadataXml(Path.Combine(tempRootFolder, headerFolderName), retVal.PackageDetails, allIgnoredFiles);
}
// Get some information about the package
var numberOfFilesInZipFile =
Directory.GetFiles(Path.Combine(tempRootFolder, contentFolderName), "*.*", SearchOption.AllDirectories).Length;
var sizeInBytes = GetSizeInBytesFromMetadata(retVal.PackageDetails, false);
var sizeInBytesOnDisk = Directory.GetFiles(tempRootFolder, "*.*", SearchOption.AllDirectories)
.Select(f => new FileInfo(f).Length).Sum();
var numberOfFilesInMetadata =
GetFileCountFromMetadata(retVal.PackageDetails, false); // Get all files according to DIR metadata
var numberOfFilesInMetadataRespectingIgnored =
GetFileCountFromMetadata(retVal.PackageDetails, true); // Get all files not counting the ignored ones
Log.Information("Package with id {packageId} has size {SizeInBytes:n0} bytes", packageId, sizeInBytes);
currentStatus = AufbereitungsStatusEnum.PrimaerdatenExtrahiert;
await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus);
// Make a zip file
ZipFile.CreateFromDirectory(tempRootFolder, zipFileName);
Log.Information("ZipFile created for package with id {packageId}", packageId);
LogFreeDiskSpace(packageId);
currentStatus = AufbereitungsStatusEnum.ZipDateiErzeugt;
await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus);
// Check if package is valid.
// Number of files must correspond. In case of just getting the files for OCR, the createMetadataXml is false and number of files is counted differently
var isValidPackage = numberOfFilesInMetadata == numberOfFilesInZipFile ||
!createMetadataXml && numberOfFilesInZipFile == numberOfFilesInMetadataRespectingIgnored;
var fi = new FileInfo(zipFileName);
if (isValidPackage)
{
// Copy the zip file to the final destination.
// Depending on the setting either by sftp or a simple file copy
if (Settings.Default.UseSFTP)
{
CopyBySftp(fi);
}
else
{
MoveFileToDestination(fi);
}
}
// Delete zip file. If it was moved it is already gone, so we check
if (fi.Exists)
{
fi.Delete();
}
// Construct the result
retVal.PackageDetails.FileCount = numberOfFilesInMetadata;
retVal.PackageDetails.SizeInBytes = sizeInBytes;
retVal.PackageDetails.PackageFileName = fi.Name;
// Adjust the download time with an estimated download speed
retVal.PackageDetails.RepositoryExtractionDuration =
watch.ElapsedTicks + GetProcessingTimeOfIgnoredFilesInTicks(sizeInBytes - sizeInBytesOnDisk);
retVal.Success = true;
retVal.Valid = isValidPackage;
currentStatus = AufbereitungsStatusEnum.PaketTransferiert;
await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus);
if (!retVal.Valid)
{
var metadata = JsonConvert.SerializeObject(retVal.PackageDetails);
Log.Error(
"Have {numberOfFilesInZipFile} files in package, but should be {numberOfFilesInMetadata} files according to metadata. Metadata is {metadata}",
numberOfFilesInZipFile,
numberOfFilesInMetadata, metadata);
}
}
catch (Exception ex)
{
Log.Error(ex, "Unknown error while creating the package with id {packageId}.", packageId);
LogFreeDiskSpace(packageId);
// Do we have a stry zip file? (out of space exception while zipping...)
if (File.Exists(zipFileName))
{
Log.Information("Found remains of zip file. Deleting zip file {zipFileName} for package {packageId}.", zipFileName,
packageId);
File.Delete(zipFileName);
Log.Information("Deleted zip file {zipFileName} for package {packageId}.", zipFileName, packageId);
}
retVal.ErrorMessage = $"Unknown error: {ex.Message}.";
while (ex.InnerException != null)
{
retVal.ErrorMessage += Environment.NewLine + ex.InnerException.Message;
ex = ex.InnerException;
}
}
finally
{
// Delete the temp files
Directory.Delete(tempRootFolder, true);
Log.Information("Deleted temp files for package {packageId}", packageId);
}
}
else
{
Log.Warning("Could not find package with id {packageId} in the repository", packageId);
retVal.ErrorMessage = $"Could not find package with id {packageId} in the repository";
}
}
catch (Exception ex)
{
Log.Error(ex, "Failed to get package with id {packageId} from repository", packageId);
LogFreeDiskSpace(packageId);
retVal.ErrorMessage = "Failed to get package from repository";
}
// Bei einem Fehlerfall melden wir den letzten Status erneut, diesmal mit ErrorText an die Priorisierungsengine
if (!retVal.Success)
{
await UpdatePrimaerdatenAuftragStatus(primaerdatenAuftragId, currentStatus, retVal.ErrorMessage);
}
return(retVal);
}
