public void ExtractPackage(LocalPackageInfo packageInfo, string targetFolder, bool validate)
{
if (packageInfo == null)
{
throw new ArgumentNullException(nameof(packageInfo));
}
// rent package lock
if (!packageInfo.LockProvider.TryLock(out object lockToken))
{
throw new InvalidOperationException("Package is marked to delete, can't extract it.");
}
try
{
if (validate)
{
// validate
var validator = new PackageDataValidator(app.LoggerFactory, app.Crypto);
var result = validator.ValidatePackageAsync(packageInfo, measure: null).Result;
if (!result.IsValid)
{
throw new InvalidOperationException($"Cannot validate package {packageInfo}:\n{string.Join("\n", result.Errors)}");
}
}
logger.LogInformation($"Extracting package {packageInfo} to folder: {targetFolder}");
// read all and extract
var sequencer = new PackagePartsSequencer();
IEnumerable <PackageDataStreamPart> allParts = sequencer.GetPartsForPackage(packageInfo.Reference.FolderPath, packageInfo.Sequence);
using (var readController = new ReadPackageDataStreamController(app.LoggerFactory, packageInfo.Reference, packageInfo.Sequence, allParts))
using (var readStream = new PackageDataStream(app.LoggerFactory, readController))
{
var archive = new PackageArchive(app.CompatibilityChecker, app.MessageSerializer);
archive.ReadToFolder(readStream, targetFolder);
}
logger.LogInformation($"Package {packageInfo} has been extracted.");
}
finally
{
packageInfo.LockProvider.Unlock(lockToken);
}
}
private async Task <PackageDataValidatorResult> ValidatePackageAsyncInternal(LocalPackageInfo packageInfo)
{
if (packageInfo == null)
{
throw new ArgumentNullException(nameof(packageInfo));
}
if (!packageInfo.DownloadStatus.IsDownloaded)
{
// remark: this can be implemented but don't need it now
throw new InvalidOperationException("Can't validate integrity of not fully downloaded package.");
}
// basic input data integrity validations
if (packageInfo.Hashes.PackageSize != packageInfo.Sequence.PackageSize)
{
return(PackageDataValidatorResult.WithError("Hashes file provided invalid package size that does not match with sequence."));
}
if (packageInfo.Metadata.PackageSize != packageInfo.Sequence.PackageSize)
{
return(PackageDataValidatorResult.WithError("Metadata file provided invalid package size that does not match with sequence."));
}
if (packageInfo.Sequence.SegmentsCount != packageInfo.Hashes.PackageSegmentsHashes.Length)
{
return(PackageDataValidatorResult.WithError("Hashes file provided invalid count of segments that does not match with sequence."));
}
// validate package hash calculated from segment hashes
var calculatedPackageHash = cryptoProvider.HashFromHashes(packageInfo.Hashes.PackageSegmentsHashes);
if (!calculatedPackageHash.Equals(packageInfo.Id))
{
return(PackageDataValidatorResult.WithError($"Hash mismatch. Calculated package hash is {calculatedPackageHash:s} but expected is {packageInfo.Id:s}."));
}
// before working with files - obtain lock to make sure package is not deleted on check
if (!packageInfo.LockProvider.TryLock(out object lockToken))
{
throw new InvalidOperationException("Can't obtain lock for this package. It is marked for deletion.");
}
try
{
// start checking files
var errors = new List <string>();
var sequencer = new PackagePartsSequencer();
// check if data files exists and if correct size
foreach (var dataFile in sequencer.GetDataFilesForPackage(packageInfo.Reference.FolderPath, packageInfo.Sequence))
{
try
{
var fileInfo = new FileInfo(dataFile.Path);
if (!fileInfo.Exists)
{
errors.Add($"Expected data file not found. File: {dataFile.Path}");
continue;
}
if (fileInfo.Length != dataFile.DataFileLength)
{
errors.Add($"Invalid length of data file. Expected is {dataFile.DataFileLength}b but actual is {fileInfo.Length}b. File: {dataFile.Path}");
continue;
}
}
catch (Exception e)
{
errors.Add($"Can't validate file \"{ dataFile.Path }\". Reason: {e.Message}");
}
}
// don't continue if files are not OK
if (errors.Any())
{
return(PackageDataValidatorResult.WithErrors(errors));
}
// do file hashes check
IEnumerable <PackageDataStreamPart> allParts = sequencer.GetPartsForPackage(packageInfo.Reference.FolderPath, packageInfo.Sequence);
try
{
using (var readPackageController = new ReadPackageDataStreamController(loggerFactory, packageInfo.Reference, packageInfo.Sequence, allParts))
using (var readPackageStream = new PackageDataStream(loggerFactory, readPackageController))
using (var validatePackageController = new ValidatePackageDataStreamController(loggerFactory, cryptoProvider, packageInfo.Sequence, packageInfo.Hashes, allParts, nestedStream: null))
using (var validatePackageStream = new PackageDataStream(loggerFactory, validatePackageController))
{
await readPackageStream.CopyToAsync(validatePackageStream);
}
}
catch (HashMismatchException e)
{
errors.Add($"Data file segment hash mismatch: {e.Message}");
}
catch (Exception e)
{
errors.Add($"Can't process data files to validation. {e.ToString()}");
}
// get result
if (errors.Any())
{
return(PackageDataValidatorResult.WithErrors(errors));
}
return(PackageDataValidatorResult.Valid);
}
finally
{
packageInfo.LockProvider.Unlock(lockToken);
}
}
public LocalPackageInfo CreatePackageFromFolder(string folderToProcess, string name, MeasureItem writeMeasure)
{
var operationMeasure = Stopwatch.StartNew();
// storage folder for package
EnsurePath();
name = string.IsNullOrWhiteSpace(name) ? FileHelper.GetFileOrDirectoryName(folderToProcess) : name;
DirectoryInfo buildDirectory = Directory.CreateDirectory(CreateBuildPath());
logger.LogInformation($"Creating package \"{name}\" from folder: {folderToProcess}");
// create package archive
PackageHashes packageHashes;
int entriesCount;
using (var controller = new CreatePackageDataStreamController(app.Version, app.LoggerFactory, app.Crypto, sequenceForNewPackages, buildDirectory.FullName))
{
using (var packageStream = new PackageDataStream(app.LoggerFactory, controller)
{
Measure = writeMeasure
})
{
var archive = new PackageArchive(app.CompatibilityChecker, app.MessageSerializer);
archive.WriteFromFolder(folderToProcess, packageStream);
entriesCount = archive.EntriesCount;
}
packageHashes = controller.PackageId;
}
// store package hashes
UpdateHashes(packageHashes, directoryPath: buildDirectory.FullName);
// store download status
PackageSequenceInfo packageSequence = packageHashes.CreatePackageSequence();
PackageDownloadInfo downloadStatus = PackageDownloadInfo.CreateForCreatedPackage(app.Version, packageHashes.PackageId, packageSequence);
UpdateDownloadStatus(downloadStatus, directoryPath: buildDirectory.FullName);
// store metadata
var metadata = new PackageMeta()
{
Created = DateTimeOffset.Now,
Name = name,
PackageSize = packageHashes.PackageSize,
Version = app.Version,
PackageId = packageHashes.PackageId
};
UpdateMetadata(metadata, directoryPath: buildDirectory.FullName);
// rename folder
string packagePath = CreatePackagePath(packageHashes.PackageId);
if (Directory.Exists(packagePath))
{
throw new InvalidOperationException($"Folder for package {packageHashes.PackageId:s} already exists. {packagePath}");
}
Directory.Move(buildDirectory.FullName, packagePath);
operationMeasure.Stop();
logger.LogInformation($"Created package \"{packagePath}\":\nHash: {packageHashes.PackageId}\nSize: {SizeFormatter.ToString(packageHashes.PackageSize)}\nFiles and directories: {entriesCount}\nTime: {operationMeasure.Elapsed}");
var reference = new PackageReference(packagePath, packageHashes.PackageId);
var result = new LocalPackageInfo(reference, downloadStatus, packageHashes, metadata, packageSequence);
return(result);
}
public void ReadToFolder(PackageDataStream readStream, string rootDirectory)
{
if (readStream == null)
{
throw new ArgumentNullException(nameof(readStream));
}
if (rootDirectory == null)
{
throw new ArgumentNullException(nameof(rootDirectory));
}
rootDirectory = RemoveTrailingSlashIfPresent(rootDirectory);
// ensure directory exists
var rootDirectoryInfo = new DirectoryInfo(rootDirectory);
rootDirectoryInfo.Create();
var version = serializer.Deserialize <ClientVersion>(readStream);
compatibilityChecker.ThrowIfNotCompatibleWith("Package", version);
var foldersStack = new Stack <string>();
while (true)
{
// read entry
var entry = serializer.Deserialize <PackageEntry>(readStream);
if (entry == null)
{
throw new InvalidOperationException("Cannot deserialize package entry.");
}
// final entry
if (entry.Name == null)
{
// really end of stream?
if (readStream.Position != readStream.Length)
{
throw new InvalidOperationException("Unexpected stream end.");
}
// final entry should pop to root directory
if (entry.PopDirectories != foldersStack.Count)
{
throw new InvalidOperationException("Invalid number of directory pops on final entry.");
}
return;
}
EntriesCount++;
// get to correct directory
for (int i = 0; i < entry.PopDirectories; i++)
{
if (foldersStack.Count == 0)
{
throw new InvalidOperationException("Invalid number of directory pops.");
}
foldersStack.Pop();
}
var currentFolder = foldersStack.Count > 0 ? foldersStack.Peek() : rootDirectory;
var path = Path.Combine(currentFolder, entry.Name);
// is it directory?
if (entry.Attributes.HasFlag(FileAttributes.Directory))
{
if (entry.FileSize != null)
{
throw new InvalidOperationException("File size is expected to be null for directory entry.");
}
var dir = new DirectoryInfo(path);
if (dir.Exists)
{
throw new InvalidOperationException($"Folder \"{dir.Name}\" already exists. Full path: {dir.FullName}");
}
dir.Create();
ApplyAttributes(dir, entry);
foldersStack.Push(dir.FullName);
continue;
}
// or is it file?
if (entry.FileSize == null)
{
throw new InvalidOperationException("File size is null.");
}
using (var fileStream = new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, DefaultBufferSize))
{
fileStream.SetLength(entry.FileSize.Value);
if (entry.FileSize.Value > 0)
{
// write content
readStream.CopyStream(fileStream, DefaultBufferSize, entry.FileSize.Value);
}
}
var fileInfo = new FileInfo(path);
ApplyAttributes(fileInfo, entry);
}
}