public void ExtractPackage(LocalPackageInfo packageInfo, string targetFolder, bool validate) { if (packageInfo == null) { throw new ArgumentNullException(nameof(packageInfo)); } // rent package lock if (!packageInfo.LockProvider.TryLock(out object lockToken)) { throw new InvalidOperationException("Package is marked to delete, can't extract it."); } try { if (validate) { // validate var validator = new PackageDataValidator(app.LoggerFactory, app.Crypto); var result = validator.ValidatePackageAsync(packageInfo, measure: null).Result; if (!result.IsValid) { throw new InvalidOperationException($"Cannot validate package {packageInfo}:\n{string.Join("\n", result.Errors)}"); } } logger.LogInformation($"Extracting package {packageInfo} to folder: {targetFolder}"); // read all and extract var sequencer = new PackagePartsSequencer(); IEnumerable <PackageDataStreamPart> allParts = sequencer.GetPartsForPackage(packageInfo.Reference.FolderPath, packageInfo.Sequence); using (var readController = new ReadPackageDataStreamController(app.LoggerFactory, packageInfo.Reference, packageInfo.Sequence, allParts)) using (var readStream = new PackageDataStream(app.LoggerFactory, readController)) { var archive = new PackageArchive(app.CompatibilityChecker, app.MessageSerializer); archive.ReadToFolder(readStream, targetFolder); } logger.LogInformation($"Package {packageInfo} has been extracted."); } finally { packageInfo.LockProvider.Unlock(lockToken); } }
private async Task <PackageDataValidatorResult> ValidatePackageAsyncInternal(LocalPackageInfo packageInfo) { if (packageInfo == null) { throw new ArgumentNullException(nameof(packageInfo)); } if (!packageInfo.DownloadStatus.IsDownloaded) { // remark: this can be implemented but don't need it now throw new InvalidOperationException("Can't validate integrity of not fully downloaded package."); } // basic input data integrity validations if (packageInfo.Hashes.PackageSize != packageInfo.Sequence.PackageSize) { return(PackageDataValidatorResult.WithError("Hashes file provided invalid package size that does not match with sequence.")); } if (packageInfo.Metadata.PackageSize != packageInfo.Sequence.PackageSize) { return(PackageDataValidatorResult.WithError("Metadata file provided invalid package size that does not match with sequence.")); } if (packageInfo.Sequence.SegmentsCount != packageInfo.Hashes.PackageSegmentsHashes.Length) { return(PackageDataValidatorResult.WithError("Hashes file provided invalid count of segments that does not match with sequence.")); } // validate package hash calculated from segment hashes var calculatedPackageHash = cryptoProvider.HashFromHashes(packageInfo.Hashes.PackageSegmentsHashes); if (!calculatedPackageHash.Equals(packageInfo.Id)) { return(PackageDataValidatorResult.WithError($"Hash mismatch. Calculated package hash is {calculatedPackageHash:s} but expected is {packageInfo.Id:s}.")); } // before working with files - obtain lock to make sure package is not deleted on check if (!packageInfo.LockProvider.TryLock(out object lockToken)) { throw new InvalidOperationException("Can't obtain lock for this package. It is marked for deletion."); } try { // start checking files var errors = new List <string>(); var sequencer = new PackagePartsSequencer(); // check if data files exists and if correct size foreach (var dataFile in sequencer.GetDataFilesForPackage(packageInfo.Reference.FolderPath, packageInfo.Sequence)) { try { var fileInfo = new FileInfo(dataFile.Path); if (!fileInfo.Exists) { errors.Add($"Expected data file not found. File: {dataFile.Path}"); continue; } if (fileInfo.Length != dataFile.DataFileLength) { errors.Add($"Invalid length of data file. Expected is {dataFile.DataFileLength}b but actual is {fileInfo.Length}b. File: {dataFile.Path}"); continue; } } catch (Exception e) { errors.Add($"Can't validate file \"{ dataFile.Path }\". Reason: {e.Message}"); } } // don't continue if files are not OK if (errors.Any()) { return(PackageDataValidatorResult.WithErrors(errors)); } // do file hashes check IEnumerable <PackageDataStreamPart> allParts = sequencer.GetPartsForPackage(packageInfo.Reference.FolderPath, packageInfo.Sequence); try { using (var readPackageController = new ReadPackageDataStreamController(loggerFactory, packageInfo.Reference, packageInfo.Sequence, allParts)) using (var readPackageStream = new PackageDataStream(loggerFactory, readPackageController)) using (var validatePackageController = new ValidatePackageDataStreamController(loggerFactory, cryptoProvider, packageInfo.Sequence, packageInfo.Hashes, allParts, nestedStream: null)) using (var validatePackageStream = new PackageDataStream(loggerFactory, validatePackageController)) { await readPackageStream.CopyToAsync(validatePackageStream); } } catch (HashMismatchException e) { errors.Add($"Data file segment hash mismatch: {e.Message}"); } catch (Exception e) { errors.Add($"Can't process data files to validation. {e.ToString()}"); } // get result if (errors.Any()) { return(PackageDataValidatorResult.WithErrors(errors)); } return(PackageDataValidatorResult.Valid); } finally { packageInfo.LockProvider.Unlock(lockToken); } }
public LocalPackageInfo CreatePackageFromFolder(string folderToProcess, string name, MeasureItem writeMeasure) { var operationMeasure = Stopwatch.StartNew(); // storage folder for package EnsurePath(); name = string.IsNullOrWhiteSpace(name) ? FileHelper.GetFileOrDirectoryName(folderToProcess) : name; DirectoryInfo buildDirectory = Directory.CreateDirectory(CreateBuildPath()); logger.LogInformation($"Creating package \"{name}\" from folder: {folderToProcess}"); // create package archive PackageHashes packageHashes; int entriesCount; using (var controller = new CreatePackageDataStreamController(app.Version, app.LoggerFactory, app.Crypto, sequenceForNewPackages, buildDirectory.FullName)) { using (var packageStream = new PackageDataStream(app.LoggerFactory, controller) { Measure = writeMeasure }) { var archive = new PackageArchive(app.CompatibilityChecker, app.MessageSerializer); archive.WriteFromFolder(folderToProcess, packageStream); entriesCount = archive.EntriesCount; } packageHashes = controller.PackageId; } // store package hashes UpdateHashes(packageHashes, directoryPath: buildDirectory.FullName); // store download status PackageSequenceInfo packageSequence = packageHashes.CreatePackageSequence(); PackageDownloadInfo downloadStatus = PackageDownloadInfo.CreateForCreatedPackage(app.Version, packageHashes.PackageId, packageSequence); UpdateDownloadStatus(downloadStatus, directoryPath: buildDirectory.FullName); // store metadata var metadata = new PackageMeta() { Created = DateTimeOffset.Now, Name = name, PackageSize = packageHashes.PackageSize, Version = app.Version, PackageId = packageHashes.PackageId }; UpdateMetadata(metadata, directoryPath: buildDirectory.FullName); // rename folder string packagePath = CreatePackagePath(packageHashes.PackageId); if (Directory.Exists(packagePath)) { throw new InvalidOperationException($"Folder for package {packageHashes.PackageId:s} already exists. {packagePath}"); } Directory.Move(buildDirectory.FullName, packagePath); operationMeasure.Stop(); logger.LogInformation($"Created package \"{packagePath}\":\nHash: {packageHashes.PackageId}\nSize: {SizeFormatter.ToString(packageHashes.PackageSize)}\nFiles and directories: {entriesCount}\nTime: {operationMeasure.Elapsed}"); var reference = new PackageReference(packagePath, packageHashes.PackageId); var result = new LocalPackageInfo(reference, downloadStatus, packageHashes, metadata, packageSequence); return(result); }
public void ReadToFolder(PackageDataStream readStream, string rootDirectory) { if (readStream == null) { throw new ArgumentNullException(nameof(readStream)); } if (rootDirectory == null) { throw new ArgumentNullException(nameof(rootDirectory)); } rootDirectory = RemoveTrailingSlashIfPresent(rootDirectory); // ensure directory exists var rootDirectoryInfo = new DirectoryInfo(rootDirectory); rootDirectoryInfo.Create(); var version = serializer.Deserialize <ClientVersion>(readStream); compatibilityChecker.ThrowIfNotCompatibleWith("Package", version); var foldersStack = new Stack <string>(); while (true) { // read entry var entry = serializer.Deserialize <PackageEntry>(readStream); if (entry == null) { throw new InvalidOperationException("Cannot deserialize package entry."); } // final entry if (entry.Name == null) { // really end of stream? if (readStream.Position != readStream.Length) { throw new InvalidOperationException("Unexpected stream end."); } // final entry should pop to root directory if (entry.PopDirectories != foldersStack.Count) { throw new InvalidOperationException("Invalid number of directory pops on final entry."); } return; } EntriesCount++; // get to correct directory for (int i = 0; i < entry.PopDirectories; i++) { if (foldersStack.Count == 0) { throw new InvalidOperationException("Invalid number of directory pops."); } foldersStack.Pop(); } var currentFolder = foldersStack.Count > 0 ? foldersStack.Peek() : rootDirectory; var path = Path.Combine(currentFolder, entry.Name); // is it directory? if (entry.Attributes.HasFlag(FileAttributes.Directory)) { if (entry.FileSize != null) { throw new InvalidOperationException("File size is expected to be null for directory entry."); } var dir = new DirectoryInfo(path); if (dir.Exists) { throw new InvalidOperationException($"Folder \"{dir.Name}\" already exists. Full path: {dir.FullName}"); } dir.Create(); ApplyAttributes(dir, entry); foldersStack.Push(dir.FullName); continue; } // or is it file? if (entry.FileSize == null) { throw new InvalidOperationException("File size is null."); } using (var fileStream = new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, DefaultBufferSize)) { fileStream.SetLength(entry.FileSize.Value); if (entry.FileSize.Value > 0) { // write content readStream.CopyStream(fileStream, DefaultBufferSize, entry.FileSize.Value); } } var fileInfo = new FileInfo(path); ApplyAttributes(fileInfo, entry); } }