public void OnStreamClosed() { EnsureNotDisposed(); // do not allow to process twice if (isClosed) { return; } // compute hash of last part ComputeCurrentPartHash(); // trim last data file if (currentPart != null) { long lastDataFileLength = currentPart.FileStream.Position; currentPart.FileStream.SetLength(lastDataFileLength); } // dispose all DisposeCurrentPart(); // build result var sequenceInfo = new PackageSequenceInfo(sequenceBaseInfo, totalSize); packageId = new Dto.PackageHashes(version, segmentHashes, cryptoProvider, sequenceInfo); logger.LogDebug($"Closed package data files. Written {SizeFormatter.ToString(totalSize)}. Hash is {packageId.PackageId:s}."); isClosed = true; }
public long GetSizeOfParts(PackageSequenceInfo sequence, int[] parts) { long result = 0; for (int i = 0; i < parts.Length; i++) { result += (parts[i] == sequence.SegmentsCount - 1) ? sequence.SegmentLastLength : sequence.SegmentLength; } return(result); }
public PackageDownloadInfo(PackageDownload dto, Packaging.PackageSequenceInfo sequenceInfo) { this.dto = dto ?? throw new ArgumentNullException(nameof(dto)); this.sequenceInfo = sequenceInfo ?? throw new ArgumentNullException(nameof(sequenceInfo)); partsInProgress = new HashSet <int>(); int lastSegmentBits = (byte)(sequenceInfo.SegmentsCount % 8); lastByteMask = (byte)((1 << (lastSegmentBits == 0 ? 8 : lastSegmentBits)) - 1); }
public void Allocate(string path, PackageSequenceInfo sequence, bool overwrite) { if (path == null) { throw new ArgumentNullException(nameof(path)); } if (sequence == null) { throw new ArgumentNullException(nameof(sequence)); } logger.LogInformation($"Allocating {SizeFormatter.ToString(sequence.PackageSize)} for package data in {path}"); Directory.CreateDirectory(path); // check disk space and throw error if not enough var driveInfo = new DriveInfo(Directory.GetDirectoryRoot(path)); long freeSpace = driveInfo.TotalFreeSpace; if (freeSpace < sequence.PackageSize) { throw new InvalidOperationException($"There is not enough disk space on drive {driveInfo.Name}. Free space is {SizeFormatter.ToString(freeSpace)} but required is {SizeFormatter.ToString(sequence.PackageSize)}."); } // prepare parts var sequencer = new PackagePartsSequencer(); var parts = sequencer.GetDataFilesForPackage(path, sequence).ToArray(); if (!overwrite) { // check if already exists foreach (var part in parts) { if (File.Exists(part.Path)) { throw new Exception($"File already exists: {part.Path}"); } } } // allocate foreach (var part in parts) { using (var fs = new FileStream(part.Path, overwrite ? FileMode.OpenOrCreate : FileMode.CreateNew, FileAccess.Write, FileShare.None)) { fs.SetLength(part.PartLength); } } logger.LogDebug("Allocation completed."); }
private IEnumerable <PackageDataStreamPart> GetPartsInternal(string packageRootPath, PackageSequenceBaseInfo sequenceBaseInfo, long?length, int[] requestedSegments) { PackageSequenceInfo sequenceInfo = null; bool isInfinite = length == null; if (!isInfinite) { sequenceInfo = new PackageSequenceInfo(sequenceBaseInfo, length.Value); } IEnumerable <int> segmentIndexEnumerable; if (requestedSegments != null) { segmentIndexEnumerable = requestedSegments; } else { segmentIndexEnumerable = Enumerable.Range(0, isInfinite ? int.MaxValue : sequenceInfo.SegmentsCount); } foreach (var segmentIndex in segmentIndexEnumerable) { // validate is requested correct index if (!isInfinite && (segmentIndex < 0 || segmentIndex >= sequenceInfo.SegmentsCount)) { throw new ArgumentOutOfRangeException(nameof(requestedSegments), "Requested part is out of range."); } int segmentIndexInDataFile = (segmentIndex % sequenceBaseInfo.SegmentsPerDataFile); int dataFileIndex = (segmentIndex / sequenceBaseInfo.SegmentsPerDataFile); yield return(new PackageDataStreamPart() { DataFileIndex = dataFileIndex, SegmentIndex = segmentIndex, PartLength = isInfinite ? sequenceBaseInfo.SegmentLength : sequenceInfo.GetSizeOfSegment(segmentIndex), Path = GetBlockFilePath(packageRootPath, dataFileIndex), SegmentOffsetInDataFile = segmentIndexInDataFile * sequenceBaseInfo.SegmentLength, DataFileLength = isInfinite ? sequenceBaseInfo.DataFileLength : sequenceInfo.GetSizeOfDataFile(dataFileIndex) }); } }
public IEnumerable <PackageDataStreamPart> GetPartsForSpecificSegments(string packageRootPath, PackageSequenceInfo sequenceInfo, int[] requestedSegments) { if (sequenceInfo == null) { throw new ArgumentNullException(nameof(sequenceInfo)); } if (requestedSegments == null) { throw new ArgumentNullException(nameof(requestedSegments)); } return(GetPartsInternal(packageRootPath, sequenceInfo, length: sequenceInfo.PackageSize, requestedSegments: requestedSegments)); }
public IEnumerable <PackageDataStreamPart> GetPartsForPackage(string packageRootPath, PackageSequenceInfo sequenceInfo) { return(GetPartsInternal(packageRootPath, sequenceInfo, length: sequenceInfo.PackageSize, requestedSegments: null)); }
public IEnumerable <PackageDataStreamPart> GetDataFilesForPackage(string packageRootPath, PackageSequenceInfo sequenceInfo) { for (int currentDataFileIndex = 0; currentDataFileIndex < sequenceInfo.DataFilesCount; currentDataFileIndex++) { long dataFileSize = (currentDataFileIndex == sequenceInfo.DataFilesCount - 1) ? sequenceInfo.DataFileLastLength : sequenceInfo.DataFileLength; string path = GetBlockFilePath(packageRootPath, currentDataFileIndex); yield return(new PackageDataStreamPart() { Path = path, PartLength = dataFileSize, DataFileLength = dataFileSize, DataFileIndex = currentDataFileIndex, SegmentOffsetInDataFile = 0 }); } }
public LocalPackageInfo(PackageReference reference, PackageDownloadInfo downloadStatus, Dto.PackageHashes hashes, Dto.PackageMeta metadata, PackageSequenceInfo sequence) { LockProvider = new PackageLocks(); DownloadMeasure = new MeasureItem(MeasureType.Throughput); UploadMeasure = new MeasureItem(MeasureType.Throughput); Reference = reference ?? throw new ArgumentNullException(nameof(reference)); DownloadStatus = downloadStatus ?? throw new ArgumentNullException(nameof(downloadStatus)); Hashes = hashes ?? throw new ArgumentNullException(nameof(hashes)); Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); Sequence = sequence ?? throw new ArgumentNullException(nameof(sequence)); if (!Reference.Id.Equals(DownloadStatus.PackageId)) { throw new ArgumentException("Invalid hash.", nameof(downloadStatus)); } if (!Reference.Id.Equals(Hashes.PackageId)) { throw new ArgumentException("Invalid hash.", nameof(hashes)); } if (!Reference.Id.Equals(Metadata.PackageId)) { throw new ArgumentException("Invalid hash.", nameof(metadata)); } if (!Metadata.PackageSize.Equals(Sequence.PackageSize)) { throw new ArgumentException("Invalid size of package sequence.", nameof(sequence)); } }
public static PackageDownloadInfo CreateForReadyForDownloadPackage(ClientVersion version, Hash packageId, Packaging.PackageSequenceInfo sequenceInfo) { if (sequenceInfo == null) { throw new ArgumentNullException(nameof(sequenceInfo)); } int bitmapSize = GetBitmapSizeForPackage(sequenceInfo.SegmentsCount); var data = new PackageDownload() { PackageId = packageId, IsDownloading = false, // don't start automatically - this needs to be handled by downloader DownloadedBytes = 0, // nothing downloaded yet Version = version, SegmentsBitmap = new byte[bitmapSize] }; return(new PackageDownloadInfo(data, sequenceInfo)); }
public static PackageDownloadInfo CreateForCreatedPackage(ClientVersion version, Hash packageId, Packaging.PackageSequenceInfo sequenceInfo) { if (sequenceInfo == null) { throw new ArgumentNullException(nameof(sequenceInfo)); } var data = new PackageDownload() { PackageId = packageId, IsDownloading = false, // already downloaded DownloadedBytes = sequenceInfo.PackageSize, // all downloaded Version = version, SegmentsBitmap = null // already downloaded (=null) }; return(new PackageDownloadInfo(data, sequenceInfo)); }
public LocalPackageInfo CreatePackageFromFolder(string folderToProcess, string name, MeasureItem writeMeasure) { var operationMeasure = Stopwatch.StartNew(); // storage folder for package EnsurePath(); name = string.IsNullOrWhiteSpace(name) ? FileHelper.GetFileOrDirectoryName(folderToProcess) : name; DirectoryInfo buildDirectory = Directory.CreateDirectory(CreateBuildPath()); logger.LogInformation($"Creating package \"{name}\" from folder: {folderToProcess}"); // create package archive PackageHashes packageHashes; int entriesCount; using (var controller = new CreatePackageDataStreamController(app.Version, app.LoggerFactory, app.Crypto, sequenceForNewPackages, buildDirectory.FullName)) { using (var packageStream = new PackageDataStream(app.LoggerFactory, controller) { Measure = writeMeasure }) { var archive = new PackageArchive(app.CompatibilityChecker, app.MessageSerializer); archive.WriteFromFolder(folderToProcess, packageStream); entriesCount = archive.EntriesCount; } packageHashes = controller.PackageId; } // store package hashes UpdateHashes(packageHashes, directoryPath: buildDirectory.FullName); // store download status PackageSequenceInfo packageSequence = packageHashes.CreatePackageSequence(); PackageDownloadInfo downloadStatus = PackageDownloadInfo.CreateForCreatedPackage(app.Version, packageHashes.PackageId, packageSequence); UpdateDownloadStatus(downloadStatus, directoryPath: buildDirectory.FullName); // store metadata var metadata = new PackageMeta() { Created = DateTimeOffset.Now, Name = name, PackageSize = packageHashes.PackageSize, Version = app.Version, PackageId = packageHashes.PackageId }; UpdateMetadata(metadata, directoryPath: buildDirectory.FullName); // rename folder string packagePath = CreatePackagePath(packageHashes.PackageId); if (Directory.Exists(packagePath)) { throw new InvalidOperationException($"Folder for package {packageHashes.PackageId:s} already exists. {packagePath}"); } Directory.Move(buildDirectory.FullName, packagePath); operationMeasure.Stop(); logger.LogInformation($"Created package \"{packagePath}\":\nHash: {packageHashes.PackageId}\nSize: {SizeFormatter.ToString(packageHashes.PackageSize)}\nFiles and directories: {entriesCount}\nTime: {operationMeasure.Elapsed}"); var reference = new PackageReference(packagePath, packageHashes.PackageId); var result = new LocalPackageInfo(reference, downloadStatus, packageHashes, metadata, packageSequence); return(result); }
public PackageDownloadInfo ReadPackageDownloadStatus(PackageReference reference, PackageSequenceInfo sequenceInfo) { var dto = ReadPackageFile <PackageDownload>(reference, PackageDownloadFileName); var result = new PackageDownloadInfo(dto, sequenceInfo); return(result); }
public ReadPackageDataStreamController(ILoggerFactory loggerFactory, PackageReference package, PackageSequenceInfo packageSequence, IEnumerable <PackageDataStreamPart> requestedParts) { if (package == null) { throw new ArgumentNullException(nameof(package)); } logger = (loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory))).CreateLogger <ReadPackageDataStreamController>(); parts = requestedParts.ToArray(); Length = parts.Sum(p => p.PartLength); }