public void OnStreamClosed()
{
EnsureNotDisposed();
// do not allow to process twice
if (isClosed)
{
return;
}
// compute hash of last part
ComputeCurrentPartHash();
// trim last data file
if (currentPart != null)
{
long lastDataFileLength = currentPart.FileStream.Position;
currentPart.FileStream.SetLength(lastDataFileLength);
}
// dispose all
DisposeCurrentPart();
// build result
var sequenceInfo = new PackageSequenceInfo(sequenceBaseInfo, totalSize);
packageId = new Dto.PackageHashes(version, segmentHashes, cryptoProvider, sequenceInfo);
logger.LogDebug($"Closed package data files. Written {SizeFormatter.ToString(totalSize)}. Hash is {packageId.PackageId:s}.");
isClosed = true;
}
public long GetSizeOfParts(PackageSequenceInfo sequence, int[] parts)
{
long result = 0;
for (int i = 0; i < parts.Length; i++)
{
result += (parts[i] == sequence.SegmentsCount - 1) ? sequence.SegmentLastLength : sequence.SegmentLength;
}
return(result);
}
public PackageDownloadInfo(PackageDownload dto, Packaging.PackageSequenceInfo sequenceInfo)
{
this.dto = dto ?? throw new ArgumentNullException(nameof(dto));
this.sequenceInfo = sequenceInfo ?? throw new ArgumentNullException(nameof(sequenceInfo));
partsInProgress = new HashSet <int>();
int lastSegmentBits = (byte)(sequenceInfo.SegmentsCount % 8);
lastByteMask = (byte)((1 << (lastSegmentBits == 0 ? 8 : lastSegmentBits)) - 1);
}
public void Allocate(string path, PackageSequenceInfo sequence, bool overwrite)
{
if (path == null)
{
throw new ArgumentNullException(nameof(path));
}
if (sequence == null)
{
throw new ArgumentNullException(nameof(sequence));
}
logger.LogInformation($"Allocating {SizeFormatter.ToString(sequence.PackageSize)} for package data in {path}");
Directory.CreateDirectory(path);
// check disk space and throw error if not enough
var driveInfo = new DriveInfo(Directory.GetDirectoryRoot(path));
long freeSpace = driveInfo.TotalFreeSpace;
if (freeSpace < sequence.PackageSize)
{
throw new InvalidOperationException($"There is not enough disk space on drive {driveInfo.Name}. Free space is {SizeFormatter.ToString(freeSpace)} but required is {SizeFormatter.ToString(sequence.PackageSize)}.");
}
// prepare parts
var sequencer = new PackagePartsSequencer();
var parts = sequencer.GetDataFilesForPackage(path, sequence).ToArray();
if (!overwrite)
{
// check if already exists
foreach (var part in parts)
{
if (File.Exists(part.Path))
{
throw new Exception($"File already exists: {part.Path}");
}
}
}
// allocate
foreach (var part in parts)
{
using (var fs = new FileStream(part.Path, overwrite ? FileMode.OpenOrCreate : FileMode.CreateNew, FileAccess.Write, FileShare.None))
{
fs.SetLength(part.PartLength);
}
}
logger.LogDebug("Allocation completed.");
}
private IEnumerable <PackageDataStreamPart> GetPartsInternal(string packageRootPath, PackageSequenceBaseInfo sequenceBaseInfo, long?length, int[] requestedSegments)
{
PackageSequenceInfo sequenceInfo = null;
bool isInfinite = length == null;
if (!isInfinite)
{
sequenceInfo = new PackageSequenceInfo(sequenceBaseInfo, length.Value);
}
IEnumerable <int> segmentIndexEnumerable;
if (requestedSegments != null)
{
segmentIndexEnumerable = requestedSegments;
}
else
{
segmentIndexEnumerable = Enumerable.Range(0, isInfinite ? int.MaxValue : sequenceInfo.SegmentsCount);
}
foreach (var segmentIndex in segmentIndexEnumerable)
{
// validate is requested correct index
if (!isInfinite && (segmentIndex < 0 || segmentIndex >= sequenceInfo.SegmentsCount))
{
throw new ArgumentOutOfRangeException(nameof(requestedSegments), "Requested part is out of range.");
}
int segmentIndexInDataFile = (segmentIndex % sequenceBaseInfo.SegmentsPerDataFile);
int dataFileIndex = (segmentIndex / sequenceBaseInfo.SegmentsPerDataFile);
yield return(new PackageDataStreamPart()
{
DataFileIndex = dataFileIndex,
SegmentIndex = segmentIndex,
PartLength = isInfinite ? sequenceBaseInfo.SegmentLength : sequenceInfo.GetSizeOfSegment(segmentIndex),
Path = GetBlockFilePath(packageRootPath, dataFileIndex),
SegmentOffsetInDataFile = segmentIndexInDataFile * sequenceBaseInfo.SegmentLength,
DataFileLength = isInfinite ? sequenceBaseInfo.DataFileLength : sequenceInfo.GetSizeOfDataFile(dataFileIndex)
});
}
}
public IEnumerable <PackageDataStreamPart> GetPartsForSpecificSegments(string packageRootPath, PackageSequenceInfo sequenceInfo, int[] requestedSegments)
{
if (sequenceInfo == null)
{
throw new ArgumentNullException(nameof(sequenceInfo));
}
if (requestedSegments == null)
{
throw new ArgumentNullException(nameof(requestedSegments));
}
return(GetPartsInternal(packageRootPath, sequenceInfo, length: sequenceInfo.PackageSize, requestedSegments: requestedSegments));
}
public IEnumerable <PackageDataStreamPart> GetPartsForPackage(string packageRootPath, PackageSequenceInfo sequenceInfo)
{
return(GetPartsInternal(packageRootPath, sequenceInfo, length: sequenceInfo.PackageSize, requestedSegments: null));
}
public IEnumerable <PackageDataStreamPart> GetDataFilesForPackage(string packageRootPath, PackageSequenceInfo sequenceInfo)
{
for (int currentDataFileIndex = 0; currentDataFileIndex < sequenceInfo.DataFilesCount; currentDataFileIndex++)
{
long dataFileSize = (currentDataFileIndex == sequenceInfo.DataFilesCount - 1) ? sequenceInfo.DataFileLastLength : sequenceInfo.DataFileLength;
string path = GetBlockFilePath(packageRootPath, currentDataFileIndex);
yield return(new PackageDataStreamPart()
{
Path = path,
PartLength = dataFileSize,
DataFileLength = dataFileSize,
DataFileIndex = currentDataFileIndex,
SegmentOffsetInDataFile = 0
});
}
}
public LocalPackageInfo(PackageReference reference, PackageDownloadInfo downloadStatus, Dto.PackageHashes hashes, Dto.PackageMeta metadata, PackageSequenceInfo sequence)
{
LockProvider = new PackageLocks();
DownloadMeasure = new MeasureItem(MeasureType.Throughput);
UploadMeasure = new MeasureItem(MeasureType.Throughput);
Reference = reference ?? throw new ArgumentNullException(nameof(reference));
DownloadStatus = downloadStatus ?? throw new ArgumentNullException(nameof(downloadStatus));
Hashes = hashes ?? throw new ArgumentNullException(nameof(hashes));
Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
Sequence = sequence ?? throw new ArgumentNullException(nameof(sequence));
if (!Reference.Id.Equals(DownloadStatus.PackageId))
{
throw new ArgumentException("Invalid hash.", nameof(downloadStatus));
}
if (!Reference.Id.Equals(Hashes.PackageId))
{
throw new ArgumentException("Invalid hash.", nameof(hashes));
}
if (!Reference.Id.Equals(Metadata.PackageId))
{
throw new ArgumentException("Invalid hash.", nameof(metadata));
}
if (!Metadata.PackageSize.Equals(Sequence.PackageSize))
{
throw new ArgumentException("Invalid size of package sequence.", nameof(sequence));
}
}
public static PackageDownloadInfo CreateForReadyForDownloadPackage(ClientVersion version, Hash packageId, Packaging.PackageSequenceInfo sequenceInfo)
{
if (sequenceInfo == null)
{
throw new ArgumentNullException(nameof(sequenceInfo));
}
int bitmapSize = GetBitmapSizeForPackage(sequenceInfo.SegmentsCount);
var data = new PackageDownload()
{
PackageId = packageId,
IsDownloading = false, // don't start automatically - this needs to be handled by downloader
DownloadedBytes = 0, // nothing downloaded yet
Version = version,
SegmentsBitmap = new byte[bitmapSize]
};
return(new PackageDownloadInfo(data, sequenceInfo));
}
public static PackageDownloadInfo CreateForCreatedPackage(ClientVersion version, Hash packageId, Packaging.PackageSequenceInfo sequenceInfo)
{
if (sequenceInfo == null)
{
throw new ArgumentNullException(nameof(sequenceInfo));
}
var data = new PackageDownload()
{
PackageId = packageId,
IsDownloading = false, // already downloaded
DownloadedBytes = sequenceInfo.PackageSize, // all downloaded
Version = version,
SegmentsBitmap = null // already downloaded (=null)
};
return(new PackageDownloadInfo(data, sequenceInfo));
}
public LocalPackageInfo CreatePackageFromFolder(string folderToProcess, string name, MeasureItem writeMeasure)
{
var operationMeasure = Stopwatch.StartNew();
// storage folder for package
EnsurePath();
name = string.IsNullOrWhiteSpace(name) ? FileHelper.GetFileOrDirectoryName(folderToProcess) : name;
DirectoryInfo buildDirectory = Directory.CreateDirectory(CreateBuildPath());
logger.LogInformation($"Creating package \"{name}\" from folder: {folderToProcess}");
// create package archive
PackageHashes packageHashes;
int entriesCount;
using (var controller = new CreatePackageDataStreamController(app.Version, app.LoggerFactory, app.Crypto, sequenceForNewPackages, buildDirectory.FullName))
{
using (var packageStream = new PackageDataStream(app.LoggerFactory, controller)
{
Measure = writeMeasure
})
{
var archive = new PackageArchive(app.CompatibilityChecker, app.MessageSerializer);
archive.WriteFromFolder(folderToProcess, packageStream);
entriesCount = archive.EntriesCount;
}
packageHashes = controller.PackageId;
}
// store package hashes
UpdateHashes(packageHashes, directoryPath: buildDirectory.FullName);
// store download status
PackageSequenceInfo packageSequence = packageHashes.CreatePackageSequence();
PackageDownloadInfo downloadStatus = PackageDownloadInfo.CreateForCreatedPackage(app.Version, packageHashes.PackageId, packageSequence);
UpdateDownloadStatus(downloadStatus, directoryPath: buildDirectory.FullName);
// store metadata
var metadata = new PackageMeta()
{
Created = DateTimeOffset.Now,
Name = name,
PackageSize = packageHashes.PackageSize,
Version = app.Version,
PackageId = packageHashes.PackageId
};
UpdateMetadata(metadata, directoryPath: buildDirectory.FullName);
// rename folder
string packagePath = CreatePackagePath(packageHashes.PackageId);
if (Directory.Exists(packagePath))
{
throw new InvalidOperationException($"Folder for package {packageHashes.PackageId:s} already exists. {packagePath}");
}
Directory.Move(buildDirectory.FullName, packagePath);
operationMeasure.Stop();
logger.LogInformation($"Created package \"{packagePath}\":\nHash: {packageHashes.PackageId}\nSize: {SizeFormatter.ToString(packageHashes.PackageSize)}\nFiles and directories: {entriesCount}\nTime: {operationMeasure.Elapsed}");
var reference = new PackageReference(packagePath, packageHashes.PackageId);
var result = new LocalPackageInfo(reference, downloadStatus, packageHashes, metadata, packageSequence);
return(result);
}
public PackageDownloadInfo ReadPackageDownloadStatus(PackageReference reference, PackageSequenceInfo sequenceInfo)
{
var dto = ReadPackageFile <PackageDownload>(reference, PackageDownloadFileName);
var result = new PackageDownloadInfo(dto, sequenceInfo);
return(result);
}
public ReadPackageDataStreamController(ILoggerFactory loggerFactory, PackageReference package, PackageSequenceInfo packageSequence, IEnumerable <PackageDataStreamPart> requestedParts)
{
if (package == null)
{
throw new ArgumentNullException(nameof(package));
}
logger = (loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory))).CreateLogger <ReadPackageDataStreamController>();
parts = requestedParts.ToArray();
Length = parts.Sum(p => p.PartLength);
}
