public static int Compress(ReadOnlySpan <byte> src, Span <byte> dst, CompressionDictionary dictionary)
{
if (_threadCompressContext == null)
_threadCompressContext = new CompressContext();
fixed(byte *srcPtr = src)
fixed(byte *dstPtr = dst)
{
UIntPtr result;
if (dictionary == null || dictionary.Compression == null)
{
result = ZSTD_compressCCtx(_threadCompressContext.Compression, dstPtr, (UIntPtr)dst.Length,
srcPtr, (UIntPtr)src.Length, 3);
}
else
{
result = ZSTD_compress_usingCDict(_threadCompressContext.Compression, dstPtr,
(UIntPtr)dst.Length, srcPtr, (UIntPtr)src.Length, dictionary.Compression);
}
AssertSuccess(result, dictionary);
return((int)result);
}
}
private async Task <(long zipSize, long tarGzSize)> GetArchiveDetails(CompressContext ctx, CancellationToken cancellationToken)
{
(var zipBlobReference, var tgzBlobReference) = GetContainerReferences(ctx);
await zipBlobReference.FetchAttributesAsync();
await tgzBlobReference.FetchAttributesAsync();
return(zipBlobReference.Properties.Length, tgzBlobReference.Properties.Length);
}
private void AddToTarArchive(CompressContext ctx, FileDetails fileDetails)
{
var tarEntry = TarEntry.CreateTarEntry(fileDetails.FullName);
tarEntry.Size = fileDetails.Length;
tarEntry.ModTime = fileDetails.Modified.UtcDateTime;
ctx.TarStream.PutNextEntry(tarEntry);
}
private void AddToZipArchive(CompressContext ctx, FileDetails fileDetails)
{
var zipEntry = new ZipEntry(fileDetails.FullName)
{
DateTime = fileDetails.Modified.UtcDateTime,
Size = fileDetails.Length,
};
ctx.ZipStream.PutNextEntry(zipEntry);
}
private void CloseOutputArchives(CompressContext ctx, CancellationToken cancellationToken)
{
Log.Add("Closing Zip Stream");
ctx.ZipStream.IsStreamOwner = true;
ctx.ZipStream.Close();
Log.Add("Closing Tar Stream");
ctx.TarStream.IsStreamOwner = true;
ctx.TarStream.Close();
}
public static void Main(string[] args)
{
Console.WriteLine("Strategy Pattern Example");
CompressContext compressContext = new CompressContext(new CompressZip());
compressContext.CreateArchive("File1");
compressContext.setCompressMethod(new CompressRar());
compressContext.CreateArchive("File2");
Console.Read();
}
private async Task AddDatasetFileToArchive(CompressContext ctx, CloudBlockBlob blobReference, CancellationToken cancellationToken)
{
AddToZipArchive(ctx, ctx.Details);
AddToTarArchive(ctx, ctx.Details);
using (var readStream = await blobReference.OpenReadAsync())
using (var writeStream = new WriteTeeStream(ctx.ZipStream, ctx.TarStream))
{
StreamUtils.Copy(readStream, writeStream, ctx.Buffer);
}
ctx.ZipStream.CloseEntry();
ctx.TarStream.CloseEntry();
}
private async Task OpenOutputArchives(CompressContext ctx, CancellationToken cancellationToken)
{
(var zipBlobReference, var tgzBlobReference) = GetContainerReferences(ctx);
Log.Add($"Writing zip to {zipBlobReference.StorageUri.PrimaryUri}");
Log.Add($"Block Size: {zipBlobReference.StreamWriteSizeInBytes:n0}");
var zipBlobStream = await zipBlobReference.OpenWriteAsync(
null,
new BlobRequestOptions
{
ParallelOperationThreadCount = 1,
SingleBlobUploadThresholdInBytes = 4 * 1024 * 1024,
},
null,
cancellationToken);
ctx.ZipStream = new ZipOutputStream(new WriteBlocksStream(zipBlobStream));
ctx.ZipStream.SetLevel(9);
Log.Add($"Writing tgz to {tgzBlobReference.StorageUri.PrimaryUri}");
Log.Add($"Block Size: {tgzBlobReference.StreamWriteSizeInBytes:n0}");
var tgzBlobStream = await tgzBlobReference.OpenWriteAsync(
null,
new BlobRequestOptions
{
ParallelOperationThreadCount = 1,
SingleBlobUploadThresholdInBytes = 4 * 1024 * 1024,
},
null,
cancellationToken);
var gzipStream = new GZipOutputStream(new WriteBlocksStream(tgzBlobStream));
gzipStream.SetLevel(9);
ctx.TarStream = new TarOutputStream(gzipStream);
}
private (CloudBlockBlob zipBlobReference, CloudBlockBlob tgzBlobReference) GetContainerReferences(CompressContext ctx)
{
var zipBlobName = $"{ctx.ContainerName}.zip";
var zipBlobReference = ctx.ArchiveContainer.GetBlockBlobReference(zipBlobName);
var tgzBlobName = $"{ctx.ContainerName}.tar.gz";
var tgzBlobReference = ctx.ArchiveContainer.GetBlockBlobReference(tgzBlobName);
return(zipBlobReference, tgzBlobReference);
}
private async Task ReadDatasetFiles(DatasetBlobStorageDetails storage, CancellationToken cancellationToken)
{
Log.Add("Reading dataset files.");
var ctx = new CompressContext
{
ContainerName = storage.Container,
Buffer = new byte[32768],
};
var credentials = new StorageCredentials(storage.Account, StorageConfig.Accounts[storage.Account]);
var storageAcct = new CloudStorageAccount(credentials, true);
var blobClient = storageAcct.CreateCloudBlobClient();
var blobContainer = blobClient.GetContainerReference(storage.Container);
ctx.ArchiveContainer = blobClient.GetContainerReference(ctx.ArchiveContainerName);
Log.Add($"Writing archives to {ctx.ArchiveContainer.Uri}");
await ctx.ArchiveContainer.CreateIfNotExistsAsync();
BlobContinuationToken continuationToken = null;
const bool useFlatBlobListing = true;
const BlobListingDetails blobListingDetails = BlobListingDetails.None;
const int maxBlobsPerRequest = 100;
int totalCount = 0;
long totalSize = 0;
await OpenOutputArchives(ctx, cancellationToken);
do
{
var listingResult = await blobContainer
.ListBlobsSegmentedAsync("", useFlatBlobListing, blobListingDetails, maxBlobsPerRequest, continuationToken, null, null, cancellationToken)
.ConfigureAwait(false);
continuationToken = listingResult.ContinuationToken;
var results = listingResult.Results
.Cast <CloudBlockBlob>()
.Where(r => r.Name != "_metadata.txt")
.Select(blob => new FileDetails
{
Name = Path.GetFileName(blob.Name),
FullName = blob.Name,
Length = blob.Properties.Length,
Modified = blob.Properties.LastModified ?? DateTimeOffset.UtcNow,
})
.ToList();
foreach (var result in results)
{
Log.Add($"- {result.FullName}");
ctx.Details = result;
var blobReference = blobContainer.GetBlockBlobReference(result.FullName);
await AddDatasetFileToArchive(ctx, blobReference, cancellationToken);
}
totalCount += results.Count();
totalSize += results.Sum(t => t.Length);
} while (continuationToken != null);
CloseOutputArchives(ctx, cancellationToken);
(var zipSize, var tgzSize) = await GetArchiveDetails(ctx, cancellationToken);
await DatasetStorage.UpdateDatasetCompressedDetails(DatasetId, zipSize, tgzSize);
Console.WriteLine($"Compressed {totalCount:n0} total files, {totalSize:n0} bytes.");
Console.WriteLine($"zip file: {zipSize:n0} bytes ({Ratio(totalSize, zipSize):n2}%).");
Console.WriteLine($"tgz file: {tgzSize:n0} bytes ({Ratio(totalSize, tgzSize):n2}%).");
}