Пример #1
0
        public static int Compress(ReadOnlySpan <byte> src, Span <byte> dst, CompressionDictionary dictionary)
        {
            if (_threadCompressContext == null)
                _threadCompressContext = new CompressContext();

            fixed(byte *srcPtr = src)
            fixed(byte *dstPtr = dst)
            {
                UIntPtr result;

                if (dictionary == null || dictionary.Compression == null)
                {
                    result = ZSTD_compressCCtx(_threadCompressContext.Compression, dstPtr, (UIntPtr)dst.Length,
                                               srcPtr, (UIntPtr)src.Length, 3);
                }
                else
                {
                    result = ZSTD_compress_usingCDict(_threadCompressContext.Compression, dstPtr,
                                                      (UIntPtr)dst.Length, srcPtr, (UIntPtr)src.Length, dictionary.Compression);
                }

                AssertSuccess(result, dictionary);
                return((int)result);
            }
        }
        private async Task <(long zipSize, long tarGzSize)> GetArchiveDetails(CompressContext ctx, CancellationToken cancellationToken)
        {
            (var zipBlobReference, var tgzBlobReference) = GetContainerReferences(ctx);
            await zipBlobReference.FetchAttributesAsync();

            await tgzBlobReference.FetchAttributesAsync();

            return(zipBlobReference.Properties.Length, tgzBlobReference.Properties.Length);
        }
        private void AddToTarArchive(CompressContext ctx, FileDetails fileDetails)
        {
            var tarEntry = TarEntry.CreateTarEntry(fileDetails.FullName);

            tarEntry.Size    = fileDetails.Length;
            tarEntry.ModTime = fileDetails.Modified.UtcDateTime;

            ctx.TarStream.PutNextEntry(tarEntry);
        }
        private void AddToZipArchive(CompressContext ctx, FileDetails fileDetails)
        {
            var zipEntry = new ZipEntry(fileDetails.FullName)
            {
                DateTime = fileDetails.Modified.UtcDateTime,
                Size     = fileDetails.Length,
            };

            ctx.ZipStream.PutNextEntry(zipEntry);
        }
        private void CloseOutputArchives(CompressContext ctx, CancellationToken cancellationToken)
        {
            Log.Add("Closing Zip Stream");
            ctx.ZipStream.IsStreamOwner = true;
            ctx.ZipStream.Close();

            Log.Add("Closing Tar Stream");
            ctx.TarStream.IsStreamOwner = true;
            ctx.TarStream.Close();
        }
Пример #6
0
        public static void Main(string[] args)
        {
            Console.WriteLine("Strategy Pattern Example");
            CompressContext compressContext = new CompressContext(new CompressZip());

            compressContext.CreateArchive("File1");

            compressContext.setCompressMethod(new CompressRar());
            compressContext.CreateArchive("File2");

            Console.Read();
        }
Пример #7
0
        private async Task AddDatasetFileToArchive(CompressContext ctx, CloudBlockBlob blobReference, CancellationToken cancellationToken)
        {
            AddToZipArchive(ctx, ctx.Details);
            AddToTarArchive(ctx, ctx.Details);

            using (var readStream = await blobReference.OpenReadAsync())
                using (var writeStream = new WriteTeeStream(ctx.ZipStream, ctx.TarStream))
                {
                    StreamUtils.Copy(readStream, writeStream, ctx.Buffer);
                }

            ctx.ZipStream.CloseEntry();
            ctx.TarStream.CloseEntry();
        }
        private async Task OpenOutputArchives(CompressContext ctx, CancellationToken cancellationToken)
        {
            (var zipBlobReference, var tgzBlobReference) = GetContainerReferences(ctx);

            Log.Add($"Writing zip to {zipBlobReference.StorageUri.PrimaryUri}");
            Log.Add($"Block Size: {zipBlobReference.StreamWriteSizeInBytes:n0}");
            var zipBlobStream = await zipBlobReference.OpenWriteAsync(
                null,
                new BlobRequestOptions
            {
                ParallelOperationThreadCount     = 1,
                SingleBlobUploadThresholdInBytes = 4 * 1024 * 1024,
            },
                null,
                cancellationToken);

            ctx.ZipStream = new ZipOutputStream(new WriteBlocksStream(zipBlobStream));
            ctx.ZipStream.SetLevel(9);

            Log.Add($"Writing tgz to {tgzBlobReference.StorageUri.PrimaryUri}");
            Log.Add($"Block Size: {tgzBlobReference.StreamWriteSizeInBytes:n0}");
            var tgzBlobStream = await tgzBlobReference.OpenWriteAsync(
                null,
                new BlobRequestOptions
            {
                ParallelOperationThreadCount     = 1,
                SingleBlobUploadThresholdInBytes = 4 * 1024 * 1024,
            },
                null,
                cancellationToken);

            var gzipStream = new GZipOutputStream(new WriteBlocksStream(tgzBlobStream));

            gzipStream.SetLevel(9);
            ctx.TarStream = new TarOutputStream(gzipStream);
        }
        private (CloudBlockBlob zipBlobReference, CloudBlockBlob tgzBlobReference) GetContainerReferences(CompressContext ctx)
        {
            var zipBlobName      = $"{ctx.ContainerName}.zip";
            var zipBlobReference = ctx.ArchiveContainer.GetBlockBlobReference(zipBlobName);

            var tgzBlobName      = $"{ctx.ContainerName}.tar.gz";
            var tgzBlobReference = ctx.ArchiveContainer.GetBlockBlobReference(tgzBlobName);

            return(zipBlobReference, tgzBlobReference);
        }
Пример #10
0
        private async Task ReadDatasetFiles(DatasetBlobStorageDetails storage, CancellationToken cancellationToken)
        {
            Log.Add("Reading dataset files.");

            var ctx = new CompressContext
            {
                ContainerName = storage.Container,
                Buffer        = new byte[32768],
            };

            var credentials   = new StorageCredentials(storage.Account, StorageConfig.Accounts[storage.Account]);
            var storageAcct   = new CloudStorageAccount(credentials, true);
            var blobClient    = storageAcct.CreateCloudBlobClient();
            var blobContainer = blobClient.GetContainerReference(storage.Container);

            ctx.ArchiveContainer = blobClient.GetContainerReference(ctx.ArchiveContainerName);
            Log.Add($"Writing archives to {ctx.ArchiveContainer.Uri}");
            await ctx.ArchiveContainer.CreateIfNotExistsAsync();

            BlobContinuationToken    continuationToken  = null;
            const bool               useFlatBlobListing = true;
            const BlobListingDetails blobListingDetails = BlobListingDetails.None;
            const int maxBlobsPerRequest = 100;

            int  totalCount = 0;
            long totalSize  = 0;

            await OpenOutputArchives(ctx, cancellationToken);

            do
            {
                var listingResult = await blobContainer
                                    .ListBlobsSegmentedAsync("", useFlatBlobListing, blobListingDetails, maxBlobsPerRequest, continuationToken, null, null, cancellationToken)
                                    .ConfigureAwait(false);

                continuationToken = listingResult.ContinuationToken;
                var results = listingResult.Results
                              .Cast <CloudBlockBlob>()
                              .Where(r => r.Name != "_metadata.txt")
                              .Select(blob => new FileDetails
                {
                    Name     = Path.GetFileName(blob.Name),
                    FullName = blob.Name,
                    Length   = blob.Properties.Length,
                    Modified = blob.Properties.LastModified ?? DateTimeOffset.UtcNow,
                })
                              .ToList();

                foreach (var result in results)
                {
                    Log.Add($"- {result.FullName}");
                    ctx.Details = result;
                    var blobReference = blobContainer.GetBlockBlobReference(result.FullName);
                    await AddDatasetFileToArchive(ctx, blobReference, cancellationToken);
                }

                totalCount += results.Count();
                totalSize  += results.Sum(t => t.Length);
            } while (continuationToken != null);

            CloseOutputArchives(ctx, cancellationToken);

            (var zipSize, var tgzSize) = await GetArchiveDetails(ctx, cancellationToken);

            await DatasetStorage.UpdateDatasetCompressedDetails(DatasetId, zipSize, tgzSize);

            Console.WriteLine($"Compressed {totalCount:n0} total files, {totalSize:n0} bytes.");
            Console.WriteLine($"zip file: {zipSize:n0} bytes ({Ratio(totalSize, zipSize):n2}%).");
            Console.WriteLine($"tgz file: {tgzSize:n0} bytes ({Ratio(totalSize, tgzSize):n2}%).");
        }