private async Task <byte[]> ConvertContentAttributeAsync( XAttribute contentsAttribute, CancellationToken cancellationToken ) { var text = contentsAttribute.Value; var compressedBytes = Convert.FromBase64String(text); using var outStream = new MemoryStream(); using (var inStream = new MemoryStream(compressedBytes)) using (var deflateStream = new DeflateStream(inStream, CompressionMode.Decompress)) { #if NETCOREAPP await deflateStream .CopyToAsync(outStream, cancellationToken) .ConfigureAwait(false); #else await deflateStream.CopyToAsync(outStream).ConfigureAwait(false); #endif } var bytes = outStream.ToArray(); await LogInfoAsync( $"Parsing complete. bytes.length={bytes.Length}", cancellationToken ) .ConfigureAwait(false); return(bytes); }
public void CopyToAsyncArgumentValidation() { using (DeflateStream ds = new DeflateStream(new MemoryStream(), CompressionMode.Decompress)) { AssertExtensions.Throws <ArgumentNullException>("destination", () => { ds.CopyToAsync(null); }); AssertExtensions.Throws <ArgumentOutOfRangeException>("bufferSize", () => { ds.CopyToAsync(new MemoryStream(), 0); }); Assert.Throws <NotSupportedException>(() => { ds.CopyToAsync(new MemoryStream(new byte[1], writable: false)); }); ds.Dispose(); Assert.Throws <ObjectDisposedException>(() => { ds.CopyToAsync(new MemoryStream()); }); } using (DeflateStream ds = new DeflateStream(new MemoryStream(), CompressionMode.Compress)) { Assert.Throws <NotSupportedException>(() => { ds.CopyToAsync(new MemoryStream()); }); } }
public async ValueTask <ArraySegment <byte> > DecompressAsync(ReadOnlyMemory <byte> compressedData) { Guard.AgainstEmpty(compressedData, nameof(compressedData)); using var uncompressedStream = RecyclableManager.GetStream(nameof(RecyclableDeflateProvider)); using (var deflateStream = new DeflateStream(compressedData.AsStream(), CompressionMode.Decompress, false)) { await deflateStream .CopyToAsync(uncompressedStream) .ConfigureAwait(false); } if (uncompressedStream.TryGetBuffer(out var buffer)) { return(buffer); } else { // dispose stream after allocation. using (uncompressedStream) { return(uncompressedStream.ToArray()); } } }
private byte[] GetResponseByte() { string encoding = string.IsNullOrWhiteSpace(this.WebResponse.CharacterSet) ? "utf-8" : this.WebResponse.CharacterSet; using (MemoryStream resStream = new MemoryStream()) { if (this.WebResponse.ContentEncoding.ToLower().Contains("gzip")) { using (GZipStream stream = new GZipStream(this.WebResponse.GetResponseStream(), CompressionMode.Decompress)) { stream.CopyToAsync(resStream, 10240); } } else if (this.WebResponse.ContentEncoding.ToLower().Contains("deflate")) { using (DeflateStream stream = new DeflateStream(this.WebResponse.GetResponseStream(), CompressionMode.Decompress)) { stream.CopyToAsync(resStream, 10240); } } else { using (Stream stream = this.WebResponse.GetResponseStream()) { stream.CopyToAsync(resStream, 10240); } } return(resStream.ToArray()); } }
/// <summary> /// Gets the definitions asynchronous. /// </summary> /// <param name="storagePath">The storage path.</param> /// <param name="game">The game.</param> /// <param name="path">The path.</param> /// <returns>Task<IEnumerable<IDefinition>>.</returns> public virtual async Task <IEnumerable <IDefinition> > GetDefinitionsAsync(string storagePath, IGame game, string path) { storagePath = ResolveStoragePath(storagePath); path = SanitizePath(path); var fullPath = Path.Combine(storagePath, game.Type, path); if (File.Exists(fullPath)) { var bytes = await File.ReadAllBytesAsync(fullPath); if (bytes.Any()) { using var source = new MemoryStream(bytes); using var destination = new MemoryStream(); using var compress = new DeflateStream(source, CompressionMode.Decompress); await compress.CopyToAsync(destination); var text = Encoding.UTF8.GetString(destination.ToArray()); if (!string.IsNullOrWhiteSpace(text)) { var result = JsonDISerializer.Deserialize <List <IDefinition> >(text); return(result); } } } return(null); }
public async Task Decompress(Stream streamToDecompress, Stream decompressed) { using (var defalateStream = new DeflateStream(streamToDecompress, CompressionMode.Decompress)) { await defalateStream.CopyToAsync(decompressed); } }
/// <summary> /// Automatically decompresses the current message's content if encoded with the Deflate compression algorithm /// </summary> /// <returns>This <see cref="HttpResponseReader"/></returns> public HttpResponseReader <T> UseDeflateDecompression() { this.funcs.Enqueue(async(message, _) => { if (message.Content.Headers.ContentEncoding.FirstOrDefault() == "deflate") { MemoryStream decompressed = new MemoryStream(); using (Stream content = await message.Content.ReadAsStreamAsync()) { using (DeflateStream deflate = new DeflateStream(content, CompressionMode.Decompress)) { await deflate.CopyToAsync(decompressed); } } decompressed.Seek(0, SeekOrigin.Begin); var newContent = new StreamContent(decompressed); newContent.Headers.Clear(); TransferHeaders(message.Content, newContent); message.Content.Dispose(); message.Content = newContent; } return(true); }); return(this); }
public static async Task <byte[]> DecompressAsync(this byte[] data, string encoding) { byte[] decompressedData; using (var outputStream = new MemoryStream()) { using (var inputStream = new MemoryStream(data)) { if (encoding == "gzip") { using (var zip = new GZipStream(inputStream, CompressionMode.Decompress)) { await zip.CopyToAsync(outputStream).AnyContext(); } } else if (encoding == "deflate") { using (var zip = new DeflateStream(inputStream, CompressionMode.Decompress)) { await zip.CopyToAsync(outputStream).AnyContext(); } } else { throw new InvalidOperationException($"Unsupported encoding type \"{encoding}\"."); } } decompressedData = outputStream.ToArray(); } return(decompressedData); }
/// <inheritdoc/> public async Task DecompressChunkAsync( byte[] chunk, byte[] aes256key, Stream output, NefsProgress p) { // Setup decryption using (var rijAlg = new RijndaelManaged()) { rijAlg.KeySize = 256; rijAlg.Key = aes256key; rijAlg.Mode = CipherMode.ECB; rijAlg.BlockSize = 128; rijAlg.Padding = PaddingMode.Zeros; var decryptor = rijAlg.CreateDecryptor(); // Copy the comrpessed chunk to a memory stream, decrypt and decompress (inflate) it using (var ms = new MemoryStream(chunk)) using (var cryptoStream = new CryptoStream(ms, decryptor, CryptoStreamMode.Read)) using (var inflater = new DeflateStream(cryptoStream, CompressionMode.Decompress)) { await inflater.CopyToAsync(output, StreamExtensions.CopyBufferSize, p.CancellationToken); } } }
public override async Task ProcessReceivedFrame(WebSocketFrame f) { if ((f.OpCode == WebSocketOpCode.BinaryFrame || f.OpCode == WebSocketOpCode.TextFrame) && f.Flags.HasFlag(WebSocketFlags.RSV3)) { try { using (MemoryStream ms = new MemoryStream()) { using (var ms2 = new MemoryStream()) { await ms2.WriteAsync(f.Payload, 0, f.Payload.Length); await ms2.WriteAsync(new byte[] { 0x00, 0x00, 0xff, 0xff }, 0, 4); ms2.Seek(0, SeekOrigin.Begin); using (DeflateStream df = new DeflateStream(ms2, CompressionMode.Decompress, true)) { await df.CopyToAsync(ms); } } f.Payload = ms.ToArray(); f.PayloadLength = (ulong)f.Payload.Length; } } catch (Exception ex) { Console.WriteLine(ex.ToString()); } } }
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (request.Method == HttpMethod.Post) { bool isGzip = request.Content.Headers.ContentEncoding.Contains("GZip"); bool isDeflate = !isGzip && request.Content.Headers.ContentEncoding.Contains("Deflate"); if (isGzip || isDeflate) { Stream decompressedStream = new MemoryStream(); if (isGzip) { using (var gzipStream = new GZipStream(await request.Content.ReadAsStreamAsync(), CompressionMode.Decompress)) { await gzipStream.CopyToAsync(decompressedStream); } } else if (isDeflate) { using (var gzipStream = new DeflateStream(await request.Content.ReadAsStreamAsync(), CompressionMode.Decompress)) { await gzipStream.CopyToAsync(decompressedStream); } } decompressedStream.Seek(0, SeekOrigin.Begin); var originContent = request.Content; request.Content = new StreamContent(decompressedStream); foreach (var header in originContent.Headers) { request.Content.Headers.Add(header.Key, header.Value); } } } return(await base.SendAsync(request, cancellationToken)); }
static async Task Decompress(byte[] bts, string dir, NItem item) { if (item.IsCompressed == 0 || (item.Unk4 == item.Unk5 && item.OriginalLength == item.CompressedLength)) { //Console.WriteLine($"Unpack {item.Id:X8}"); var header = bts.Take(32).ToArray(); var fileName = InferExtension(header, item, out var category); Console.WriteLine($"Unpack: {fileName}"); var path = Path.Combine(dir, category); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } using (var fs = File.Create(Path.Combine(path, fileName))) { await fs.WriteAsync(bts, 0, bts.Length); } } else { //Console.WriteLine($"Decompress: {item.Id:X8}"); using (var output = new MemoryStream((int)item.OriginalLength)) { try { using (var ms = new MemoryStream(bts, 2, bts.Length - 2)) using (var zs = new DeflateStream(ms, CompressionMode.Decompress, false)) { await zs.CopyToAsync(output); output.Position = 0; } } catch (InvalidDataException e) { Console.WriteLine($"Decompress {item.Id:X8}({item.Offset}) failed."); } var header = new byte[32]; await output.ReadAsync(header, 0, header.Length); var fileName = InferExtension(header, item, out var category); Console.WriteLine($"Decompress: {fileName}"); var path = Path.Combine(dir, category); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); } output.Position = 0; using (var fs = File.Create(Path.Combine(path, fileName))) { await output.CopyToAsync(fs); } } } }
/// <inheritdoc/> public async Task <uint> DetransformChunkAsync( Stream input, Stream output, NefsDataChunk chunk, uint maxOutputSize, NefsProgress p) { using (var detransformedStream = new MemoryStream()) { // Copy chunk to temp stream await input.CopyPartialAsync(detransformedStream, chunk.Size, p.CancellationToken); detransformedStream.Seek(0, SeekOrigin.Begin); // Decrypt if (chunk.Transform.IsAesEncrypted) { using (var aesManager = this.CreateAesManager(chunk.Transform.Aes256Key)) using (var cryptoStream = new CryptoStream(detransformedStream, aesManager.CreateDecryptor(), CryptoStreamMode.Read, leaveOpen: true)) using (var tempStream = new MemoryStream()) { await cryptoStream.CopyToAsync(tempStream, p.CancellationToken); tempStream.Seek(0, SeekOrigin.Begin); detransformedStream.Seek(0, SeekOrigin.Begin); await tempStream.CopyToAsync(detransformedStream, p.CancellationToken); detransformedStream.Seek(0, SeekOrigin.Begin); detransformedStream.SetLength(tempStream.Length); } } // Decompress if (chunk.Transform.IsZlibCompressed) { using (var inflater = new DeflateStream(detransformedStream, CompressionMode.Decompress, leaveOpen: true)) using (var tempStream = new MemoryStream()) { await inflater.CopyToAsync(tempStream, p.CancellationToken); tempStream.Seek(0, SeekOrigin.Begin); detransformedStream.Seek(0, SeekOrigin.Begin); await tempStream.CopyToAsync(detransformedStream, p.CancellationToken); detransformedStream.Seek(0, SeekOrigin.Begin); detransformedStream.SetLength(tempStream.Length); } } // Copy detransformed chunk to output stream var chunkSize = Math.Min(detransformedStream.Length, maxOutputSize); await detransformedStream.CopyPartialAsync(output, chunkSize, p.CancellationToken); return((uint)chunkSize); } }
private static async Task <MemoryStream> DecompressAsync(this Stream stream) { stream.Position = 0; using var decompressionStream = new DeflateStream(stream, CompressionMode.Decompress); var memoryStream = new MemoryStream(); await decompressionStream.CopyToAsync(memoryStream); return(memoryStream); }
/// <inheritdoc/> protected override async Task BaseDecompressAsync(Stream inputStream, Stream outputStream, CancellationToken cancellationToken = default) { using var deflateStream = new DeflateStream(inputStream, CompressionMode.Decompress, true); await deflateStream.CopyToAsync(outputStream, DefaultBufferSize, cancellationToken).ConfigureAwait(false); await outputStream.FlushAsync(cancellationToken).ConfigureAwait(false); await deflateStream.FlushAsync(cancellationToken).ConfigureAwait(false); }
// Compresses or decompresses a stream using the specified compression method. public static async Task <MemoryStream> CompressAsync( this Stream @this, CompressionMethod method, bool compress, CancellationToken cancellationToken) { @this.Position = 0; var targetStream = new MemoryStream(); switch (method) { case CompressionMethod.Deflate: if (compress) { using var compressor = new DeflateStream(targetStream, CompressionMode.Compress, true); await @this.CopyToAsync(compressor, 1024, cancellationToken).ConfigureAwait(false); await @this.CopyToAsync(compressor).ConfigureAwait(false); // WebSocket use this targetStream.Write(LastByte, 0, 1); targetStream.Position = 0; } else { using var compressor = new DeflateStream(@this, CompressionMode.Decompress); await compressor.CopyToAsync(targetStream).ConfigureAwait(false); } break; case CompressionMethod.Gzip: if (compress) { using var compressor = new GZipStream(targetStream, CompressionMode.Compress, true); await @this.CopyToAsync(compressor).ConfigureAwait(false); } else { using var compressor = new GZipStream(@this, CompressionMode.Decompress); await compressor.CopyToAsync(targetStream).ConfigureAwait(false); } break; case CompressionMethod.None: await @this.CopyToAsync(targetStream).ConfigureAwait(false); break; default: throw new ArgumentOutOfRangeException(nameof(method), method, null); } return(targetStream); }
public async Task CopyToAsync_Roundtrip_OutputMatchesInput(byte[] expectedDecrypted, int copyBufferSize, Stream source) { var m = new MemoryStream(); using (DeflateStream ds = new DeflateStream(source, CompressionMode.Decompress)) { await ds.CopyToAsync(m); } Assert.Equal(expectedDecrypted, m.ToArray()); }
async public static Task <byte[]> DecompressAsync(byte[] data) { using (MemoryStream outputStream = new MemoryStream()) using (MemoryStream inputStream = new MemoryStream(data)) using (DeflateStream deflateStream = new DeflateStream(inputStream, CompressionMode.Decompress)) { await deflateStream.CopyToAsync(outputStream); return(outputStream.ToArray()); } }
/// <summary> /// Decompresses a byte array that has been compressed using DEFLATE compression. Does not block threads while waiting for IO to complete. /// </summary> public static async Task <byte[]> DecompressDeflateAsync(this byte[] b) { using (var source = new MemoryStream(b)) using (var destination = new MemoryStream()) { using (var decompressor = new DeflateStream(source, CompressionMode.Decompress)) { await decompressor.CopyToAsync(destination); } return(destination.ToArray()); } }
/// <inheritdoc/> public async Task DecompressChunkAsync( byte[] chunk, Stream output, NefsProgress p) { // Copy the comrpessed chunk to a memory stream and decompress (inflate) it using (var ms = new MemoryStream(chunk)) using (var inflater = new DeflateStream(ms, CompressionMode.Decompress)) { await inflater.CopyToAsync(output, StreamExtensions.CopyBufferSize, p.CancellationToken); } }
public static async Task DecompressToFileAsync(string filePath, string decompressedFilePath) { var file = File.Open(filePath, FileMode.Open); using (var decompressedFileStream = File.Create(decompressedFilePath)) { using (var decompressionStream = new DeflateStream(file, CompressionMode.Decompress)) { await decompressionStream.CopyToAsync(decompressedFileStream); } } }
/// <inheritdoc/> public async Task <Stream> DecompressAsync(Stream stream, int bufferSize, CancellationToken cancellationToken) { var decompressedStream = new MemoryStream(); using (var decompressionStream = new DeflateStream(stream, CompressionMode.Decompress)) { await decompressionStream.CopyToAsync(decompressedStream, bufferSize, cancellationToken).ConfigureAwait(false); await decompressionStream.FlushAsync().ConfigureAwait(false); } return(decompressedStream); }
protected override async Task <HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (request.Method == HttpMethod.Post) { var sourceContent = request.Content; var encodings = sourceContent.Headers.ContentEncoding; var isGzip = encodings.Contains("gzip"); var isDeflate = !isGzip && encodings.Contains("deflate"); if (isGzip || isDeflate) { var compressStream = await sourceContent.ReadAsStreamAsync(); var decompressStream = new MemoryStream(); if (isGzip) { using (var gzipStream = new GZipStream(compressStream, CompressionMode.Decompress, CompressionLevel.BestCompression, true)) { await gzipStream.CopyToAsync(decompressStream); } } else if (isDeflate) { using (var gzipStream = new DeflateStream(compressStream, CompressionMode.Decompress, CompressionLevel.BestCompression, true)) { await gzipStream.CopyToAsync(decompressStream); } } decompressStream.Seek(0, SeekOrigin.Begin); var targetContent = new StreamContent(decompressStream); foreach (var header in sourceContent.Headers) { targetContent.Headers.Add(header.Key, header.Value); } request.Content = targetContent; } } return(await base.SendAsync(request, cancellationToken)); }
public async Task InvalidateAsync() { if (!_environment.IsDevelopment()) { File.Delete(GetDatabasePath()); File.Delete(GetSuffixTreePath()); } var azureConnectionString = _configuration["AzureStorageConnectionString"]; var databasePath = GetDatabasePath(); if (!File.Exists(databasePath)) { var blobClient = new BlobClient(azureConnectionString, "catalog", "apicatalog.dat"); await blobClient.DownloadToAsync(databasePath); } var catalog = await ApiCatalogModel.LoadAsync(databasePath); var availabilityContext = ApiAvailabilityContext.Create(catalog); var apiByGuid = catalog.GetAllApis().ToDictionary(a => a.Guid); var suffixTreePath = GetSuffixTreePath(); if (!File.Exists(suffixTreePath)) { // TODO: Ideally the underlying file format uses compression. This seems weird. var blobClient = new BlobClient(azureConnectionString, "catalog", "suffixtree.dat.deflate"); using var blobStream = await blobClient.OpenReadAsync(); using var deflateStream = new DeflateStream(blobStream, CompressionMode.Decompress); using var fileStream = File.Create(suffixTreePath); await deflateStream.CopyToAsync(fileStream); } var suffixTree = SuffixTree.Load(suffixTreePath); var jobBlobClient = new BlobClient(azureConnectionString, "catalog", "job.json"); using var jobStream = await jobBlobClient.OpenReadAsync(); var jobInfo = await JsonSerializer.DeserializeAsync <CatalogJobInfo>(jobStream); _catalog = catalog; _availabilityContext = availabilityContext; _statistics = catalog.GetStatistics(); _apiByGuid = apiByGuid; _suffixTree = suffixTree; _jobInfo = jobInfo; }
/// <summary> /// Parses the message received by the WebSocket. /// </summary> /// <param name="messageType">The type of message received.</param> /// <param name="ms">The stream containing the actual message.</param> /// <exception cref="ArgumentException">Thrown if message type is 'Close' or an unknown type.</exception> /// <exception cref="IOException">Thrown if the binary message could not be decompressed.</exception> async Task <string> ParseMessage(WebSocketMessageType messageType, MemoryStream ms) { string StreamToString(MemoryStream decompressedMemoryStream) { ArraySegment <byte> buffer; if (!decompressedMemoryStream.TryGetBuffer(out buffer)) { // The memory stream should be 'exposable' but as a fallback just write the stream to an array. buffer = new ArraySegment <byte>(decompressedMemoryStream.ToArray()); } return(Encoding.UTF8.GetString(buffer.Array, buffer.Offset, buffer.Count)); } if (messageType == WebSocketMessageType.Text) { // Message is already decompressed. return(StreamToString(ms)); } else if (messageType == WebSocketMessageType.Binary) { using (MemoryStream decompressed = new MemoryStream()) { try { // Skip first two bytes ms.Seek(2, SeekOrigin.Begin); // Decompress message using (DeflateStream deflateStream = new DeflateStream(ms, CompressionMode.Decompress, true)) await deflateStream.CopyToAsync(decompressed).ConfigureAwait(false); } catch (Exception ex) { throw new IOException("Failed to decompress binary message.", ex); } // Message is now compressed, return as string. return(StreamToString(decompressed)); } } else if (messageType == WebSocketMessageType.Close) { throw new ArgumentException("Message type cannot be \"Close\"", nameof(messageType)); } else { throw new ArgumentException($"Unknown message type: \"{messageType}\"", nameof(messageType)); } }
public async Task <Stream> DecompressAsync(byte[] data) { var outputStream = new MemoryStream(); using (var compressedStream = new MemoryStream(data)) { using (var deflateStream = new DeflateStream(compressedStream, CompressionMode.Decompress)) { await deflateStream.CopyToAsync(outputStream); } } outputStream.Position = 0; return(outputStream); }
protected virtual async Task <byte[]> GetDecompressedData(byte[] data, CancellationToken cancellationToken) { var dataStream = new MemoryStream(data); using (var compressionStream = new DeflateStream(dataStream, CompressionMode.Decompress)) { using (var outputStream = new MemoryStream()) { await compressionStream.CopyToAsync(outputStream, _copyBuffer, cancellationToken); return(outputStream.ToArray()); } } }
async public static Task <byte[]> DecryptWithDecompressAsync(byte[] data, byte[] key, byte[] iv) { using (MemoryStream outputStream = new MemoryStream()) using (MemoryStream inputStream = new MemoryStream(data)) using (Aes algorithm = Aes.Create()) using (ICryptoTransform decryptor = algorithm.CreateDecryptor(key, iv)) using (CryptoStream cryptoStream = new CryptoStream(inputStream, decryptor, CryptoStreamMode.Read)) using (DeflateStream deflateStream = new DeflateStream(cryptoStream, CompressionMode.Decompress)) { await deflateStream.CopyToAsync(outputStream); return(outputStream.ToArray()); } }
public static async Task <byte[]> DecompressAsync(this byte[] data) { using (MemoryStream output = new MemoryStream(data)) { using (DeflateStream compressionStream = new DeflateStream(output, CompressionMode.Decompress)) { await compressionStream.CopyToAsync(output); compressionStream.Close(); } return(output.ToArray()); } }
public async Task <byte[]> DecompressAsync(ReadOnlyMemory <byte> data) { using var uncompressedStream = new MemoryStream(); using (var compressedStream = new MemoryStream(data.ToArray())) using (var deflateStream = new DeflateStream(compressedStream, CompressionMode.Decompress, false)) { await deflateStream .CopyToAsync(uncompressedStream) .ConfigureAwait(false); } return(uncompressedStream.ToArray()); }
public void CopyToAsyncArgumentValidation() { using (DeflateStream ds = new DeflateStream(new MemoryStream(), CompressionMode.Decompress)) { Assert.Throws<ArgumentNullException>("destination", () => { ds.CopyToAsync(null); }); Assert.Throws<ArgumentOutOfRangeException>("bufferSize", () => { ds.CopyToAsync(new MemoryStream(), 0); }); Assert.Throws<NotSupportedException>(() => { ds.CopyToAsync(new MemoryStream(new byte[1], writable: false)); }); ds.Dispose(); Assert.Throws<ObjectDisposedException>(() => { ds.CopyToAsync(new MemoryStream()); }); } using (DeflateStream ds = new DeflateStream(new MemoryStream(), CompressionMode.Compress)) { Assert.Throws<NotSupportedException>(() => { ds.CopyToAsync(new MemoryStream()); }); } }