public void TestZipUnzipRandomSucceeds() { //// ARRANGE var source = new byte[992]; var r = new Random(42); r.NextBytes(source); //// ACT var deflatedSource = DeflateHelper.Zip(source); Assert.AreEqual(997, deflatedSource.Length); var unzipped = DeflateHelper.Unzip(deflatedSource); //// ASSERT Assert.AreEqual(992, unzipped.Length); for (int i = 0; i < 992; i++) { Assert.AreEqual(source[i], unzipped[i]); } }
public void TestZipUnzipSucceeds() { //// ARRANGE var source = new byte[992]; for (int i = 0; i < source.Length; i++) { source[i] = 42; } //// ACT var deflatedSource = DeflateHelper.Zip(source); Assert.AreEqual(11, deflatedSource.Length); var unzipped = DeflateHelper.Unzip(deflatedSource); //// ASSERT Assert.AreEqual(992, unzipped.Length); for (int i = 0; i < 992; i++) { Assert.AreEqual(source[i], unzipped[i]); } }
/// <inheritdoc/> public async Task <UInt32> TransformChunkAsync( Stream input, UInt32 inputChunkSize, Stream output, NefsDataTransform transform, NefsProgress p) { using (var transformedStream = new MemoryStream()) { // Copy raw chunk to temp stream await input.CopyPartialAsync(transformedStream, inputChunkSize, p.CancellationToken); transformedStream.Seek(0, SeekOrigin.Begin); // Compress if (transform.IsZlibCompressed) { using (var tempStream = new MemoryStream()) { await DeflateHelper.DeflateAsync(transformedStream, (int)inputChunkSize, tempStream); tempStream.Seek(0, SeekOrigin.Begin); transformedStream.Seek(0, SeekOrigin.Begin); await tempStream.CopyPartialAsync(transformedStream, tempStream.Length, p.CancellationToken); transformedStream.Seek(0, SeekOrigin.Begin); transformedStream.SetLength(tempStream.Length); } } // Encrypt if (transform.IsAesEncrypted) { using (var aesManager = this.CreateAesManager(transform.Aes256Key)) using (var cryptoStream = new CryptoStream(transformedStream, aesManager.CreateEncryptor(), CryptoStreamMode.Read, leaveOpen: true)) using (var tempStream = new MemoryStream()) { await cryptoStream.CopyToAsync(tempStream, p.CancellationToken); tempStream.Seek(0, SeekOrigin.Begin); transformedStream.Seek(0, SeekOrigin.Begin); await tempStream.CopyPartialAsync(transformedStream, tempStream.Length, p.CancellationToken); transformedStream.Seek(0, SeekOrigin.Begin); transformedStream.SetLength(tempStream.Length); } } // Copy transformed chunk to output stream await transformedStream.CopyToAsync(output, p.CancellationToken); // Return size of transformed chunk return((uint)transformedStream.Length); } }
/// <inheritdoc/> public async Task <NefsItemSize> CompressAsync( Stream input, Int64 inputOffset, UInt32 inputLength, Stream output, Int64 outputOffset, UInt32 chunkSize, NefsProgress p) { var chunkSizes = new List <UInt32>(); input.Seek(inputOffset, SeekOrigin.Begin); output.Seek(outputOffset, SeekOrigin.Begin); // Split file into chunks and compress them using (var t = p.BeginTask(1.0f, $"Compressing stream")) { var lastChunkSize = 0; var totalChunkSize = 0; var lastBytesRead = 0; var bytesRemaining = (int)inputLength; // Determine how many chunks to split file into var numChunks = (int)Math.Ceiling(inputLength / (double)chunkSize); for (var i = 0; i < numChunks; ++i) { using (var st = p.BeginSubTask(1.0f / numChunks, $"Compressing chunk {i + 1}/{numChunks}")) { var nextBytes = Math.Min(chunkSize, bytesRemaining); // Compress this chunk and write it to the output file (lastBytesRead, lastChunkSize) = await DeflateHelper.DeflateAsync(input, (int)nextBytes, output, p.CancellationToken); totalChunkSize += lastChunkSize; bytesRemaining -= lastBytesRead; // Record the total compressed size after this chunk chunkSizes.Add((UInt32)totalChunkSize); } } } // Return item size return(new NefsItemSize(inputLength, chunkSizes)); }
/// <summary> /// Compresses the specified input file and updates the archive /// header entries. The item is marked as pending injection /// so the comrpessed data is injected when the archive is saved. /// </summary> /// <param name="inputFilePath">Path to the uncompressed file to compress and inject.</param> public void Inject(string inputFilePath, NefsProgressInfo p) { float taskWeightPrep = 0.05f; float taskWeightCompress = 0.85f; float taskWeightUpdateMetadata = 0.05f; float taskWeightCleanup = 0.05f; p.BeginTask(taskWeightPrep, "Preparing file injection..."); /* Prepare the temporary working directory */ if (!Directory.Exists(FilePathHelper.TempDirectory)) { Directory.CreateDirectory(FilePathHelper.TempDirectory); } /* * Create a temporary directory to compress input file in */ /* Temp working directory */ var workDir = Path.Combine(FilePathHelper.TempDirectory, _archive.FilePathHash, FilePathInArchiveHash); /* Delete the working directory if exists and recreate it */ if (Directory.Exists(workDir)) { Directory.Delete(workDir, true); } Directory.CreateDirectory(workDir); p.EndTask(); /* * SPLIT INPUT FILE INTO CHUNKS AND COMPRESS THEM */ p.BeginTask(taskWeightCompress, "Compressing file..."); int compressedSizeDiff = 0; int currentChunk = 0; var destFilePath = Path.Combine(workDir, "inject.dat"); int numChunks = 0; int numChunksDiff = 0; int oldNumChunks = ChunkSizes.Count(); /* Open the input file */ using (var inputFile = new FileStream(inputFilePath, FileMode.Open)) { inputFile.Seek(0, SeekOrigin.Begin); /* Determine how many chunks to split file into */ numChunks = (int)Math.Ceiling(inputFile.Length / (double)CHUNK_SIZE); _extractedSize = (UInt32)inputFile.Length; /* Clear out chunk sizes list so we can rebuild it */ ChunkSizes.Clear(); int lastBytesRead = 0; int totalBytesRead = 0; int lastChunkSize = 0; int totalChunkSize = 0; using (var outputFile = new FileStream(destFilePath, FileMode.Create)) { do { p.BeginTask(1.0f / (float)numChunks, String.Format("Compressing chunk {0}/{1}...", currentChunk + 1, numChunks)); /* Compress this chunk and write it to the output file */ lastBytesRead = DeflateHelper.DeflateToFile(inputFile, CHUNK_SIZE, outputFile, out lastChunkSize); totalBytesRead += lastBytesRead; totalChunkSize += lastChunkSize; currentChunk++; /* Record the total compressed size after this chunk */ ChunkSizes.Add((uint)totalChunkSize); p.EndTask(); } while (lastBytesRead == CHUNK_SIZE); /* Get difference in number of chunks compared to item we replaced */ numChunksDiff = numChunks - oldNumChunks; /* Get the difference in compressed size */ compressedSizeDiff = (int)outputFile.Length - (int)_compressedSize; /* Update new compressed size */ _compressedSize = (uint)outputFile.Length; /* Quick sanity check */ if (_compressedSize != totalChunkSize) { log.Error("Compressed file size different than what was expected."); } } } /* Quick sanity check */ if (currentChunk != numChunks) { log.Error("Did not create the expected number of chunks."); } p.EndTask(); /* * UPDATE METADATA FOR ITEMS AFTER THIS ONE */ p.BeginTask(taskWeightUpdateMetadata, "Updating archive metadata..."); /* Update data offsets for each item AFTER this one */ for (int i = 0; i < Archive.Items.Count; i++) { var item = Archive.Items[i]; /* Directories don't have data offsets, skip them */ if (item.Type == NefsItemType.Directory) { continue; } /* Update an item's data offset if it was after the item we are updating now */ if (item.DataOffset > this.DataOffset) { /* Update the data offset */ UInt64 prevOffset = item.DataOffset; UInt64 newOffset = (ulong)((long)prevOffset + (long)compressedSizeDiff); if (newOffset < 0) { throw new Exception(String.Format( "New data offset less than zero. [file={0}]", item.Filename)); } item.DataOffset = newOffset; } /* Skip the weird 0xFFFFFFFF offsets into pt 4 */ if (item.OffsetIntoPt4Raw == 0xFFFFFFFF) { continue; } /* Update an item's part 4 offset if it was after the item we are updating now */ if (item.OffsetIntoPt4Raw > this.OffsetIntoPt4Raw) { /* Update the header part 4 offset */ int prevOffsetIntoPt4Raw = (int)item.OffsetIntoPt4Raw; int newOffsetIntoPt4Raw = prevOffsetIntoPt4Raw + numChunksDiff; if (newOffsetIntoPt4Raw < 0) { throw new Exception(String.Format( "New offset into part 4 less than zero. [file={0}]", item.Filename)); } item.OffsetIntoPt4Raw = (UInt32)newOffsetIntoPt4Raw; } } p.EndTask(); /* * CLEANUP */ p.BeginTask(taskWeightCleanup, "Cleaning up..."); /* Whenever the archive is saved, this data will be injected */ _pendingInjection = true; _fileToInject = destFilePath; _archive.Modified = true; p.EndTask(); }
/// <summary> /// This method is called whenever the module is sent a message from the EdgeHub. /// It just pipe the messages without any change. /// It prints all the incoming messages. /// </summary> private static async Task <MessageResponse> PipeMessageInputOne(Message message, object userContext) { int counterValue = Interlocked.Increment(ref counter); var moduleClient = userContext as ModuleClient; if (moduleClient == null) { throw new InvalidOperationException("UserContext doesn't contain " + "expected values"); } var connectionDeviceId = message.ConnectionDeviceId; var connectionModuleId = message.ConnectionModuleId; byte[] messageBytes = message.GetBytes(); if (message.ContentEncoding == "gzip" && message.ContentType == "application/zip") { var zippedLength = messageBytes.Length; messageBytes = GZipHelper.Unzip(messageBytes); System.Console.WriteLine($"Uncompressed from GZIP {zippedLength} bytes to {messageBytes.Length} bytes"); } if (message.ContentEncoding == "deflate" && message.ContentType == "application/zip") { var zippedLength = messageBytes.Length; messageBytes = DeflateHelper.Unzip(messageBytes); System.Console.WriteLine($"Uncompressed from Deflate {zippedLength} bytes to {messageBytes.Length} bytes"); } string messageString = Encoding.UTF8.GetString(messageBytes); Console.WriteLine($"-> Received echo message: {counterValue}, Body: '{messageString}'"); if (!string.IsNullOrEmpty(messageString)) { var messageBody = JsonConvert.DeserializeObject(messageString); var moduleOutput = _moduleOutputs.GetModuleOutput("output1"); if (moduleOutput != null) { moduleOutput.Properties.Clear(); foreach (var prop in message.Properties) { moduleOutput.Properties.Add(prop.Key, prop.Value); Console.WriteLine($"Property added: key:'{prop.Key}' value:'{prop.Value}'"); } if (!string.IsNullOrEmpty(connectionDeviceId)) { Console.WriteLine($"connectionDeviceId: '{connectionDeviceId}'"); } if (!string.IsNullOrEmpty(connectionModuleId)) { Console.WriteLine($"ConnectionModuleId: '{connectionModuleId}'"); } await moduleOutput.SendMessage(messageBody); Console.WriteLine("Received message echoed"); } } return(MessageResponse.Completed); }