Beispiel #1
0
 public LzhamStream(Stream stream, CompressionParameters mode, bool leaveOpen)
 {
     if (stream == null)
         throw new ArgumentNullException(nameof(stream));
     if (!stream.CanWrite)
         throw new ArgumentException("The base stream is not writeable", nameof(stream));
     _stream = stream;
     _leaveOpen = leaveOpen;
     _buffer = new byte[DefaultBufferSize];
     _compressionHandle = LzhamInterop.CompressInit(mode);
     if (_compressionHandle.IsInvalid)
     {
         throw new ApplicationException("Could not initialize compression stream with specified parameters");
     }
 }
        public CompressionParameters GetCompressionParameters()
        {
            CompressionParameters result = new CompressionParameters
            {
                DictionarySize = DictionarySize,
                Level = Level,
                UpdateRate = UpdateRate,
                HelperThreads = HelperThreads
            };

            result.Flags |= Zlib ? CompressionFlag.WriteZlibStream : 0;
            result.Flags |= Tradeoff ? CompressionFlag.TradeIffDecompressionForCompressionRatio : 0;
            result.Flags |= DeterministicParsing ? CompressionFlag.DeterministicParsing : 0;
            result.Flags |= ExtremeParsing ? CompressionFlag.ExtremeParsing : 0;
            return result;
        }
Beispiel #3
0
 public LzhamStream(Stream stream, CompressionParameters mode) : this(stream, mode, false)
 {
 }
Beispiel #4
0
        private static int Repack(RepackOptions options)
        {
            if (options.Level < 0 || options.Level > CompressionLevel.Uber)
            {
                Console.WriteLine("Compresson level should be between 0 and 4 (inclusive)");
                return 9;
            }
            CompressionLevel level = options.Level;
            string inputFolder = options.InputFolder ?? Environment.CurrentDirectory;
            File.Delete(options.OutputFile);
            using (FileStream fs = File.OpenWrite(options.OutputFile))
            {
                int headerSize = Marshal.SizeOf(typeof (TimHeader));
                byte[] headerBytes = new byte[headerSize];
                fs.Write(headerBytes,0, headerSize);
                Packer packer = new Packer(inputFolder,fs, level);
                Packer.PackerTables tables = packer.Pack();
                MD5 md5 = MD5.Create();
                TimHeader header = new TimHeader
                {
                    headerSize = (uint) headerSize,
                    version = TimHeader.currentVersion,
                    id = TimHeader.signature,
                    directoryListingDigest = md5.ComputeHash(tables.Directory),
                    fileRegistryDigest = md5.ComputeHash(tables.FileRegistry),
                    bucketTableDigest = md5.ComputeHash(tables.BucketTable),
                    hashtableDigest = md5.ComputeHash(tables.HashTable),
                    directoryListngSize = (uint) tables.Directory.Length,
                    fileCount = (uint) tables.FileCount
                };

                CompressionParameters p = new CompressionParameters
                {
                    DictionarySize = Util.GetDictLength((uint)tables.Directory.Length),
                    UpdateRate = TableUpdateRate.Fastest,
                    Level = level
                };

                byte[] outputBufer = new byte[Math.Max(tables.Directory.Length*2,128)];
                uint addler = 0;
                int outsize = outputBufer.Length;
                CompressStatus status = Lzham.CompressMemory(p, tables.Directory, tables.Directory.Length, 0, outputBufer, ref outsize, 0, ref addler);

                if (status != CompressStatus.Success)
                {
                    Console.WriteLine("Failed to pack directory listing. ({status})");
                    Environment.FailFast("Failed to pack directory listing. ({status})");
                }

                header.directoryListingCompressedSize = (uint)outsize;
                header.directoryListingOffset = (ulong)fs.Position;
                fs.Write(outputBufer,0, outsize);

                p = new CompressionParameters
                {
                    DictionarySize = Util.GetDictLength((uint)tables.FileRegistry.Length),
                    UpdateRate = TableUpdateRate.Fastest,
                    Level = level
                };

                outputBufer = new byte[Math.Max(tables.FileRegistry.Length*2,128)];
                outsize = outputBufer.Length;
                status = Lzham.CompressMemory(p, tables.FileRegistry, tables.FileRegistry.Length, 0, outputBufer, ref outsize, 0, ref addler);

                if (status != CompressStatus.Success)
                {
                    Console.WriteLine("Failed to pack file registry. ({status})");
                    Environment.FailFast("Failed to pack file registry. ({status})");
                }

                header.fileRegistryCompressedSize = (uint)outsize;
                header.fileRegistryOffset = (ulong)fs.Position;
                fs.Write(outputBufer, 0, outsize);

                p = new CompressionParameters
                {
                    DictionarySize = Util.GetDictLength((uint)tables.HashTable.Length),
                    UpdateRate = TableUpdateRate.Fastest,
                    Level = level
                };

                outputBufer = new byte[Math.Max(tables.HashTable.Length*2,128)];
                outsize = outputBufer.Length;
                status = Lzham.CompressMemory(p, tables.HashTable, tables.HashTable.Length, 0, outputBufer, ref outsize, 0, ref addler);

                if (status != CompressStatus.Success)
                {
                    Console.WriteLine("Failed to pack hash table. ({status})");
                    Environment.FailFast("Failed to pack hash table. ({status})");
                }

                header.hashtableCompressedSize = (uint)outsize;
                header.hashtableOffset = (ulong)fs.Position;
                fs.Write(outputBufer, 0, outsize);

                p = new CompressionParameters
                {
                    DictionarySize = Util.GetDictLength((uint)tables.BucketTable.Length),
                    UpdateRate = TableUpdateRate.Fastest,
                    Level = level
                };

                outputBufer = new byte[Math.Max(tables.BucketTable.Length*2,128)];
                outsize = outputBufer.Length;
                status = Lzham.CompressMemory(p, tables.BucketTable, tables.BucketTable.Length, 0, outputBufer, ref outsize, 0, ref addler);

                if (status != CompressStatus.Success)
                {
                    Console.WriteLine("Failed to pack bucket table. ({status})");
                    Environment.FailFast("Failed to pack bucket table. ({status})");
                }

                header.bucketTableCompressedSize = (uint)outsize;
                header.bucketTableOffset = (ulong)fs.Position;
                fs.Write(outputBufer, 0, outsize);

                fs.Seek(0, SeekOrigin.Begin);

                headerBytes = Util.StructureToByteArray(header);
                fs.Write(headerBytes,0, headerSize);
            }
            return 0;
        }
Beispiel #5
0
 public static CompressStatus CompressMemory(CompressionParameters parameters, byte[] inBuf, int inBufSize, int inBufOffset, byte[] outBuf, ref int outBufSize, int outBufOffset, ref uint adler32)
 {
     return LzhamInterop.CompressMemory(parameters, outBuf, ref outBufSize, outBufOffset, inBuf, inBufSize, inBufOffset, ref adler32);
 }
Beispiel #6
0
 public static unsafe CompressStatus CompressMemory(CompressionParameters parameters, byte[] outBuf, ref int outBufSize, int outBufOffset, byte[] inBuf, int inBufSize, int inBufOffset, ref uint adler32)
 {
     if (outBufOffset + outBufSize > outBuf.Length)
     {
         throw new ArgumentException("Offset plus count is larger than the length of array", nameof(outBuf));
     }
     if (inBufOffset + inBufSize > inBuf.Length)
     {
         throw new ArgumentException("Offset plus count is larger than the length of array", nameof(inBuf));
     }
     CompressionParametersInternal p;
     p.m_struct_size = (uint)sizeof(CompressionParametersInternal);
     p.m_compress_flags = parameters.Flags;
     p.m_dict_size_log2 = parameters.DictionarySize;
     p.m_level = parameters.Level;
     p.m_max_helper_threads = parameters.HelperThreads;
     p.m_table_max_update_interval = parameters.MaxUpdateInterval;
     p.m_table_update_interval_slow_rate = parameters.UpdateIntervalSlowRate;
     p.m_table_update_rate = parameters.UpdateRate;
     if (parameters.SeedBytes != null)
     {
         p.m_num_seed_bytes = (uint) parameters.SeedBytes.Length;
     }
     p.m_struct_size = (uint)sizeof(CompressionParametersInternal);
     fixed (byte* seedBytes = parameters.SeedBytes)
     fixed (byte* outBytes = outBuf)
     fixed (byte* inBytes = inBuf)
     {
         p.m_pSeed_bytes = seedBytes;
         byte* pBytes = (byte*)&p;
         IntPtr outSize = new IntPtr(outBufSize);
         CompressStatus result = (CompressStatus)lzham_compress_memory(pBytes, outBytes+outBufOffset, ref outSize, inBytes+inBufOffset,  inBufSize, ref adler32);
         outBufSize = outSize.ToInt32();
         return result;
     }
 }
Beispiel #7
0
 public static unsafe CompressionHandle CompressInit(CompressionParameters parameters)
 {
     CompressionParametersInternal p;
     p.m_struct_size = (uint)sizeof(CompressionParametersInternal);
     p.m_compress_flags = parameters.Flags;
     p.m_dict_size_log2 = parameters.DictionarySize;
     p.m_level = parameters.Level;
     p.m_max_helper_threads = parameters.HelperThreads;
     p.m_table_max_update_interval = parameters.MaxUpdateInterval;
     p.m_table_update_interval_slow_rate = parameters.UpdateIntervalSlowRate;
     p.m_table_update_rate = parameters.UpdateRate;
     if (parameters.SeedBytes != null)
     {
         p.m_num_seed_bytes = (uint) parameters.SeedBytes.Length;
     }
     fixed (byte* seedBytes = parameters.SeedBytes)
     {
         p.m_pSeed_bytes = seedBytes;
         byte* pBytes = (byte*)&p;
         return lzham_compress_init(pBytes);
     }
 }
Beispiel #8
0
        private ushort PackFolder(string path)
        {
            long pos = _directory.Position;
            DirectoryListingEntryHeader dleh = new DirectoryListingEntryHeader();
            string[] folders = Directory.GetDirectories(path);
            string[] files = Directory.GetFiles(path);
            dleh.numDirectories = (ushort)folders.Length;
            dleh.numFiles = (ushort)files.Length;
            string dlehName = Path.GetFileName(path.Substring(_pathPrefixLength));
            byte[] folderNameBytes = Encoding.UTF8.GetBytes(dlehName);
            ushort headerSize = (ushort)(Marshal.SizeOf(dleh) + folderNameBytes.Length);
            dleh.nameLength = (ushort)(folderNameBytes.Length);
            _directory.Position += headerSize;
            ushort foldersSize = Directory.GetDirectories(path).Aggregate<string, ushort>(0, (current, folder) => (ushort)(current + PackFolder(folder)));

            ushort filesSize = 0;
            foreach (string file in files)
            {
                Console.WriteLine(file);
                string shortName = file.Substring(_pathPrefixLength).Replace("\\","/");
                byte[] nameBytes = Encoding.UTF8.GetBytes(shortName);
                Bucket bucket = new Bucket
                {
                    Index = _fileCounter++,
                    IndexHash = Util.Murmur2(nameBytes, 0),
                    BucketHash = Util.Murmur2(nameBytes, BitConverter.ToUInt32(Encoding.UTF8.GetBytes("AMIT"), 0))
                };
                _buckets.AddOrUpdate(bucket.IndexHash, (new[] {bucket}).ToList(), (hash, list) =>
                {
                    list.Add(bucket);
                    return list;
                });
                uint fileSize = (uint)(new FileInfo(file)).Length;
                FileRegistryEntry fre = new FileRegistryEntry
                {
                    uncompressed = fileSize,
                    fileOffset = (ulong) _fs.Position
                };
                byte[] fileData = File.ReadAllBytes(file);
                CompressionParameters p = new CompressionParameters
                {
                    DictionarySize = Util.GetDictLength(fileSize),
                    UpdateRate = TableUpdateRate.Fastest,
                    Level = _level
                };
                byte[] compressedData = new byte[fileSize];
                int compressedSize = (int)fileSize;
                uint addler = 0;

                //Yikes!!!!
                GC.Collect();
                GC.WaitForFullGCComplete();

                CompressStatus status = Lzham.CompressMemory(p, fileData, fileData.Length, 0, compressedData, ref compressedSize, 0, ref addler);
                if (status != CompressStatus.Success && status != CompressStatus.OutputBufferTooSmall)
                {
                    Console.WriteLine($"Error compressing file {file}. ({status})");
                    Environment.FailFast($"Error compressing file {file}. ({status})");
                }
                if (status != CompressStatus.OutputBufferTooSmall)
                {

                    fre.flag = 1;
                    fre.compressed = (uint)compressedSize;
                    _fs.Write(compressedData, 0, compressedSize);
                }
                else
                {
                    fre.flag = 0;
                    fre.compressed = fileSize;
                    using (FileStream f = File.OpenRead(file))
                    {
                        f.CopyTo(_fs);
                    }
                }
                _files.Add(fre);
                string justTheName = Path.GetFileName(file);
                byte[] fileNameBytes = Encoding.UTF8.GetBytes(justTheName);
                _directory.Write(BitConverter.GetBytes((ushort)fileNameBytes.Length), 0, 2);
                _directory.Write(fileNameBytes, 0, fileNameBytes.Length);
                filesSize += (ushort)(fileNameBytes.Length + 2);
            }

            dleh.size = (ushort)(headerSize + foldersSize + filesSize);

            long savePos = _directory.Position;
            _directory.Position = pos;
            byte[] data = Util.StructureToByteArray(dleh);
            _directory.Write(data,0, data.Length);
            _directory.Write(folderNameBytes, 0, folderNameBytes.Length);
            _directory.Position = savePos;

            return dleh.size;
        }