private static string LZMACompressFileForUpload(string relativePath, string stagingPath, string modPath, Func <bool?> cancelCheckCallback = null) { Log.Information(@"Compressing " + relativePath); var destPath = Path.Combine(stagingPath, relativePath + @".lzma"); var sourcePath = Path.Combine(modPath, relativePath); Directory.CreateDirectory(Directory.GetParent(destPath).FullName); var src = File.ReadAllBytes(sourcePath); var compressedBytes = LZMA.Compress(src); byte[] fixedBytes = new byte[compressedBytes.Count() + 8]; //needs 8 byte header written into it (only mem version needs this) Buffer.BlockCopy(compressedBytes, 0, fixedBytes, 0, 5); fixedBytes.OverwriteRange(5, BitConverter.GetBytes((int)new FileInfo(sourcePath).Length)); Buffer.BlockCopy(compressedBytes, 5, fixedBytes, 13, compressedBytes.Length - 5); File.WriteAllBytes(destPath, fixedBytes); //Test! //var decomp = SevenZipHelper.LZMA.DecompressLZMAFile(fixedBytes); //if (decomp == null) //{ // Debug.WriteLine("NOT LZMA"); //} //else if (decomp.Length != src.Length) //{ // Debug.WriteLine("Decompressed data does not match source length!"); //} return(destPath); }
/// <summary> /// 自动压缩 /// </summary> /// <param name="buffer"></param> /// <returns>返回压缩后的字节</returns> public byte[] CompressBuffer(byte[] buffer) { if (buffer.Length > Enable7zipMinByte) { return(LZMA.Compress(buffer));//GzipUtils.EnCompress(buffer, 0, buffer.Length); } return(buffer); }
public static void UpdateMD5Map(MEGame game, string directory, string outName) { var outF = $@"C:\Users\mgame\source\repos\ME3Tweaks\MassEffectModManager\MassEffectModManagerCore\modmanager\gamemd5\{outName}"; var db = VanillaDatabaseService.LoadDatabaseFor(game, false); if (game.IsLEGame()) { db.RemoveAll(x => x.Key.Contains(@"BioGame\Config")); // Do not include config files } MemoryStream mapStream = new MemoryStream(); // Name Table mapStream.WriteInt32(db.Count); // Num Entries foreach (var f in db.Keys) { mapStream.WriteStringASCIINull(f); } // Data Table mapStream.WriteInt32(db.Count); int idx = 0; foreach (var f in db) { mapStream.WriteInt32(idx); // Name Table IDX. Update this code for duplicates support mapStream.WriteInt32(f.Value[0].size); // Size var md5 = f.Value[0].md5; for (int i = 0; i < 32; i++) { byte b = 0; b |= HexToInt(md5[i]); b = (byte)(b << 4); i++; b |= HexToInt(md5[i]); mapStream.WriteByte(b); } idx++; } var compBytes = LZMA.Compress(mapStream.ToArray()); MemoryStream ms = new MemoryStream(); ms.WriteStringASCII(@"MD5T"); ms.WriteInt32((int)mapStream.Length); ms.Write(compBytes); ms.WriteToFile(outF); }
public virtual void Assemble(FlashWriter output, CompressionKind compression, Action <TagItem> callback) { output.Write(((char)compression) + "WS", true); output.Write(Version); output.Write(uint.MinValue); int fileLength = 8; FlashWriter compressor = null; switch (compression) { case CompressionKind.LZMA: { compressor = new FlashWriter((int)FileLength); break; } case CompressionKind.ZLIB: { compressor = ZLIB.WrapCompressor(output.BaseStream, true); break; } } /* Body Start */ Frame.WriteTo(compressor ?? output); fileLength += (Frame.Area.GetByteSize() + 4); for (int i = 0; i < Tags.Count; i++) { TagItem tag = Tags[i]; callback?.Invoke(tag); WriteTag(tag, compressor ?? output); fileLength += tag.Header.Length; fileLength += (tag.Header.IsLongTag ? 6 : 2); } if (compression == CompressionKind.LZMA) { byte[] uncompressedBody = ((MemoryStream)compressor.BaseStream).ToArray(); byte[] compressedBody = LZMA.Compress(uncompressedBody); output.Write(compressedBody); } compressor?.Dispose(); /* Body End */ output.Position = 4; output.Write((uint)fileLength); output.Position = output.Length; }
public static void GenerateMD5Map(MEGame game, string directory, string outName) { var allFiles = Directory.GetFiles(directory, @"*.*", SearchOption.AllDirectories);//.Take(10).ToArray(); if (game.IsLEGame()) { allFiles.RemoveAll(x => x.Contains(@"BioGame\Config")); // Do not include config files } MemoryStream mapStream = new MemoryStream(); // Name Table mapStream.WriteInt32(allFiles.Length); // Num Entries foreach (var f in allFiles) { mapStream.WriteStringASCIINull(f.Substring(directory.Length + 1)); } // Data Table mapStream.WriteInt32(allFiles.Length); int idx = 0; foreach (var f in allFiles) { mapStream.WriteInt32(idx); // Name Table IDX. Update this code for duplicates support mapStream.WriteInt32((int)new FileInfo(f).Length); // Size var md5 = Utilities.CalculateMD5(f); for (int i = 0; i < 32; i++) { byte b = 0; b |= HexToInt(md5[i]); b = (byte)(b << 4); i++; b |= HexToInt(md5[i]); mapStream.WriteByte(b); } idx++; } var compBytes = LZMA.Compress(mapStream.ToArray()); MemoryStream outStream = new MemoryStream(); outStream.WriteStringASCII(@"MD5T"); outStream.WriteInt32((int)mapStream.Length); outStream.Write(compBytes); outStream.WriteToFile($@"C:\Users\mgame\source\repos\ME3Tweaks\MassEffectModManager\MassEffectModManagerCore\modmanager\gamemd5\{outName}"); }
/// <summary> /// LZMA compression helper function. /// </summary> static public byte[] Compress(byte[] data, byte[] prefix = null) { MemoryStream stream = new MemoryStream(); stream.Write(data, 0, data.Length); stream.Position = 0; MemoryStream outStream = LZMA.Compress(stream, prefix); stream.Close(); if (outStream != null) { data = outStream.ToArray(); } outStream.Close(); return(data); }
/// <summary> /// LZMA compression helper function. /// </summary> static public byte[] Compress(byte[] data, byte[] prefix = null) { var stream = new MemoryStream(); stream.Write(data, 0, data.Length); stream.Position = 0; var outStream = LZMA.Compress(stream, prefix); stream.Close(); byte[] retVal = null; if (outStream != null) { retVal = outStream.ToArray(); } outStream.Close(); return(retVal); }
public static void Deploy() { var debug_path = Application.dataPath + kLuaScriptDebugPath; var resources_path = Application.dataPath + "/Resources"; var deploy_path = resources_path + kLuaScriptPath; if (Directory.Exists(deploy_path)) { DirectoryInfo deploy_di = new DirectoryInfo(deploy_path); deploy_di.Delete(true); } FileInfo[] fileinfos = null; DirectoryInfo di = new DirectoryInfo(debug_path); var debug_path_length = di.FullName.Length; if (di != null) { fileinfos = di.GetFiles("*", SearchOption.AllDirectories); } foreach (var fi in fileinfos) { var suffix = Path.GetExtension(fi.FullName); if (suffix.ToLower() != ".lua") { continue; } var filepath = fi.FullName.Substring(debug_path_length); filepath = Path.GetDirectoryName(filepath); var filename = Path.GetFileName(fi.FullName); filename = Path.GetFileNameWithoutExtension(filename); byte[] bytes = null; using (var s = new StreamReader(fi.FullName)) { var b = s.ReadToEnd(); bytes = LZMA.Compress(Encoding.UTF8.GetBytes(b)); } var path = deploy_path + "/" + filepath; DeployFile(path, filename, bytes); } }
/// <summary> /// Just here for consistency. /// </summary> public void Write(BinaryWriter writer, bool compressed = false) { if (compressed) { LZMA lzma = new LZMA(); lzma.BeginWriting().WriteObject(this); byte[] data = lzma.Compress(); if (data != null) { for (int i = 0; i < 4; ++i) { writer.Write(mLZMA[i]); } writer.Write(data); return; } } writer.WriteObject(this); }
/// <summary> /// Write the node hierarchy to the specified filename. /// </summary> public bool Write(string path, SaveType type = SaveType.Text, bool allowConfigAccess = false) { bool retVal = false; MemoryStream stream = new MemoryStream(); if (type == SaveType.Binary) { BinaryWriter writer = new BinaryWriter(stream); writer.WriteObject(this); retVal = Tools.WriteFile(path, stream, false, allowConfigAccess); writer.Close(); } else if (type == SaveType.Compressed) { BinaryWriter writer = new BinaryWriter(stream); writer.WriteObject(this); stream.Position = 0; MemoryStream comp = LZMA.Compress(stream, mLZMA); if (comp != null) { retVal = Tools.WriteFile(path, comp, false, allowConfigAccess); comp.Close(); } else { retVal = Tools.WriteFile(path, stream, false, allowConfigAccess); } writer.Close(); } else { StreamWriter writer = new StreamWriter(stream); Write(writer, 0); retVal = Tools.WriteFile(path, stream, false, allowConfigAccess); writer.Close(); } return(retVal); }
/// <summary> /// Envia datos al servidor /// </summary> public void EnviarComando(ushort comando, string datos) { if (TcpClient.Connected == false) { return; } byte[] array = Encoding.UTF8.GetBytes(datos); bool comprimido = false; if (Comprimir) { byte[] arrayComprimido = LZMA.Compress(array); if (arrayComprimido.Length < array.Length) { comprimido = true; array = arrayComprimido; } } TcpClient.SendBufferSize = array.Length + 15; if (TcpClient != null) { NetworkStream stream = TcpClient.GetStream(); BinaryWriter escritor = new BinaryWriter(stream); escritor.Write(comprimido); escritor.Write((UInt16)comando); escritor.Write((Int32)array.Length); escritor.Write(array); escritor.Flush(); System.Windows.Forms.Application.DoEvents(); } }
private static void Save(string path, string token) { if (!KK_SaveLoadCompression.Enable.Value || !KK_SaveLoadCompression.Notice.Value) { return; } byte[] pngData; string TempPath = Path.Combine(KK_SaveLoadCompression.CacheDirectory.CreateSubdirectory("Compressed").FullName, Path.GetFileName(path)); //這裡用cleanedPath作"_compressed"字串清理 string cleanedPath = path; while (cleanedPath.Contains("_compressed")) { cleanedPath = cleanedPath.Replace("_compressed", ""); } if (cleanedPath != path) { File.Copy(path, cleanedPath, true); Logger.LogDebug($"Clean Path: {cleanedPath}"); } //Update Cache string decompressPath = Path.Combine(KK_SaveLoadCompression.CacheDirectory.CreateSubdirectory("Decompressed").FullName, Path.GetFileName(cleanedPath)); File.Copy(path, decompressPath, true); using (FileStream fileStreamReader = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { using (BinaryReader binaryReader = new BinaryReader(fileStreamReader)) { Logger.LogInfo("Start Compress"); pngData = PngFile.LoadPngBytes(binaryReader); Texture2D png = new Texture2D(2, 2); png.LoadImage(pngData); Texture2D watermark = Extension.Extension.LoadDllResource($"KK_SaveLoadCompression.Resources.zip_watermark.png"); float scaleTimes = (token == StudioToken) ? .14375f : .30423f; watermark = Extension.Extension.Scale(watermark, Convert.ToInt32(png.width * scaleTimes), Convert.ToInt32(png.width * scaleTimes)); png = Extension.Extension.OverwriteTexture( png, watermark, 0, png.height - watermark.height ); //Logger.LogDebug($"Add Watermark: zip"); pngData = png.EncodeToPNG(); } } Thread newThread = new Thread(doMain); newThread.Start(); void doMain() { bool successFlag = true; try { using (FileStream fileStreamReader = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { float startTime = Time.time; using (FileStream fileStreamWriter = new FileStream(TempPath, FileMode.Create, FileAccess.Write)) { using (BinaryWriter binaryWriter = new BinaryWriter(fileStreamWriter)) { binaryWriter.Write(pngData); switch (token) { case StudioToken: //Studio binaryWriter.Write(new Version(101, 0, 0, 0).ToString()); break; case CoordinateToken: //Coordinate binaryWriter.Write(101); break; default: //Chara if (token.IndexOf(CharaToken) >= 0) { binaryWriter.Write(101); break; } throw new Exception("Token not match."); } //為了通過 InvalidSceneFileProtection 和 DragAndDrop binaryWriter.Write(token); using (MemoryStream msCompressed = new MemoryStream()) { PngFile.SkipPng(fileStreamReader); long fileStreamPos = fileStreamReader.Position; LZMA.Compress(fileStreamReader, msCompressed, LzmaSpeed.Fastest, KK_SaveLoadCompression.DictionarySize.Value, delegate(long inSize, long _) { KK_SaveLoadCompression.Progress = $"Compressing: {Convert.ToInt32(inSize * 100 / (fileStreamReader.Length - fileStreamPos))}%"; } ); KK_SaveLoadCompression.Progress = ""; Logger.LogInfo("Start compression test..."); using (MemoryStream msDecompressed = new MemoryStream()) { if (!KK_SaveLoadCompression.SkipSaveCheck.Value) { msCompressed.Seek(0, SeekOrigin.Begin); LZMA.Decompress(msCompressed, msDecompressed, delegate(long inSize, long _) { KK_SaveLoadCompression.Progress = $"Decompressing: {Convert.ToInt32(inSize * 100 / (fileStreamReader.Length - fileStreamPos))}%"; } ); KK_SaveLoadCompression.Progress = ""; fileStreamReader.Seek(fileStreamPos, SeekOrigin.Begin); msDecompressed.Seek(0, SeekOrigin.Begin); for (int i = 0; i < msDecompressed.Length; i++) { KK_SaveLoadCompression.Progress = $"Comparing: {i * 100 / msDecompressed.Length}%"; int aByte = fileStreamReader.ReadByte(); int bByte = msDecompressed.ReadByte(); if (aByte.CompareTo(bByte) != 0) { successFlag = false; break; } } KK_SaveLoadCompression.Progress = ""; } if (successFlag) { long newSize = msCompressed.Length + token.Length + pngData.Length; binaryWriter.Write(msCompressed.ToArray()); LogLevel logLevel = KK_SaveLoadCompression.DisplayMessage.Value ? (LogLevel.Message | LogLevel.Info) : LogLevel.Info; Logger.LogInfo($"Compression test SUCCESS"); Logger.Log(logLevel, $"Compression finish in {Math.Round(Time.time - startTime, 2)} seconds"); Logger.Log(logLevel, $"Size compress from {fileStreamReader.Length} bytes to {newSize} bytes"); Logger.Log(logLevel, $"Compress ratio: {Math.Round(Convert.ToDouble(fileStreamReader.Length) / newSize, 2)}, which means it is now {Math.Round(100 / (Convert.ToDouble(fileStreamReader.Length) / newSize), 2)}% big."); } else { Logger.LogError($"Compression test FAILED"); } } } } } } //複製或刪除檔案 if (successFlag) { string compressedPath = cleanedPath; if (!KK_SaveLoadCompression.DeleteTheOri.Value) { compressedPath = cleanedPath.Substring(0, cleanedPath.Length - 4) + "_compressed.png"; } File.Copy(TempPath, compressedPath, true); Logger.LogDebug($"Write to: {compressedPath}"); //因為File.Delete()不是立即執行完畢,不能有「砍掉以後立即在同位置寫入」的操作,所以是這個邏輯順序 //如果相同的話,上方就已經覆寫了;不同的話此處再做刪除 if (path != compressedPath && path != cleanedPath) { File.Delete(path); Logger.LogDebug($"Delete Original File: {path}"); } } } catch (Exception e) { if (e is IOException && successFlag) { //覆寫時遇到讀取重整會IOException: Sharing violation on path,這在Compress太快時會發生 //Retry try { if (File.Exists(TempPath)) { if (KK_SaveLoadCompression.DeleteTheOri.Value) { File.Copy(TempPath, path, true); } } } catch (Exception) { //Copy to a new name if failed twice File.Copy(TempPath, path.Substring(0, path.Length - 4) + "_compressed2.png"); Logger.LogError("Overwrite was FAILED twice. Fallback to use the '_compressed2' path."); } } else { Logger.LogError($"An unknown error occurred. If your files are lost, please find them at %TEMP%/{KK_SaveLoadCompression.GUID}"); throw; } } finally { if (File.Exists(TempPath)) { File.Delete(TempPath); } } } }
public long Save(Stream inputStream, Stream outputStream, string token = null, byte[] pngData = null, Action <decimal> compressProgress = null, bool doComapre = true, Action <decimal> compareProgress = null) { long dataSize = 0; Action <long, long> _compressProgress = null; if (null != compressProgress) { _compressProgress = (long inSize, long _) => compressProgress(Convert.ToDecimal(inSize) / dataSize); } //Make png watermarked using (BinaryReader binaryReader = new BinaryReader(inputStream)) using (BinaryWriter binaryWriter = new BinaryWriter(outputStream)) { if (null == pngData) { pngData = ImageHelper.LoadPngBytes(binaryReader); } else { ImageHelper.SkipPng(binaryReader); Logger.LogDebug("Skip Png:" + inputStream.Position); } dataSize = inputStream.Length - inputStream.Position; binaryWriter.Write(pngData); if (null == token) { token = GuessToken(binaryReader); } switch (token) { case Token.StudioToken: //Studio binaryWriter.Write(new Version(101, 0, 0, 0).ToString()); break; case Token.CoordinateToken: //Coordinate binaryWriter.Write(101); break; default: //Chara if (token.IndexOf(Token.CharaToken) >= 0) { binaryWriter.Write(101); break; } throw new Exception("Token not match."); } //為了通過 InvalidSceneFileProtection 和 DragAndDrop binaryWriter.Write(token); using (MemoryStream msCompressed = new MemoryStream()) { //PngFile.SkipPng(inputStream); long fileStreamPos = inputStream.Position; LZMA.Compress( inputStream, msCompressed, LzmaSpeed.Fastest, DictionarySize.VeryLarge, _compressProgress ); Logger.LogDebug("Start compression test..."); if (doComapre) { using (MemoryStream msDecompressed = new MemoryStream()) { msCompressed.Seek(0, SeekOrigin.Begin); LZMA.Decompress(msCompressed, msDecompressed); inputStream.Seek(fileStreamPos, SeekOrigin.Begin); msDecompressed.Seek(0, SeekOrigin.Begin); int bufferSize = 1 << 10; byte[] aByteA = new byte[(int)bufferSize]; byte[] bByteA = new byte[(int)bufferSize]; if ((inputStream.Length - inputStream.Position) != msDecompressed.Length) { return(0); } for (long i = 0; i < msDecompressed.Length;) { if (null != compressProgress) { compareProgress(Convert.ToDecimal(i) / msDecompressed.Length); } inputStream.Read(aByteA, 0, (int)bufferSize); i += msDecompressed.Read(bByteA, 0, (int)bufferSize); if (!aByteA.SequenceEqual(bByteA)) { return(0); } } } } binaryWriter.Write(msCompressed.ToArray()); return(binaryWriter.BaseStream.Length); } } }
/// <summary> /// Creates a aptch file and writes its output to the given stream. /// </summary> /// <param name="ROMStream">The ROM stream to read from</param> /// <param name="outStream">The output stream to write to</param> public static void CreatePatchFile(Stream ROMStream, Stream outStream) { bool GeoLayoutInSeg04 = false; uint features = 0; long Seg04Offset = Task.Run(() => GetSegmentOffset(ROMStream)).Result; long Seg04Length = Task.Run(() => GetSegmentLength(ROMStream)).Result; GeoLayoutInformation GeoInfo = Task.Run(() => GetGeoLayoutLength(ROMStream)).Result; byte[] GeoLayoutData = new byte[GeoInfo.Length]; byte[] Seg04Data = new byte[Seg04Length]; ROMStream.Seek(Seg04Offset, SeekOrigin.Begin); ROMStream.Read(Seg04Data, 0, (int)Seg04Length); // Original length of Bank 04 is 0x35378 if (Seg04Length > 0x35378) { features |= 1; } if (GeoInfo.StartMargin > Seg04Offset && GeoInfo.StartMargin < Seg04Offset + Seg04Length) { GeoLayoutInSeg04 = true; features |= 2; } else { ROMStream.Seek(GeoInfo.StartMargin, SeekOrigin.Begin); ROMStream.Read(GeoLayoutData, 0, (int)GeoInfo.Length); } MemoryStream compressedS04 = new MemoryStream(); MemoryStream compressedGL = new MemoryStream(); // TODO: Implement Checksum // It'll be hard, can't rely on 7z's since it takes too much mem uint CRCChecksum; byte[] GeoLayoutSegAddr = new byte[8]; using (MemoryStream ms = new MemoryStream(Seg04Data)) { LZMA.Compress(ms, compressedS04, LzmaSpeed.Medium, DictionarySize.Medium); ms.Flush(); } if (!GeoLayoutInSeg04) { using (MemoryStream ms = new MemoryStream(GeoLayoutData)) { LZMA.Compress(ms, compressedGL, LzmaSpeed.Medium, DictionarySize.Medium); ms.Flush(); } } ROMStream.Seek(0x2ABCE0, SeekOrigin.Begin); ROMStream.Read(GeoLayoutSegAddr, 0, 8); // Time to write everything outStream.Write(ASCII.GetBytes("CMTP".ToCharArray()), 0, 4); outStream.Write(new byte[] { 00, 01 }, 0, 2); //CMTP v0.1 outStream.Write(BitConverter.GetBytes((ushort)features), 0, 2); outStream.Write(BitConverter.GetBytes((uint)compressedS04.Length), 0, 4); outStream.Write(BitConverter.GetBytes((uint)compressedGL.Length), 0, 4); outStream.Write(BitConverter.GetBytes((uint)GeoInfo.StartMargin), 0, 4); outStream.Write(BitConverter.GetBytes(Seg04Length), 0, 4); outStream.Write(BitConverter.GetBytes(GeoInfo.Length), 0, 4); outStream.Write(GeoLayoutSegAddr.Skip(4).ToArray(), 0, 4); outStream.Write(compressedS04.GetBuffer(), 0, (int)compressedS04.Length); outStream.Write(compressedGL.GetBuffer(), 0, (int)compressedGL.Length); compressedGL.Flush(); compressedS04.Flush(); ROMStream.Dispose(); }
private static byte[] CompressGzipBuffer(byte[] data) { return(LZMA.Compress(data));//GzipUtils.EnCompress(data, 0, data.Length); }