private static string LZMACompressFileForUpload(string relativePath, string stagingPath, string modPath, Func <bool?> cancelCheckCallback = null)
{
Log.Information(@"Compressing " + relativePath);
var destPath = Path.Combine(stagingPath, relativePath + @".lzma");
var sourcePath = Path.Combine(modPath, relativePath);
Directory.CreateDirectory(Directory.GetParent(destPath).FullName);
var src = File.ReadAllBytes(sourcePath);
var compressedBytes = LZMA.Compress(src);
byte[] fixedBytes = new byte[compressedBytes.Count() + 8]; //needs 8 byte header written into it (only mem version needs this)
Buffer.BlockCopy(compressedBytes, 0, fixedBytes, 0, 5);
fixedBytes.OverwriteRange(5, BitConverter.GetBytes((int)new FileInfo(sourcePath).Length));
Buffer.BlockCopy(compressedBytes, 5, fixedBytes, 13, compressedBytes.Length - 5);
File.WriteAllBytes(destPath, fixedBytes);
//Test!
//var decomp = SevenZipHelper.LZMA.DecompressLZMAFile(fixedBytes);
//if (decomp == null)
//{
// Debug.WriteLine("NOT LZMA");
//}
//else if (decomp.Length != src.Length)
//{
// Debug.WriteLine("Decompressed data does not match source length!");
//}
return(destPath);
}
/// <summary>
/// 自动压缩
/// </summary>
/// <param name="buffer"></param>
/// <returns>返回压缩后的字节</returns>
public byte[] CompressBuffer(byte[] buffer)
{
if (buffer.Length > Enable7zipMinByte)
{
return(LZMA.Compress(buffer));//GzipUtils.EnCompress(buffer, 0, buffer.Length);
}
return(buffer);
}
public static void UpdateMD5Map(MEGame game, string directory, string outName)
{
var outF = $@"C:\Users\mgame\source\repos\ME3Tweaks\MassEffectModManager\MassEffectModManagerCore\modmanager\gamemd5\{outName}";
var db = VanillaDatabaseService.LoadDatabaseFor(game, false);
if (game.IsLEGame())
{
db.RemoveAll(x => x.Key.Contains(@"BioGame\Config")); // Do not include config files
}
MemoryStream mapStream = new MemoryStream();
// Name Table
mapStream.WriteInt32(db.Count); // Num Entries
foreach (var f in db.Keys)
{
mapStream.WriteStringASCIINull(f);
}
// Data Table
mapStream.WriteInt32(db.Count);
int idx = 0;
foreach (var f in db)
{
mapStream.WriteInt32(idx); // Name Table IDX. Update this code for duplicates support
mapStream.WriteInt32(f.Value[0].size); // Size
var md5 = f.Value[0].md5;
for (int i = 0; i < 32; i++)
{
byte b = 0;
b |= HexToInt(md5[i]);
b = (byte)(b << 4);
i++;
b |= HexToInt(md5[i]);
mapStream.WriteByte(b);
}
idx++;
}
var compBytes = LZMA.Compress(mapStream.ToArray());
MemoryStream ms = new MemoryStream();
ms.WriteStringASCII(@"MD5T");
ms.WriteInt32((int)mapStream.Length);
ms.Write(compBytes);
ms.WriteToFile(outF);
}
public virtual void Assemble(FlashWriter output, CompressionKind compression, Action <TagItem> callback)
{
output.Write(((char)compression) + "WS", true);
output.Write(Version);
output.Write(uint.MinValue);
int fileLength = 8;
FlashWriter compressor = null;
switch (compression)
{
case CompressionKind.LZMA:
{
compressor = new FlashWriter((int)FileLength);
break;
}
case CompressionKind.ZLIB:
{
compressor = ZLIB.WrapCompressor(output.BaseStream, true);
break;
}
}
/* Body Start */
Frame.WriteTo(compressor ?? output);
fileLength += (Frame.Area.GetByteSize() + 4);
for (int i = 0; i < Tags.Count; i++)
{
TagItem tag = Tags[i];
callback?.Invoke(tag);
WriteTag(tag, compressor ?? output);
fileLength += tag.Header.Length;
fileLength += (tag.Header.IsLongTag ? 6 : 2);
}
if (compression == CompressionKind.LZMA)
{
byte[] uncompressedBody = ((MemoryStream)compressor.BaseStream).ToArray();
byte[] compressedBody = LZMA.Compress(uncompressedBody);
output.Write(compressedBody);
}
compressor?.Dispose();
/* Body End */
output.Position = 4;
output.Write((uint)fileLength);
output.Position = output.Length;
}
public static void GenerateMD5Map(MEGame game, string directory, string outName)
{
var allFiles = Directory.GetFiles(directory, @"*.*", SearchOption.AllDirectories);//.Take(10).ToArray();
if (game.IsLEGame())
{
allFiles.RemoveAll(x => x.Contains(@"BioGame\Config")); // Do not include config files
}
MemoryStream mapStream = new MemoryStream();
// Name Table
mapStream.WriteInt32(allFiles.Length); // Num Entries
foreach (var f in allFiles)
{
mapStream.WriteStringASCIINull(f.Substring(directory.Length + 1));
}
// Data Table
mapStream.WriteInt32(allFiles.Length);
int idx = 0;
foreach (var f in allFiles)
{
mapStream.WriteInt32(idx); // Name Table IDX. Update this code for duplicates support
mapStream.WriteInt32((int)new FileInfo(f).Length); // Size
var md5 = Utilities.CalculateMD5(f);
for (int i = 0; i < 32; i++)
{
byte b = 0;
b |= HexToInt(md5[i]);
b = (byte)(b << 4);
i++;
b |= HexToInt(md5[i]);
mapStream.WriteByte(b);
}
idx++;
}
var compBytes = LZMA.Compress(mapStream.ToArray());
MemoryStream outStream = new MemoryStream();
outStream.WriteStringASCII(@"MD5T");
outStream.WriteInt32((int)mapStream.Length);
outStream.Write(compBytes);
outStream.WriteToFile($@"C:\Users\mgame\source\repos\ME3Tweaks\MassEffectModManager\MassEffectModManagerCore\modmanager\gamemd5\{outName}");
}
/// <summary>
/// LZMA compression helper function.
/// </summary>
static public byte[] Compress(byte[] data, byte[] prefix = null)
{
MemoryStream stream = new MemoryStream();
stream.Write(data, 0, data.Length);
stream.Position = 0;
MemoryStream outStream = LZMA.Compress(stream, prefix);
stream.Close();
if (outStream != null)
{
data = outStream.ToArray();
}
outStream.Close();
return(data);
}
/// <summary>
/// LZMA compression helper function.
/// </summary>
static public byte[] Compress(byte[] data, byte[] prefix = null)
{
var stream = new MemoryStream();
stream.Write(data, 0, data.Length);
stream.Position = 0;
var outStream = LZMA.Compress(stream, prefix);
stream.Close();
byte[] retVal = null;
if (outStream != null)
{
retVal = outStream.ToArray();
}
outStream.Close();
return(retVal);
}
public static void Deploy()
{
var debug_path = Application.dataPath + kLuaScriptDebugPath;
var resources_path = Application.dataPath + "/Resources";
var deploy_path = resources_path + kLuaScriptPath;
if (Directory.Exists(deploy_path))
{
DirectoryInfo deploy_di = new DirectoryInfo(deploy_path);
deploy_di.Delete(true);
}
FileInfo[] fileinfos = null;
DirectoryInfo di = new DirectoryInfo(debug_path);
var debug_path_length = di.FullName.Length;
if (di != null)
{
fileinfos = di.GetFiles("*", SearchOption.AllDirectories);
}
foreach (var fi in fileinfos)
{
var suffix = Path.GetExtension(fi.FullName);
if (suffix.ToLower() != ".lua")
{
continue;
}
var filepath = fi.FullName.Substring(debug_path_length);
filepath = Path.GetDirectoryName(filepath);
var filename = Path.GetFileName(fi.FullName);
filename = Path.GetFileNameWithoutExtension(filename);
byte[] bytes = null;
using (var s = new StreamReader(fi.FullName))
{
var b = s.ReadToEnd();
bytes = LZMA.Compress(Encoding.UTF8.GetBytes(b));
}
var path = deploy_path + "/" + filepath;
DeployFile(path, filename, bytes);
}
}
/// <summary>
/// Just here for consistency.
/// </summary>
public void Write(BinaryWriter writer, bool compressed = false)
{
if (compressed)
{
LZMA lzma = new LZMA();
lzma.BeginWriting().WriteObject(this);
byte[] data = lzma.Compress();
if (data != null)
{
for (int i = 0; i < 4; ++i)
{
writer.Write(mLZMA[i]);
}
writer.Write(data);
return;
}
}
writer.WriteObject(this);
}
/// <summary>
/// Write the node hierarchy to the specified filename.
/// </summary>
public bool Write(string path, SaveType type = SaveType.Text, bool allowConfigAccess = false)
{
bool retVal = false;
MemoryStream stream = new MemoryStream();
if (type == SaveType.Binary)
{
BinaryWriter writer = new BinaryWriter(stream);
writer.WriteObject(this);
retVal = Tools.WriteFile(path, stream, false, allowConfigAccess);
writer.Close();
}
else if (type == SaveType.Compressed)
{
BinaryWriter writer = new BinaryWriter(stream);
writer.WriteObject(this);
stream.Position = 0;
MemoryStream comp = LZMA.Compress(stream, mLZMA);
if (comp != null)
{
retVal = Tools.WriteFile(path, comp, false, allowConfigAccess);
comp.Close();
}
else
{
retVal = Tools.WriteFile(path, stream, false, allowConfigAccess);
}
writer.Close();
}
else
{
StreamWriter writer = new StreamWriter(stream);
Write(writer, 0);
retVal = Tools.WriteFile(path, stream, false, allowConfigAccess);
writer.Close();
}
return(retVal);
}
/// <summary>
/// Envia datos al servidor
/// </summary>
public void EnviarComando(ushort comando, string datos)
{
if (TcpClient.Connected == false)
{
return;
}
byte[] array = Encoding.UTF8.GetBytes(datos);
bool comprimido = false;
if (Comprimir)
{
byte[] arrayComprimido = LZMA.Compress(array);
if (arrayComprimido.Length < array.Length)
{
comprimido = true;
array = arrayComprimido;
}
}
TcpClient.SendBufferSize = array.Length + 15;
if (TcpClient != null)
{
NetworkStream stream = TcpClient.GetStream();
BinaryWriter escritor = new BinaryWriter(stream);
escritor.Write(comprimido);
escritor.Write((UInt16)comando);
escritor.Write((Int32)array.Length);
escritor.Write(array);
escritor.Flush();
System.Windows.Forms.Application.DoEvents();
}
}
private static void Save(string path, string token)
{
if (!KK_SaveLoadCompression.Enable.Value || !KK_SaveLoadCompression.Notice.Value)
{
return;
}
byte[] pngData;
string TempPath = Path.Combine(KK_SaveLoadCompression.CacheDirectory.CreateSubdirectory("Compressed").FullName, Path.GetFileName(path));
//這裡用cleanedPath作"_compressed"字串清理
string cleanedPath = path;
while (cleanedPath.Contains("_compressed"))
{
cleanedPath = cleanedPath.Replace("_compressed", "");
}
if (cleanedPath != path)
{
File.Copy(path, cleanedPath, true);
Logger.LogDebug($"Clean Path: {cleanedPath}");
}
//Update Cache
string decompressPath = Path.Combine(KK_SaveLoadCompression.CacheDirectory.CreateSubdirectory("Decompressed").FullName, Path.GetFileName(cleanedPath));
File.Copy(path, decompressPath, true);
using (FileStream fileStreamReader = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
using (BinaryReader binaryReader = new BinaryReader(fileStreamReader)) {
Logger.LogInfo("Start Compress");
pngData = PngFile.LoadPngBytes(binaryReader);
Texture2D png = new Texture2D(2, 2);
png.LoadImage(pngData);
Texture2D watermark = Extension.Extension.LoadDllResource($"KK_SaveLoadCompression.Resources.zip_watermark.png");
float scaleTimes = (token == StudioToken) ? .14375f : .30423f;
watermark = Extension.Extension.Scale(watermark, Convert.ToInt32(png.width * scaleTimes), Convert.ToInt32(png.width * scaleTimes));
png = Extension.Extension.OverwriteTexture(
png,
watermark,
0,
png.height - watermark.height
);
//Logger.LogDebug($"Add Watermark: zip");
pngData = png.EncodeToPNG();
}
}
Thread newThread = new Thread(doMain);
newThread.Start();
void doMain()
{
bool successFlag = true;
try {
using (FileStream fileStreamReader = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
float startTime = Time.time;
using (FileStream fileStreamWriter = new FileStream(TempPath, FileMode.Create, FileAccess.Write)) {
using (BinaryWriter binaryWriter = new BinaryWriter(fileStreamWriter)) {
binaryWriter.Write(pngData);
switch (token)
{
case StudioToken:
//Studio
binaryWriter.Write(new Version(101, 0, 0, 0).ToString());
break;
case CoordinateToken:
//Coordinate
binaryWriter.Write(101);
break;
default:
//Chara
if (token.IndexOf(CharaToken) >= 0)
{
binaryWriter.Write(101);
break;
}
throw new Exception("Token not match.");
}
//為了通過 InvalidSceneFileProtection 和 DragAndDrop
binaryWriter.Write(token);
using (MemoryStream msCompressed = new MemoryStream()) {
PngFile.SkipPng(fileStreamReader);
long fileStreamPos = fileStreamReader.Position;
LZMA.Compress(fileStreamReader, msCompressed, LzmaSpeed.Fastest, KK_SaveLoadCompression.DictionarySize.Value,
delegate(long inSize, long _) {
KK_SaveLoadCompression.Progress = $"Compressing: {Convert.ToInt32(inSize * 100 / (fileStreamReader.Length - fileStreamPos))}%";
}
);
KK_SaveLoadCompression.Progress = "";
Logger.LogInfo("Start compression test...");
using (MemoryStream msDecompressed = new MemoryStream()) {
if (!KK_SaveLoadCompression.SkipSaveCheck.Value)
{
msCompressed.Seek(0, SeekOrigin.Begin);
LZMA.Decompress(msCompressed, msDecompressed,
delegate(long inSize, long _) {
KK_SaveLoadCompression.Progress = $"Decompressing: {Convert.ToInt32(inSize * 100 / (fileStreamReader.Length - fileStreamPos))}%";
}
);
KK_SaveLoadCompression.Progress = "";
fileStreamReader.Seek(fileStreamPos, SeekOrigin.Begin);
msDecompressed.Seek(0, SeekOrigin.Begin);
for (int i = 0; i < msDecompressed.Length; i++)
{
KK_SaveLoadCompression.Progress = $"Comparing: {i * 100 / msDecompressed.Length}%";
int aByte = fileStreamReader.ReadByte();
int bByte = msDecompressed.ReadByte();
if (aByte.CompareTo(bByte) != 0)
{
successFlag = false;
break;
}
}
KK_SaveLoadCompression.Progress = "";
}
if (successFlag)
{
long newSize = msCompressed.Length + token.Length + pngData.Length;
binaryWriter.Write(msCompressed.ToArray());
LogLevel logLevel = KK_SaveLoadCompression.DisplayMessage.Value ? (LogLevel.Message | LogLevel.Info) : LogLevel.Info;
Logger.LogInfo($"Compression test SUCCESS");
Logger.Log(logLevel, $"Compression finish in {Math.Round(Time.time - startTime, 2)} seconds");
Logger.Log(logLevel, $"Size compress from {fileStreamReader.Length} bytes to {newSize} bytes");
Logger.Log(logLevel, $"Compress ratio: {Math.Round(Convert.ToDouble(fileStreamReader.Length) / newSize, 2)}, which means it is now {Math.Round(100 / (Convert.ToDouble(fileStreamReader.Length) / newSize), 2)}% big.");
}
else
{
Logger.LogError($"Compression test FAILED");
}
}
}
}
}
}
//複製或刪除檔案
if (successFlag)
{
string compressedPath = cleanedPath;
if (!KK_SaveLoadCompression.DeleteTheOri.Value)
{
compressedPath = cleanedPath.Substring(0, cleanedPath.Length - 4) + "_compressed.png";
}
File.Copy(TempPath, compressedPath, true);
Logger.LogDebug($"Write to: {compressedPath}");
//因為File.Delete()不是立即執行完畢，不能有「砍掉以後立即在同位置寫入」的操作，所以是這個邏輯順序
//如果相同的話，上方就已經覆寫了；不同的話此處再做刪除
if (path != compressedPath && path != cleanedPath)
{
File.Delete(path);
Logger.LogDebug($"Delete Original File: {path}");
}
}
} catch (Exception e) {
if (e is IOException && successFlag)
{
//覆寫時遇到讀取重整會IOException: Sharing violation on path，這在Compress太快時會發生
//Retry
try {
if (File.Exists(TempPath))
{
if (KK_SaveLoadCompression.DeleteTheOri.Value)
{
File.Copy(TempPath, path, true);
}
}
} catch (Exception) {
//Copy to a new name if failed twice
File.Copy(TempPath, path.Substring(0, path.Length - 4) + "_compressed2.png");
Logger.LogError("Overwrite was FAILED twice. Fallback to use the '_compressed2' path.");
}
}
else
{
Logger.LogError($"An unknown error occurred. If your files are lost, please find them at %TEMP%/{KK_SaveLoadCompression.GUID}");
throw;
}
} finally {
if (File.Exists(TempPath))
{
File.Delete(TempPath);
}
}
}
}
public long Save(Stream inputStream,
Stream outputStream,
string token = null,
byte[] pngData = null,
Action <decimal> compressProgress = null,
bool doComapre = true,
Action <decimal> compareProgress = null)
{
long dataSize = 0;
Action <long, long> _compressProgress = null;
if (null != compressProgress)
{
_compressProgress = (long inSize, long _) => compressProgress(Convert.ToDecimal(inSize) / dataSize);
}
//Make png watermarked
using (BinaryReader binaryReader = new BinaryReader(inputStream))
using (BinaryWriter binaryWriter = new BinaryWriter(outputStream))
{
if (null == pngData)
{
pngData = ImageHelper.LoadPngBytes(binaryReader);
}
else
{
ImageHelper.SkipPng(binaryReader);
Logger.LogDebug("Skip Png:" + inputStream.Position);
}
dataSize = inputStream.Length - inputStream.Position;
binaryWriter.Write(pngData);
if (null == token)
{
token = GuessToken(binaryReader);
}
switch (token)
{
case Token.StudioToken:
//Studio
binaryWriter.Write(new Version(101, 0, 0, 0).ToString());
break;
case Token.CoordinateToken:
//Coordinate
binaryWriter.Write(101);
break;
default:
//Chara
if (token.IndexOf(Token.CharaToken) >= 0)
{
binaryWriter.Write(101);
break;
}
throw new Exception("Token not match.");
}
//為了通過 InvalidSceneFileProtection 和 DragAndDrop
binaryWriter.Write(token);
using (MemoryStream msCompressed = new MemoryStream())
{
//PngFile.SkipPng(inputStream);
long fileStreamPos = inputStream.Position;
LZMA.Compress(
inputStream,
msCompressed,
LzmaSpeed.Fastest,
DictionarySize.VeryLarge,
_compressProgress
);
Logger.LogDebug("Start compression test...");
if (doComapre)
{
using (MemoryStream msDecompressed = new MemoryStream())
{
msCompressed.Seek(0, SeekOrigin.Begin);
LZMA.Decompress(msCompressed, msDecompressed);
inputStream.Seek(fileStreamPos, SeekOrigin.Begin);
msDecompressed.Seek(0, SeekOrigin.Begin);
int bufferSize = 1 << 10;
byte[] aByteA = new byte[(int)bufferSize];
byte[] bByteA = new byte[(int)bufferSize];
if ((inputStream.Length - inputStream.Position) != msDecompressed.Length)
{
return(0);
}
for (long i = 0; i < msDecompressed.Length;)
{
if (null != compressProgress)
{
compareProgress(Convert.ToDecimal(i) / msDecompressed.Length);
}
inputStream.Read(aByteA, 0, (int)bufferSize);
i += msDecompressed.Read(bByteA, 0, (int)bufferSize);
if (!aByteA.SequenceEqual(bByteA))
{
return(0);
}
}
}
}
binaryWriter.Write(msCompressed.ToArray());
return(binaryWriter.BaseStream.Length);
}
}
}
/// <summary>
/// Creates a aptch file and writes its output to the given stream.
/// </summary>
/// <param name="ROMStream">The ROM stream to read from</param>
/// <param name="outStream">The output stream to write to</param>
public static void CreatePatchFile(Stream ROMStream, Stream outStream)
{
bool GeoLayoutInSeg04 = false;
uint features = 0;
long Seg04Offset = Task.Run(() => GetSegmentOffset(ROMStream)).Result;
long Seg04Length = Task.Run(() => GetSegmentLength(ROMStream)).Result;
GeoLayoutInformation GeoInfo = Task.Run(() => GetGeoLayoutLength(ROMStream)).Result;
byte[] GeoLayoutData = new byte[GeoInfo.Length];
byte[] Seg04Data = new byte[Seg04Length];
ROMStream.Seek(Seg04Offset, SeekOrigin.Begin);
ROMStream.Read(Seg04Data, 0, (int)Seg04Length);
// Original length of Bank 04 is 0x35378
if (Seg04Length > 0x35378)
{
features |= 1;
}
if (GeoInfo.StartMargin > Seg04Offset && GeoInfo.StartMargin < Seg04Offset + Seg04Length)
{
GeoLayoutInSeg04 = true;
features |= 2;
}
else
{
ROMStream.Seek(GeoInfo.StartMargin, SeekOrigin.Begin);
ROMStream.Read(GeoLayoutData, 0, (int)GeoInfo.Length);
}
MemoryStream compressedS04 = new MemoryStream();
MemoryStream compressedGL = new MemoryStream();
// TODO: Implement Checksum
// It'll be hard, can't rely on 7z's since it takes too much mem
uint CRCChecksum;
byte[] GeoLayoutSegAddr = new byte[8];
using (MemoryStream ms = new MemoryStream(Seg04Data))
{
LZMA.Compress(ms, compressedS04, LzmaSpeed.Medium, DictionarySize.Medium);
ms.Flush();
}
if (!GeoLayoutInSeg04)
{
using (MemoryStream ms = new MemoryStream(GeoLayoutData))
{
LZMA.Compress(ms, compressedGL, LzmaSpeed.Medium, DictionarySize.Medium);
ms.Flush();
}
}
ROMStream.Seek(0x2ABCE0, SeekOrigin.Begin);
ROMStream.Read(GeoLayoutSegAddr, 0, 8);
// Time to write everything
outStream.Write(ASCII.GetBytes("CMTP".ToCharArray()), 0, 4);
outStream.Write(new byte[] { 00, 01 }, 0, 2); //CMTP v0.1
outStream.Write(BitConverter.GetBytes((ushort)features), 0, 2);
outStream.Write(BitConverter.GetBytes((uint)compressedS04.Length), 0, 4);
outStream.Write(BitConverter.GetBytes((uint)compressedGL.Length), 0, 4);
outStream.Write(BitConverter.GetBytes((uint)GeoInfo.StartMargin), 0, 4);
outStream.Write(BitConverter.GetBytes(Seg04Length), 0, 4);
outStream.Write(BitConverter.GetBytes(GeoInfo.Length), 0, 4);
outStream.Write(GeoLayoutSegAddr.Skip(4).ToArray(), 0, 4);
outStream.Write(compressedS04.GetBuffer(), 0, (int)compressedS04.Length);
outStream.Write(compressedGL.GetBuffer(), 0, (int)compressedGL.Length);
compressedGL.Flush();
compressedS04.Flush();
ROMStream.Dispose();
}
private static byte[] CompressGzipBuffer(byte[] data)
{
return(LZMA.Compress(data));//GzipUtils.EnCompress(data, 0, data.Length);
}
