/*
* Name function: Compress
* Purpose: compress a part of the byte array into a Zlib Block
* Input: - buffer: byte array
* - offset: starting offset inside the array
* - count: num of bytes to compress starting from the offset
* Output: compressed byte array block, the structure is:
* - magic word
* - max segment size
* - total compressed size
* - total uncompressed size
* - segment list
* - compressed data list
*/
public static byte[] Compress(byte[] buffer, int offset, int count)
{
if(buffer == null)
throw new ArgumentNullException();
if (count < 0)
throw new FormatException();
if (offset + count > buffer.Length)
throw new IndexOutOfRangeException();
MemoryStream headBlock = new MemoryStream();
MemoryStream dataBlock = new MemoryStream();
DeflaterOutputStream zipStream;
int numSeg = (int)Math.Ceiling((double)count / (double)maxSegmentSize);
headBlock.WriteUInt32(magic);
headBlock.WriteUInt32(maxSegmentSize);
headBlock.WriteUInt32(0x0); //total compressed size, still to calculate
headBlock.WriteInt32(count); //total uncompressed size
for (int i = count; i > 0; i -= (int)maxSegmentSize)
{
int copyBytes = Math.Min(i, (int)maxSegmentSize);
uint precCompSize = (uint)dataBlock.Length;
zipStream = new DeflaterOutputStream(dataBlock);
zipStream.Write(buffer, offset + (count - i), copyBytes);
zipStream.Flush();
zipStream.Finish();
headBlock.WriteUInt32((uint)dataBlock.Length - precCompSize); //compressed segment size
headBlock.WriteInt32(copyBytes); //uncompressed segment size
//Console.WriteLine(" Segment size: {0}, total read: {1}, compr size: {2}", maxSegmentSize, copyBytes, (uint)dataBlock.Length - precCompSize);
}
headBlock.Seek(8, SeekOrigin.Begin);
headBlock.WriteInt32((int)dataBlock.Length); // total compressed size
byte[] finalBlock = new byte[headBlock.Length + dataBlock.Length];
Buffer.BlockCopy(headBlock.ToArray(), 0, finalBlock, 0, (int)headBlock.Length);
Buffer.BlockCopy(dataBlock.ToArray(), 0, finalBlock, (int)headBlock.Length, (int)dataBlock.Length);
headBlock.Close();
dataBlock.Close();
return finalBlock;
}
public static void WriteProperty(this Stream stream, string fieldName, Type fieldType,
object fieldValue, List<string> nameTable, int? arrayIndex = null)
{
if (fieldType.IsArray)
{
var array = (Array)fieldValue;
for (var i = 0; i < array.Length; i++)
{
var value = array.GetValue(i);
if (value != null)
stream.WriteProperty(fieldName, fieldType.GetElementType(), array.GetValue(i), nameTable, i);
}
return;
}
var fieldBodyStream = new MemoryStream();
var baseProperty = new BaseProperty
{
NameIdx = AddToNameTable(nameTable, fieldName),
ArrayIdx = arrayIndex ?? 0
};
if (fieldType == typeof (int))
{
baseProperty.TypeIdx = AddToNameTable(nameTable, "IntProperty");
var value = (int) fieldValue;
fieldBodyStream.WriteInt32(value);
}
else if (fieldType == typeof (bool))
{
baseProperty.TypeIdx = AddToNameTable(nameTable, "BoolProperty");
var value = (bool) fieldValue;
fieldBodyStream.WriteBool(value);
}
else if (fieldType.IsEnum)
{
baseProperty.TypeIdx = AddToNameTable(nameTable, "ByteProperty");
var value1 = AddToNameTable(nameTable, fieldType.Name); // Enum type name
var value2 = AddToNameTable(nameTable, fieldValue.ToString()); // Enum value
fieldBodyStream.WriteInt32(value1);
fieldBodyStream.WriteInt32(value2);
}
else if (fieldType == typeof (float))
{
baseProperty.TypeIdx = AddToNameTable(nameTable, "FloatProperty");
var value = (float) fieldValue;
fieldBodyStream.WriteFloat(value);
}
else if (fieldType == typeof (NameProperty))
{
baseProperty.TypeIdx = AddToNameTable(nameTable, "NameProperty");
var nameProp = (NameProperty) fieldValue;
var value = AddToNameTable(nameTable, nameProp.Name);
fieldBodyStream.WriteInt32(value);
}
else if (fieldType == typeof (string))
{
baseProperty.TypeIdx = AddToNameTable(nameTable, "StrProperty");
var value = (string) fieldValue;
switch (fieldName)
{
case "SaveDetails":
case "FriendlyName":
fieldBodyStream.WriteUnicodeString(value);
break;
default:
fieldBodyStream.WriteString(value);
break;
}
//fieldBodyStream.WriteUnicodeString(value); // Save every string as unicode
}
else if (fieldType.IsGenericType && (fieldType.GetGenericTypeDefinition() == typeof(List<>)))
{
baseProperty.TypeIdx = AddToNameTable(nameTable, "ArrayProperty");
var list = fieldValue as IList;
var elementType = fieldType.GetGenericArguments()[0];
fieldBodyStream.WriteInt32(list.Count); // numElements
for (var i = 0; i < list.Count; i++)
{
if (elementType == typeof (int))
{
fieldBodyStream.WriteInt32((int) list[i]);
}
else if (elementType == typeof (string))
{
fieldBodyStream.WriteString((string) list[i]);
}
else if (elementType == typeof (NameProperty))
{
var value = AddToNameTable(nameTable, ((NameProperty) list[i]).Name);
fieldBodyStream.WriteInt32(value);
}
else
{
fieldBodyStream.SerializeObject(list[i], nameTable);
}
}
}
else if (!fieldType.IsGenericType && !fieldType.IsValueType && !fieldType.IsPrimitive && fieldType.IsClass)
{
baseProperty.TypeIdx = AddToNameTable(nameTable, "StructProperty");
fieldBodyStream.WriteInt32(AddToNameTable(nameTable, fieldType.Name)); // StructNameIdx
if (fieldType == typeof (Vector))
{
var value = (Vector) fieldValue;
fieldBodyStream.WriteFloat(value.X);
fieldBodyStream.WriteFloat(value.Y);
fieldBodyStream.WriteFloat(value.Z);
}
else if (fieldType == typeof (Rotator))
{
var value = (Rotator) fieldValue;
fieldBodyStream.WriteInt32(value.Pitch);
fieldBodyStream.WriteInt32(value.Yaw);
fieldBodyStream.WriteInt32(value.Roll);
}
else
{
fieldBodyStream.SerializeObject(fieldValue, nameTable);
}
}
baseProperty.PropertySize = (int) fieldBodyStream.Length;
stream.WriteBaseProperty(baseProperty);
fieldBodyStream.WriteTo(stream);
}
/// <summary>
/// Append all array data to the binary stream.
/// </summary>
private void WriteBinaryArrayData(NpgsqlNativeTypeInfo TypeInfo, Array nativeData, NativeToBackendTypeConverterOptions options, MemoryStream dst, int dimensionOffset, int[] dimensionOffsets)
{
int dimensionLength = nativeData.GetLength(dimensionOffset);
int dimensionLBound = nativeData.GetLowerBound(dimensionOffset);
if (dimensionOffset < nativeData.Rank - 1)
{
// Drill down recursively until we hit a single dimension array.
for (int i = dimensionLBound ; i < dimensionLBound + dimensionLength ; i++)
{
dimensionOffsets[dimensionOffset] = i;
WriteBinaryArrayData(TypeInfo, nativeData, options, dst, dimensionOffset + 1, dimensionOffsets);
}
}
else
{
// Write the individual array elements to the output stream.
for (int i = dimensionLBound ; i < dimensionLBound + dimensionLength ; i++)
{
object elementNative;
dimensionOffsets[dimensionOffset] = i;
elementNative = nativeData.GetValue(dimensionOffsets);
if (elementNative == null || elementNative == DBNull.Value)
{
// Write length identifier -1 indicating NULL value.
dst.WriteInt32(-1);
}
else
{
byte[] elementBinary;
elementBinary = (byte[])_elementConverter.ConvertToBackend(elementNative, true, options);
// Write lenght identifier.
dst.WriteInt32(elementBinary.Length);
// Write element data.
dst.Write(elementBinary, 0, elementBinary.Length);
}
}
}
}
/// <summary>
/// Convert a System.Array to PG binary format.
/// Write the array header and prepare to write array data to the stream.
/// </summary>
public byte[] ArrayToArrayBinary(NpgsqlNativeTypeInfo TypeInfo, object oNativeData, NativeToBackendTypeConverterOptions options)
{
Array NativeData = (Array)oNativeData;
MemoryStream dst = new MemoryStream();
// Write the number of dimensions in the array.
dst.WriteInt32(NativeData.Rank);
// Placeholder for null bitmap flag, which isn't used?
dst.WriteInt32(0);
// Write the OID of the elements of the array.
dst.WriteInt32(options.OidToNameMapping[_elementConverter.Name].OID);
// White dimension descriptors.
for (int i = 0 ; i < NativeData.Rank ; i++)
{
// Number of elements in the dimension.
dst.WriteInt32(NativeData.GetLength(i));
// Lower bounds of the dimension, 1-based for SQL.
dst.WriteInt32(NativeData.GetLowerBound(i) + 1);
}
int[] dimensionOffsets = new int[NativeData.Rank];
// Write all array data.
WriteBinaryArrayData(TypeInfo, NativeData, options, dst, 0, dimensionOffsets);
return dst.ToArray();
}
void ReadCompressedME1And2(MemoryStream tempStream)
{
DebugOutput.PrintLn("File is compressed");
listsStream = SaltLZOHelper.DecompressPCC(tempStream, this);
//Correct the header
compressed = false;
listsStream.Seek(0, SeekOrigin.Begin);
listsStream.WriteBytes(header);
// Set numblocks to zero
listsStream.WriteInt32(0);
//Write the magic number
if (GameVersion == 1)
listsStream.WriteBytes(new byte[] { 0xF2, 0x56, 0x1B, 0x4E });
else
listsStream.WriteInt32(1026281201);
// Write 4 bytes of 0
listsStream.WriteInt32(0);
// Write 4 more for ME2
if (GameVersion == 2)
listsStream.WriteInt32(0);
}
private static void Main(string[] args)
{
if (args.Length == 0)
{
Console.WriteLine("Usage: ScriptEncoder.exe <input_file.txt>");
Console.ReadKey();
return;
}
string fileInput = args[0];
// Load script file.
var br = new BinaryReader(new FileStream(fileInput.Replace(".txt", ""), FileMode.Open));
var scriptBuffer = br.ReadBytes((int) br.BaseStream.Length);
// Load translation file.
var sr = new StreamReader(fileInput, Encoding.UTF8, true);
var lines = sr.ReadToEnd().Replace("\r\n", "\n").Split('\n');
sr.Close();
if (lines.Length == 0)
return;
// headerLength includes MAGIC and @[0x1C].
int headerLength = 0;
// Check whether the file is in new format.
// The most significant thing is the new format have the magic "BurikoCompiledScriptVer1.00\x00".
// The difference between old and new is that, the old one DOES NOT have the HEADER which has
// the length discribed at [0x1C] as a DWORD.
if (
scriptBuffer.Slice(0, 0x1C)
.EqualWith(new byte[]
{
0x42, 0x75, 0x72, 0x69, 0x6B,
0x6F, 0x43, 0x6F, 0x6D, 0x70,
0x69, 0x6C, 0x65, 0x64, 0x53,
0x63, 0x72, 0x69, 0x70, 0x74,
0x56, 0x65, 0x72, 0x31, 0x2E,
0x30, 0x30, 0x00
}))
{
headerLength = 0x1C + BitConverter.ToInt32(scriptBuffer, 0x1C);
}
// else headerLength = 0;
// Get control bytes from original buffer.
//var controlStream = new MemoryStream(scriptBuffer.Slice(headerLength, GetSmallestOffset(lines) + headerLength));
var controlStream = new MemoryStream(scriptBuffer.Slice(headerLength, scriptBuffer.Length));
// Let's begin.
var textStream = new MemoryStream();
foreach (var line in lines)
{
if (String.IsNullOrEmpty(line)) continue;
var info = GetLineInfo(line);
controlStream.WriteInt32(info[0], (int) (controlStream.Length + textStream.Length));
string curline = GetText(line);
if (curline.IndexOf("_") == -1)
{
textStream.WriteBytes(Encoding.GetEncoding(936).GetBytes(curline));
}
else
{
textStream.WriteBytes(Encoding.GetEncoding(932).GetBytes(curline));
}
textStream.WriteByte(0x00);
}
// Build new script file.
var bw = new BinaryWriter(new FileStream(fileInput + ".new", FileMode.Create));
// Write HEADER.
if (headerLength != 0) bw.Write(scriptBuffer.Slice(0, headerLength));
// Control bytes.
bw.Write(controlStream.ToArray());
// Text bytes.
bw.Write(textStream.ToArray());
bw.Close();
br.Close();
}
public static byte[] CompressChunk(Chunk chunk)
{
int numBlocks = (chunk.Uncompressed.Length + maxBlockSize - 1) / maxBlockSize;
if (numBlocks > 8)
throw new FormatException("Maximum block number exceeded");
ChunkHeader head = new ChunkHeader();
head.magic = -1641380927;
head.blocksize = maxBlockSize;
head.uncompressedsize = chunk.Uncompressed.Length;
int pos = 0;
MemoryStream mem = new MemoryStream();
List<Block> blockList = new List<Block>();
int startData = 16 + 8 * numBlocks;
mem.Seek(startData, SeekOrigin.Begin);
for (int i = 0; i < numBlocks; i++)
{
Block block = new Block();
byte[] result, temp;
if (i != numBlocks - 1)
{
block.uncompressedsize = maxBlockSize;
temp = new byte[maxBlockSize];
}
else
{
block.uncompressedsize = head.uncompressedsize - pos;
temp = new byte[block.uncompressedsize];
}
Buffer.BlockCopy(chunk.Uncompressed, pos, temp, 0, temp.Length);
result = LZOCompress(temp);
if (result.Length == 0)
throw new Exception("LZO compression error!");
block.compressedsize = result.Length;
mem.WriteBytes(result);
blockList.Add(block);
pos += maxBlockSize;
}
head.compressedsize = (int)mem.Length;
mem.Seek(0, SeekOrigin.Begin);
mem.WriteInt32(head.magic);
mem.WriteInt32(head.blocksize);
mem.WriteInt32(head.compressedsize);
mem.WriteInt32(head.uncompressedsize);
foreach (Block block in blockList)
{
mem.WriteInt32(block.compressedsize);
mem.WriteInt32(block.uncompressedsize);
}
return mem.ToArray();
}
public void Serialize(Stream output)
{
var endian = ByteOrder;
const uint headerSize = 32;
output.WriteUInt32(0x42424947, endian);
output.WriteUInt32(Version, endian);
var keys = new List<string>
{
""
};
var maxValueLength = 0;
var blob = new StringBuilder();
foreach (var file in Files)
{
keys.Add(file.Name);
foreach (var section in file.Sections)
{
keys.Add(section.Key);
foreach (var value in section.Value)
{
keys.Add(value.Key);
foreach (var item in value.Value)
{
if (item.Value != null)
{
blob.Append(item.Value + '\0');
maxValueLength = Math.Max(maxValueLength, item.Value.Length);
}
}
}
}
}
var huffmanEncoder = new Encoder();
huffmanEncoder.Build(blob.ToString());
keys = keys.Distinct().OrderBy(k => k.HashCrc32()).ToList();
var maxKeyLength = keys.Max(k => k.Length);
uint stringTableSize;
using (var data = new MemoryStream())
{
data.Position = 4;
data.WriteInt32(keys.Count, endian);
data.Position = 4 + 4 + (8 * keys.Count);
var offsets = new List<KeyValuePair<uint, uint>>();
foreach (var key in keys)
{
var offset = (uint) data.Position;
data.WriteUInt16((ushort) key.Length, endian);
data.WriteString(key, Encoding.UTF8);
offsets.Add(new KeyValuePair<uint, uint>(key.HashCrc32(), offset));
}
data.Position = 8;
foreach (var kv in offsets)
{
data.WriteUInt32(kv.Key, endian);
data.WriteUInt32(kv.Value - 8, endian);
}
data.Position = 0;
data.WriteUInt32((uint) data.Length, endian);
data.Position = 0;
stringTableSize = (uint) data.Length;
output.Seek(headerSize, SeekOrigin.Begin);
output.WriteFromStream(data, data.Length);
}
uint huffmanSize;
using (var data = new MemoryStream())
{
var pairs = huffmanEncoder.GetPairs();
data.WriteUInt16((ushort) pairs.Length, endian);
foreach (var pair in pairs)
{
data.WriteInt32(pair.Left, endian);
data.WriteInt32(pair.Right, endian);
}
data.Position = 0;
huffmanSize = (uint) data.Length;
output.Seek(headerSize + stringTableSize, SeekOrigin.Begin);
output.WriteFromStream(data, data.Length);
}
var bits = new BitArray(huffmanEncoder.TotalBits);
var bitOffset = 0;
uint indexSize;
using (var index = new MemoryStream())
{
var fileDataOffset = 2 + (Files.Count * 6);
var files = new List<KeyValuePair<ushort, int>>();
foreach (var file in Files.OrderBy(f => keys.IndexOf(f.Name)))
{
files.Add(new KeyValuePair<ushort, int>((ushort) keys.IndexOf(file.Name), fileDataOffset));
var sectionDataOffset = 2 + (file.Sections.Count * 6);
var sections = new List<KeyValuePair<ushort, int>>();
foreach (var section in file.Sections.OrderBy(s => keys.IndexOf(s.Key)))
{
sections.Add(new KeyValuePair<ushort, int>((ushort) keys.IndexOf(section.Key), sectionDataOffset));
var valueDataOffset = 2 + (section.Value.Count * 6);
var values = new List<KeyValuePair<ushort, int>>();
foreach (var value in section.Value.OrderBy(v => keys.IndexOf(v.Key)))
{
index.Position = fileDataOffset + sectionDataOffset + valueDataOffset;
values.Add(new KeyValuePair<ushort, int>((ushort) keys.IndexOf(value.Key), valueDataOffset));
index.WriteUInt16((ushort) value.Value.Count, endian);
valueDataOffset += 2;
foreach (var item in value.Value)
{
switch (item.Type)
{
case -1:
{
continue;
}
case 1:
{
index.WriteInt32((1 << 29) | bitOffset, endian);
break;
}
case 0:
case 2:
case 3:
case 4:
{
var type = item.Type;
if (OverrideCompileValueTypes >= 0)
{
type = OverrideCompileValueTypes;
}
index.WriteInt32((type << 29) | bitOffset, endian);
bitOffset += huffmanEncoder.Encode((item.Value ?? "") + '\0', bits, bitOffset);
break;
}
}
valueDataOffset += 4;
}
}
index.Position = fileDataOffset + sectionDataOffset;
index.WriteUInt16((ushort) values.Count, endian);
sectionDataOffset += 2;
foreach (var value in values)
{
index.WriteUInt16(value.Key, endian);
index.WriteInt32(value.Value, endian);
sectionDataOffset += 6;
}
sectionDataOffset += valueDataOffset;
}
index.Position = fileDataOffset;
index.WriteUInt16((ushort) sections.Count, endian);
fileDataOffset += 2;
foreach (var section in sections)
{
index.WriteUInt16(section.Key, endian);
index.WriteInt32(section.Value, endian);
fileDataOffset += 6;
}
fileDataOffset += sectionDataOffset;
}
index.Position = 0;
index.WriteUInt16((ushort) files.Count, endian);
foreach (var file in files)
{
index.WriteUInt16(file.Key, endian);
index.WriteInt32(file.Value, endian);
}
index.Position = 0;
indexSize = (uint) index.Length;
output.Seek(headerSize + stringTableSize + huffmanSize, SeekOrigin.Begin);
output.WriteFromStream(index, index.Length);
}
output.Seek(headerSize + stringTableSize + huffmanSize + indexSize, SeekOrigin.Begin);
output.WriteInt32(bits.Length, endian);
var bytes = new byte[(bits.Length - 1) / 8 + 1];
bits.CopyTo(bytes, 0);
output.WriteBytes(bytes);
output.Seek(8, SeekOrigin.Begin);
output.WriteInt32(maxKeyLength, endian);
output.WriteInt32(maxValueLength, endian);
output.WriteUInt32(stringTableSize, endian);
output.WriteUInt32(huffmanSize, endian);
output.WriteUInt32(indexSize, endian);
output.WriteInt32(bytes.Length, endian);
output.Seek(0, SeekOrigin.Begin);
output.WriteUInt32(0x666D726D, endian);
}
private void SezializeSaveFile(Stream stream, SaveFile save)
{
var saveDataStream = new MemoryStream();
var nameTableStream = new MemoryStream();
var newSave = new SaveFile
{
Header =
{
UnkInt1 = save.Header.UnkInt1
},
SaveDataHeader =
{
HeaderSize = save.SaveDataHeader.HeaderSize,
UnkInt2 = save.SaveDataHeader.UnkInt2
}
};
newSave.SaveData = FillSaveData();
saveDataStream.SerializeObject(newSave.SaveData, newSave.NameTable.Names);
nameTableStream.WriteInt32(newSave.NameTable.Names.Count);
for (var i = 0; i < newSave.NameTable.Names.Count; i++)
{
nameTableStream.WriteString(newSave.NameTable.Names[i]);
}
newSave.SaveDataHeader.SaveDataSize = (int) saveDataStream.Length + 12; // 12 = SaveDataHeader size
newSave.Header.FileSize = (int) saveDataStream.Length + 12 + (int) nameTableStream.Length;
stream.Seek(0, SeekOrigin.Begin);
// File header
stream.WriteInt32(newSave.Header.UnkInt1);
stream.WriteInt32(newSave.Header.FileSize);
// Save data header
stream.WriteInt32(newSave.SaveDataHeader.HeaderSize);
stream.WriteInt32(newSave.SaveDataHeader.SaveDataSize);
stream.WriteInt32(newSave.SaveDataHeader.UnkInt2);
// Save data
saveDataStream.WriteTo(stream);
//stream.WriteInt32(255);
// Name table
nameTableStream.WriteTo(stream);
}