public void Serialize(Stream output)
{
var endian = this.Endian;
const uint headerSize = 32;
output.WriteValueU32(0x42424947, endian);
output.WriteValueU32(this.Version, endian);
var keys = new List<string>() {""};
int maxValueLength = 0;
var blob = new StringBuilder();
foreach (var file in this.Files)
{
keys.Add(file.Name);
foreach (var section in file.Sections)
{
keys.Add(section.Key);
foreach (var value in section.Value)
{
keys.Add(value.Key);
foreach (var item in value.Value)
{
if (item.Value != null)
{
blob.Append(item.Value + '\0');
maxValueLength = Math.Max(maxValueLength, item.Value.Length);
}
}
}
}
}
var huffmanEncoder = new Huffman.Encoder();
huffmanEncoder.Build(blob.ToString());
keys = keys.Distinct().OrderBy(k => k.HashCrc32()).ToList();
int maxKeyLength = keys.Max(k => k.Length);
uint stringTableSize;
using (var data = new MemoryStream())
{
data.Position = 4;
data.WriteValueS32(keys.Count, endian);
data.Position = 4 + 4 + (8 * keys.Count);
var offsets = new List<KeyValuePair<uint, uint>>();
foreach (var key in keys)
{
var offset = (uint)data.Position;
data.WriteValueU16((ushort)key.Length, endian);
data.WriteString(key, Encoding.UTF8);
offsets.Add(new KeyValuePair<uint, uint>(key.HashCrc32(), offset));
}
data.Position = 8;
foreach (var kv in offsets)
{
data.WriteValueU32(kv.Key, endian);
data.WriteValueU32(kv.Value - 8, endian);
}
data.Position = 0;
data.WriteValueU32((uint)data.Length, endian);
data.Position = 0;
stringTableSize = (uint)data.Length;
output.Seek(headerSize, SeekOrigin.Begin);
output.WriteFromStream(data, data.Length);
}
uint huffmanSize;
using (var data = new MemoryStream())
{
var pairs = huffmanEncoder.GetPairs();
data.WriteValueU16((ushort)pairs.Length, endian);
foreach (var pair in pairs)
{
data.WriteValueS32(pair.Left, endian);
data.WriteValueS32(pair.Right, endian);
}
data.Position = 0;
huffmanSize = (uint)data.Length;
output.Seek(headerSize + stringTableSize, SeekOrigin.Begin);
output.WriteFromStream(data, data.Length);
}
var bits = new BitArray(huffmanEncoder.TotalBits);
var bitOffset = 0;
uint indexSize;
using (var index = new MemoryStream())
{
var fileDataOffset = 2 + (this.Files.Count * 6);
var files = new List<KeyValuePair<ushort, int>>();
foreach (var file in this.Files.OrderBy(f => keys.IndexOf(f.Name)))
{
files.Add(new KeyValuePair<ushort, int>(
(ushort)keys.IndexOf(file.Name), fileDataOffset));
var sectionDataOffset = 2 + (file.Sections.Count * 6);
var sections = new List<KeyValuePair<ushort, int>>();
foreach (var section in file.Sections.OrderBy(s => keys.IndexOf(s.Key)))
{
sections.Add(new KeyValuePair<ushort, int>(
(ushort)keys.IndexOf(section.Key), sectionDataOffset));
var valueDataOffset = 2 + (section.Value.Count * 6);
var values = new List<KeyValuePair<ushort, int>>();
foreach (var value in section.Value.OrderBy(v => keys.IndexOf(v.Key)))
{
index.Position = fileDataOffset + sectionDataOffset + valueDataOffset;
values.Add(new KeyValuePair<ushort, int>(
(ushort)keys.IndexOf(value.Key), valueDataOffset));
index.WriteValueU16((ushort)value.Value.Count, endian);
valueDataOffset += 2;
foreach (var item in value.Value)
{
if (item.Type == 1)
{
index.WriteValueS32((1 << 29) | bitOffset, endian);
}
else if (item.Type == 0 || item.Type == 2 || item.Type == 3 || item.Type == 4)
{
index.WriteValueS32((item.Type << 29) | bitOffset, endian);
bitOffset += huffmanEncoder.Encode((item.Value ?? "") + '\0', bits, bitOffset);
}
valueDataOffset += 4;
}
}
index.Position = fileDataOffset + sectionDataOffset;
index.WriteValueU16((ushort)values.Count, endian);
sectionDataOffset += 2;
foreach (var value in values)
{
index.WriteValueU16(value.Key, endian);
index.WriteValueS32(value.Value, endian);
sectionDataOffset += 6;
}
sectionDataOffset += valueDataOffset;
}
index.Position = fileDataOffset;
index.WriteValueU16((ushort)sections.Count, endian);
fileDataOffset += 2;
foreach (var section in sections)
{
index.WriteValueU16(section.Key, endian);
index.WriteValueS32(section.Value, endian);
fileDataOffset += 6;
}
fileDataOffset += sectionDataOffset;
}
index.Position = 0;
index.WriteValueU16((ushort)files.Count, endian);
foreach (var file in files)
{
index.WriteValueU16(file.Key, endian);
index.WriteValueS32(file.Value, endian);
}
index.Position = 0;
indexSize = (uint)index.Length;
output.Seek(headerSize + stringTableSize + huffmanSize, SeekOrigin.Begin);
output.WriteFromStream(index, index.Length);
}
output.Seek(headerSize + stringTableSize + huffmanSize + indexSize, SeekOrigin.Begin);
output.WriteValueS32(bits.Length, endian);
var bytes = new byte[(bits.Length - 1) / 8 + 1];
bits.CopyTo(bytes, 0);
output.WriteBytes(bytes);
output.Seek(8, SeekOrigin.Begin);
output.WriteValueS32(maxKeyLength, endian);
output.WriteValueS32(maxValueLength, endian);
output.WriteValueU32(stringTableSize, endian);
output.WriteValueU32(huffmanSize, endian);
output.WriteValueU32(indexSize, endian);
output.WriteValueS32(bytes.Length, endian);
output.Seek(0, SeekOrigin.Begin);
output.WriteValueU32(0x666D726D, endian);
}
/// <summary>
/// Encoded an integer to save space.
/// </summary>
/// <remarks>
/// Integer can be encoded depending on the represented value to save space. Variable length integers always precede
/// an array/vector of a type of data that may vary in length. Longer numbers are encoded in little endian.
/// </remarks>
/// <specification>https://en.bitcoin.it/wiki/Protocol_specification#Variable_length_integer</specification>
/// <example>
/// nodejs: https://c9.io/raistlinthewiz/bitcoin-coinbase-varint-nodejs
/// </example>
/// <returns></returns>
public static byte[] VarInt(UInt32 value)
{
if (value < 0xfd)
return new[] { (byte)value };
byte[] result;
using (var stream = new MemoryStream())
{
if (value < 0xffff)
{
stream.WriteValueU8(0xfd);
stream.WriteValueU16(((UInt16)value).LittleEndian());
}
else if (value < 0xffffffff)
{
stream.WriteValueU8(0xfe);
stream.WriteValueU32(value.LittleEndian());
}
else
{
stream.WriteValueU8(0xff);
stream.WriteValueU16(((UInt16)value).LittleEndian());
}
result = stream.ToArray();
}
return result;
}
public Stream Serialize()
{
MemoryStream data = new MemoryStream();
uint dataSize = 0;
// Header stuff
data.WriteValueU32(0x39444350);
data.WriteValueEnum<PCD9.Format>(this.Format);
data.Seek(4, SeekOrigin.Current); //skip dataSize for now
data.WriteValueU32(this.Unknown0C);
data.WriteValueU16(this.Width);
data.WriteValueU16(this.Height);
data.WriteValueU8(this.BPP);
data.WriteValueU8((byte)(this.Mipmaps.Count - 1));
data.WriteValueU16(this.Unknown16);
// Data stuff
//TODO sort to make sure the biggest is first? will the order ever change?
for (int i = 0; i < this.Mipmaps.Count; i++)
{
// Write image data
data.WriteBytes(this.Mipmaps[i].Data);
// Add length to total
dataSize += (uint)this.Mipmaps[i].Data.Length;
}
// Write dataSize
data.Seek(8, SeekOrigin.Begin);
data.WriteValueU32(dataSize);
data.Position = 0;
return data;
}
/// <summary>
/// Creates a serialized string used in script signature.
/// </summary>
/// <remarks>
/// </remarks>
/// <example>
/// python: http://runnable.com/U3Mya-5oZntF5Ira/bitcoin-coinbase-serialize-string-python
/// nodejs: https://github.com/zone117x/node-stratum-pool/blob/dfad9e58c661174894d4ab625455bb5b7428881c/lib/util.js#L153
/// </example>
/// <param name="input"></param>
/// <returns></returns>
public static byte[] SerializeString(string input)
{
if (input.Length < 253)
return ArrayHelpers.Combine(new[] { (byte)input.Length }, Encoding.UTF8.GetBytes(input));
// if input string is >=253, we need need a special format.
byte[] result;
using (var stream = new MemoryStream())
{
if (input.Length < 0x10000)
{
stream.WriteValueU8(253);
stream.WriteValueU16(((UInt16)input.Length).LittleEndian()); // write packed length.
}
else if ((long)input.Length < 0x100000000)
{
stream.WriteValueU8(254);
stream.WriteValueU32(((UInt32)input.Length).LittleEndian()); // write packed length.
}
else
{
stream.WriteValueU8(255);
stream.WriteValueU16(((UInt16)input.Length).LittleEndian()); // write packed length.
}
stream.WriteString(input);
result = stream.ToArray();
}
return result;
}
public Stream Serialize()
{
MemoryStream data = new MemoryStream();
uint sectionCount = (uint)this.Sections.Count;
Stream[] resolvers = new MemoryStream[sectionCount]; // for serialized resolvers
uint unknown04_size = 0;
uint unknown08_size = 0;
// Write DRM Header
data.WriteValueU32(this.Version, this.Endianness);
data.WriteValueU32(0); // skip for now
data.WriteValueU32(0); // skip for now
data.WriteValueU32(0); // unknown0C
data.WriteValueU32(0); // unknown10
data.WriteValueU32(sectionCount, this.Endianness);
// Write DRM Section Headers
for (int i = 0; i < sectionCount; i++)
{
DRM.Section section = this.Sections[i];
// Serialize resolvers to get length, data will be used later
uint resolverLen;
if (section.Resolver != null)
{
resolvers[i] = section.Resolver.Serialize(this.Endianness);
resolverLen = (uint)resolvers[i].Length;
}
else
{
resolvers[i] = null;
resolverLen = 0;
}
data.WriteValueU32((uint)section.Data.Length, this.Endianness);
data.WriteValueU8((byte)section.Type);
data.WriteValueU8(section.Unknown05);
data.WriteValueU16(section.Unknown06, this.Endianness);
data.WriteValueU32((uint)section.Flags | (resolverLen << 8), this.Endianness);
data.WriteValueU32(section.Id, this.Endianness);
data.WriteValueU32(section.Unknown10, this.Endianness);
}
// Write Unknown08s
for (int i = 0; i < Unknown08s.Count; i++)
{
unknown08_size += ((uint)Unknown08s[i].Length + 1);
data.WriteStringZ(Unknown08s[i]);
}
// Write Unknown04s
for (int i = 0; i < Unknown04s.Count; i++)
{
unknown04_size += ((uint)Unknown04s[i].Length + 1);
data.WriteStringZ(Unknown04s[i]);
}
// Write DRM Section Data
for (int i = 0; i < sectionCount; i++)
{
if (resolvers[i] != null)
{
data.WriteFromStream(resolvers[i], resolvers[i].Length);
}
data.WriteFromStream(this.Sections[i].Data, this.Sections[i].Data.Length);
this.Sections[i].Data.Position = 0;
}
// Go back and write unknowns length
data.Seek(4, SeekOrigin.Begin);
data.WriteValueU32(unknown04_size);
data.WriteValueU32(unknown08_size);
data.Position = 0;
return data;
}
