protected NomadValue ReadRmlAttribute(BinaryStream _stream, NomadObject parent = null)
{
Context.State = ContextStateType.Member;
Context.ObjectIndex++;
var unk = (byte)_stream.ReadByte();
if (unk != 0)
{
throw new InvalidOperationException("Invalid RML attribute data.");
}
var nameIdx = DescriptorTag.Read(_stream, ReferenceType.Index);
var valIdx = DescriptorTag.Read(_stream, ReferenceType.Index);
var buffer = Utils.GetStringBuffer(_strings[valIdx]);
var result = new NomadValue(DataType.RML, buffer)
{
Id = _strings[nameIdx],
};
if (parent != null)
{
parent.Attributes.Add(result);
}
return(result);
}
public override void Serialize(BinaryStream stream)
{
Offset = (int)stream.Position;
var nChildren = Children.Count;
var nAttributes = Attributes.Count;
var nD = DescriptorTag.Create(nChildren);
nD.WriteTo(stream);
stream.Write(Hash);
if (nAttributes > 0)
{
// write attributes
foreach (var attribute in Attributes)
{
attribute.Serialize(stream, true); // hash
attribute.Serialize(stream, false); // data
}
}
else
{
// no attributes to write!
stream.WriteByte(0);
}
// now write the children out
foreach (var child in Children)
{
child.Serialize(stream);
}
}
public void Serialize(BinaryStream stream, int baseOffset = 0)
{
if (Type == DataType.RML)
{
throw new InvalidOperationException("Cannot serialize RML data directly!");
}
var ptr = (int)stream.Position;
var oldSize = Size;
if (Size > 0)
{
var writeData = true;
if (CanBeCached)
{
if (WriteCache.IsCached(this))
{
var cache = WriteCache.GetData(this);
if (cache.Size == Size)
{
stream.Position = (cache.Offset + 1);
var buf = stream.ReadBytes(cache.Size);
var key = Memory.GetCRC32(Buffer);
var bufKey = Memory.GetCRC32(buf);
stream.Position = ptr;
// slow as f**k, but there's no room for error
if (key == bufKey)
{
var nD = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset);
nD.WriteTo(stream, baseOffset);
writeData = false;
}
}
}
else
{
WriteCache.Cache(ptr, this);
}
}
if (writeData)
{
var nD = DescriptorTag.Create(Size);
nD.WriteTo(stream, baseOffset);
stream.Write(Buffer);
}
}
else
{
// nothing to write!
stream.WriteByte(0);
}
}
protected void WriteAttributeRmlData(BinaryStream stream, NomadValue attr)
{
var rmlBuffer = attr.Data.Buffer;
var rmlSize = DescriptorTag.Create(rmlBuffer.Length);
rmlSize.WriteTo(stream);
stream.Write(rmlBuffer);
}
protected void WriteAttributeData(BinaryStream stream, NomadValue attr, int baseOffset = 0)
{
var ptr = (int)stream.Position + baseOffset;
var data = attr.Data;
var type = data.Type;
var size = data.Size;
var buffer = data.Buffer;
if (data.Type == DataType.RML)
{
throw new InvalidOperationException("Cannot serialize RML data directly!");
}
var oldSize = size;
var attrData = (Format != FormatType.Resource)
? Utils.GetAttributeDataMiniBuffer(buffer, type)
: Utils.GetAttributeDataBuffer(buffer, type);
size = attrData.Length;
var writeData = true;
if (size > 4)
{
// return cached instance, else cache it and return empty
var cache = WriteCache.PreCache(ptr, attrData, size);
if (!cache.IsEmpty)
{
// sizes must match
if (cache.Size == size)
{
var nD = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset);
nD.WriteTo(stream, baseOffset);
writeData = false;
}
}
}
if (writeData)
{
var nD = DescriptorTag.Create(size);
nD.WriteTo(stream);
stream.Write(attrData);
}
}
protected void WriteObject_FmtA(BinaryStream stream, NomadObject obj)
{
Context.State = ContextStateType.Object;
Context.ObjectIndex++;
var ptr = (int)stream.Position;
var idx = NomadCache.Find(obj);
if (idx != -1)
{
var cached = NomadCache.Refs[idx];
var tag = DescriptorTag.CreateReference(Context.GetIdx(cached), ReferenceType.Index);
tag.WriteTo(stream);
}
else
{
var nChildren = DescriptorTag.Create(obj.Children.Count);
var nAttributes = DescriptorTag.Create(obj.Attributes.Count);
Context.AddRef(obj, ptr);
nChildren.WriteTo(stream);
stream.Write(obj.Id.Hash);
if (obj.IsRml)
{
WriteRmlData(stream, obj);
}
else
{
nAttributes.WriteTo(stream);
Context.State = ContextStateType.Member;
foreach (var attr in obj.Attributes)
{
WriteAttribute_FmtA(stream, attr);
}
foreach (var child in obj.Children)
{
WriteObject_FmtA(stream, child);
}
}
}
}
protected void WriteAttributesData_FmtB(BinaryStream stream, List <NomadValue> attributes, int baseOffset)
{
var ptr = (int)stream.Position + baseOffset;
var nAttrs = attributes.Count;
if (nAttrs > 0)
{
var attrData = new byte[nAttrs * 4];
using (var bs = new BinaryStream(attrData))
{
foreach (var attr in attributes)
{
bs.Write(attr.Id.Hash);
}
}
var cache = WriteCache.PreCache(ptr, attrData, nAttrs);
if (!cache.IsEmpty)
{
var ndAttrs = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset);
ndAttrs.WriteTo(stream, baseOffset);
}
else
{
var count = DescriptorTag.Create(nAttrs);
count.WriteTo(stream);
stream.Write(attrData);
}
foreach (var attr in attributes)
{
WriteAttribute_FmtB(stream, attr, baseOffset);
}
}
else
{
// nothing to write
stream.WriteByte(0);
}
}
protected void WriteRmlData(BinaryStream stream, NomadObject data)
{
byte[] rmlBuffer = null;
using (var bs = new BinaryStream(1024))
{
var rmlData = new NomadRmlSerializer();
rmlData.Serialize(bs, data);
rmlBuffer = bs.ToArray();
}
var size = DescriptorTag.Create(rmlBuffer.Length);
var next = DescriptorTag.Create(size);
next.WriteTo(stream);
size.WriteTo(stream);
stream.Write(rmlBuffer, 0, rmlBuffer.Length);
}
public void Deserialize(BinaryStream stream)
{
var ptr = (int)stream.Position;
var nD = DescriptorTag.Read(stream, ReferenceType.Offset);
if (nD.IsOffset)
{
stream.Position = nD.Value;
Deserialize(stream);
// move past offset
stream.Position = (ptr + nD.Size);
}
else
{
var size = nD.Value;
Buffer = new byte[size];
stream.Read(Buffer, 0, size);
}
}
private void WriteAttributeHashes(BinaryStream stream)
{
var ptr = (int)stream.Position;
var nAttrs = Attributes.Count;
if (nAttrs > 0)
{
var attrHBuf = new byte[(nAttrs * 4) + 1];
using (var buf = new BinaryStream(attrHBuf))
{
buf.WriteByte(nAttrs);
foreach (var attr in Attributes)
{
attr.Serialize(buf, true);
}
}
if (WriteCache.IsCached(attrHBuf, nAttrs))
{
var cache = WriteCache.GetData(attrHBuf, nAttrs);
var nhD = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset);
nhD.WriteTo(stream);
}
else
{
WriteCache.Cache(ptr, attrHBuf, nAttrs);
stream.Write(attrHBuf);
}
}
else
{
// nothing to write
stream.WriteByte(0);
}
}
protected NomadObject ReadRmlObject(BinaryStream _stream, NomadObject parent = null)
{
Context.State = ContextStateType.Object;
Context.ObjectIndex++;
var nameIdx = DescriptorTag.Read(_stream, ReferenceType.Index);
var valIdx = DescriptorTag.Read(_stream, ReferenceType.Index);
var nAttrs = DescriptorTag.Read(_stream, ReferenceType.Index);
var nElems = DescriptorTag.Read(_stream, ReferenceType.Index);
_attrCount += nAttrs;
_elemCount += nElems;
var result = new NomadObject(true)
{
Id = _strings[nameIdx],
Tag = _strings[valIdx],
};
if (parent != null)
{
parent.Children.Add(result);
}
for (int n = 0; n < nAttrs; n++)
{
ReadRmlAttribute(_stream, result);
}
for (int o = 0; o < nElems; o++)
{
ReadRmlObject(_stream, result);
}
return(result);
}
protected void WriteObject_FmtB(BinaryStream stream, NomadObject obj)
{
Context.State = ContextStateType.Object;
Context.ObjectIndex++;
var ptr = (int)stream.Position;
var idx = NomadCache.Find(obj);
if (idx != -1)
{
var cached = NomadCache.Refs[idx];
var offset = (int)Context.GetPtr(cached);
var tag = DescriptorTag.CreateReference(offset, ReferenceType.Offset);
tag.WriteTo(stream);
}
else
{
var reference = new NomadReference(obj);
var cached = reference.Get();
Context.AddRef(cached, ptr);
var count = DescriptorTag.Create(obj.Children.Count);
count.WriteTo(stream);
stream.Write(obj.Id.Hash);
WriteAttributesList_FmtB(stream, obj);
foreach (var child in obj.Children)
{
WriteObject_FmtB(stream, child);
}
}
}
public override void Deserialize(BinaryStream stream)
{
var ptr = (int)stream.Position;
var nD = DescriptorTag.Read(stream, ReferenceType.Offset);
if (nD.IsOffset)
{
stream.Position = nD.Value;
Deserialize(stream);
stream.Position = (ptr + nD.Size);
}
else
{
Offset = ptr;
var nChildren = nD.Value;
Children = new List <NodeClass>(nChildren);
var hash = stream.ReadInt32();
var size = stream.ReadInt16();
var name = StringHasher.ResolveHash(hash);
if (name != null)
{
Name = name;
}
else
{
Hash = hash;
}
var attrsPtr = (int)stream.Position;
var next = (attrsPtr + size);
if (size != 0)
{
var nhD = DescriptorTag.Read(stream, ReferenceType.Offset);
var adjustPtr = false;
if (nhD.IsOffset)
{
stream.Position = nhD.Value;
// read again
nhD = DescriptorTag.Read(stream, ReferenceType.Offset);
if (nhD.IsOffset)
{
throw new InvalidOperationException("Cannot have nested offsets!");
}
// adjust ptr to attributes
attrsPtr += nhD.Size;
adjustPtr = true;
}
var nAttrs = nhD.Value;
Attributes = new List <NodeAttribute>(nAttrs);
for (int i = 0; i < nAttrs; i++)
{
var attr = new NodeAttribute(stream, Name);
Attributes.Add(attr);
}
// move to the attributes if needed
if (adjustPtr)
{
stream.Position = attrsPtr;
}
// deserialize attribute data
foreach (var attr in Attributes)
{
attr.Deserialize(stream);
}
}
else
{
throw new NotImplementedException("Zero-length nodes are not covered under TrumpCare™.");
}
if (stream.Position != next)
{
throw new InvalidOperationException("You dun f****d up, son!");
}
// read children
for (int n = 0; n < nChildren; n++)
{
var child = new NodeClass(stream);
Children.Add(child);
}
}
}
public override void Serialize(Stream stream, NomadObject data)
{
if (Context.State == ContextStateType.End)
{
Context.Reset();
}
if (data.Id != "RML_DATA")
{
throw new InvalidOperationException("RML data wasn't prepared before initializing.");
}
if ((data.Children.Count != 1) || (data.Attributes.Count != 0))
{
throw new InvalidOperationException("RML data is malformed and cannot be serialized properly.");
}
var _stream = (stream as BinaryStream)
?? new BinaryStream(stream);
var rmlRoot = data.Children[0];
if (!rmlRoot.IsRml)
{
throw new InvalidOperationException("You can't serialize non-RML data as RML data, dumbass!");
}
_strings.Clear();
var strLookup = new Dictionary <string, int>();
var strPtr = 0;
var getStrIdx = new Func <string, int>((str) => {
var ptr = 0;
if (str == null)
{
str = String.Empty;
}
if (strLookup.ContainsKey(str))
{
ptr = strLookup[str];
}
else
{
// add to lookup
ptr = strPtr;
strLookup.Add(str, strPtr);
// add to string table
_strings.Add(strPtr, str);
// must have null-terminator!
var strLen = 1;
if (str != null)
{
strLen += str.Length;
}
strPtr += strLen;
}
return(ptr);
});
var entries = new List <NomadData>();
var elemsCount = 1;
var attrsCount = 0;
entries.Add(rmlRoot);
// iterates through attributes then children (and children's children, etc.)
foreach (var nd in rmlRoot)
{
if (!nd.IsRml)
{
throw new InvalidOperationException("Can't serialize non-RML data!");
}
if (nd.IsAttribute)
{
++attrsCount;
}
else if (nd.IsObject)
{
++elemsCount;
}
entries.Add(nd);
}
// rough size estimate
var rmlSize = ((elemsCount * 4) + (attrsCount * 2));
var strTableLen = -1;
byte[] rmlBuffer = null;
using (var ms = new BinaryStream(rmlSize))
{
var writeInt = new Action <int>((ptr) => {
var nD = DescriptorTag.Create(ptr);
nD.WriteTo(ms);
});
var writeRml = new Action <NomadData>((nd) => {
var nameIdx = getStrIdx(nd.Id);
var valIdx = -1;
if (nd.IsObject)
{
Context.State = ContextStateType.Object;
Context.ObjectIndex++;
var obj = (NomadObject)nd;
valIdx = getStrIdx(obj.Tag);
writeInt(nameIdx);
writeInt(valIdx);
writeInt(obj.Attributes.Count);
writeInt(obj.Children.Count);
}
else if (nd.IsAttribute)
{
Context.State = ContextStateType.Member;
Context.MemberIndex++;
var attr = (NomadValue)nd;
valIdx = getStrIdx(attr.Data);
// required for attributes
ms.WriteByte(0);
writeInt(nameIdx);
writeInt(valIdx);
}
});
writeRml(rmlRoot);
// enumerates attributes, then children (+ nested children)
foreach (var rml in rmlRoot)
{
writeRml(rml);
}
// setup string table size
strTableLen = strPtr;
// write out string table
foreach (var kv in _strings)
{
var str = kv.Value;
var strLen = (str != null) ? str.Length : 0;
var strBuf = new byte[strLen + 1];
if (strLen > 0)
{
Encoding.UTF8.GetBytes(str, 0, strLen, strBuf, 0);
}
ms.Write(strBuf);
}
// commit buffer
rmlBuffer = ms.ToArray();
rmlSize = rmlBuffer.Length;
}
var bufSize = 5; // header + 3 small ints
// expand size as needed
if (strTableLen >= 254)
{
bufSize += 4;
}
if (elemsCount >= 254)
{
bufSize += 4;
}
if (attrsCount >= 254)
{
bufSize += 4;
}
// calculate the final size (hopefully)
bufSize += rmlSize;
byte[] result = null;
using (var ms = new BinaryStream(bufSize))
{
ms.WriteByte(0);
ms.WriteByte(Reserved);
DescriptorTag[] descriptors =
{
DescriptorTag.Create(strTableLen),
DescriptorTag.Create(elemsCount),
DescriptorTag.Create(attrsCount),
};
foreach (var desc in descriptors)
{
desc.WriteTo(ms);
}
// write RML data (+ string table)
ms.Write(rmlBuffer);
// profit!!!
result = ms.ToArray();
}
_stream.Write(result, 0, result.Length);
Context.State = ContextStateType.End;
}
public void Deserialize(BinaryStream stream, List <NodeObject> objRefs)
{
Offset = (int)stream.Position;
// define reference type just in case we f**k up somehow
var nD = DescriptorTag.Read(stream, ReferenceType.Index);
if (nD.Type == DescriptorType.Reference)
{
throw new InvalidOperationException("Cannot deserialize an object reference directly!");
}
var nChildren = nD.Value;
Children = new List <NodeObject>(nChildren);
var hash = stream.ReadInt32();
var name = StringHasher.ResolveHash(hash);
if (name != null)
{
Name = name;
}
else
{
Hash = hash;
}
// add a reference to this object
objRefs.Add(this);
var aD = DescriptorTag.Read(stream, ReferenceType.Index);
var nAttrs = aD.Value;
Attributes = new List <NodeAttribute>(nAttrs);
if (nAttrs > 0)
{
for (int i = 0; i < nAttrs; i++)
{
// hash and data inline
var attr = new NodeAttribute(stream, Name);
attr.Deserialize(stream);
Attributes.Add(attr);
}
}
if (nChildren > 0)
{
// read children
for (int n = 0; n < nChildren; n++)
{
var cP = (int)stream.Position;
var cD = DescriptorTag.Read(stream, ReferenceType.Index);
// rip
if (cD.IsIndex)
{
var idx = cD.Value;
var childRef = objRefs[idx];
Children.Add(childRef);
}
else
{
// move back
stream.Position = cP;
var child = new NodeObject(stream, objRefs);
Children.Add(child);
}
}
}
}
protected NomadObject ReadObject_FmtB(BinaryStream stream, NomadObject parent = null)
{
Context.State = ContextStateType.Object;
Context.ObjectIndex++;
var ptr = (int)stream.Position;
var nD = DescriptorTag.Read(stream, ReferenceType.Offset);
NomadObject result = null;
if (nD.IsOffset)
{
result = Context.GetRefByPtr(nD.Value) as NomadObject;
// this should never happen
if (result == null)
{
throw new InvalidDataException("Malformed data!");
}
}
else
{
var nChildren = nD.Value;
var hash = stream.ReadInt32();
var size = stream.ReadInt16();
if (size == 0)
{
throw new NotImplementedException("Zero-length nodes are not covered under TrumpCare(tm).");
}
var id = StringId.Parse(hash);
result = new NomadObject(id);
Context.AddRef(result, ptr);
var attrsPtr = (int)stream.Position;
var next = (attrsPtr + size);
var nhD = DescriptorTag.Read(stream, ReferenceType.Offset);
var adjustPtr = false;
if (nhD.IsOffset)
{
stream.Position = nhD.Value;
// adjust ptr to attributes
attrsPtr += nhD.Size;
adjustPtr = true;
// read again
nhD = DescriptorTag.Read(stream, ReferenceType.Offset);
if (nhD.IsOffset)
{
throw new InvalidOperationException("Cannot have nested offsets!");
}
}
var nAttrs = nhD.Value;
var hashes = new int[nAttrs];
// read attribute hash list
for (int i = 0; i < nAttrs; i++)
{
hashes[i] = stream.ReadInt32();
}
// move to the attributes if needed
if (adjustPtr)
{
stream.Position = attrsPtr;
}
// deserialize attributes
if (nAttrs > 0)
{
ReadAttributes_FmtB(stream, result, hashes);
}
if (stream.Position != next)
{
Context.LogDebug($"Something went wrong when reading attributes for '{result.Id}':");
Context.LogDebug($" - Expected to read {size} bytes but only read {stream.Position - attrsPtr}");
foreach (var attr in result.Attributes)
{
Context.LogDebug($" - '{attr.Id}' : {attr.Data.Type} ({attr.Data.Size} bytes)");
}
stream.Position = next;
}
// read children
for (int n = 0; n < nChildren; n++)
{
ReadObject_FmtB(stream, result);
}
}
if (parent != null)
{
parent.Children.Add(result);
}
return(result);
}
protected NomadObject ReadObject_FmtA(BinaryStream stream, NomadObject parent = null)
{
Context.State = ContextStateType.Object;
Context.ObjectIndex++;
var ptr = (int)stream.Position;
var nChildren = DescriptorTag.Read(stream, ReferenceType.Index);
if (nChildren.Type == DescriptorType.Reference)
{
throw new InvalidOperationException("Cannot deserialize an object reference directly!");
}
var hash = stream.ReadInt32();
var id = StringId.Parse(hash);
var result = new NomadObject(id);
Context.AddRef(result, ptr);
if (result.IsRml)
{
var next = DescriptorTag.Read(stream, ReferenceType.Index);
var rmlBase = (int)stream.Position;
var rmlSize = stream.ReadInt32();
var rmlBuffer = stream.ReadBytes(rmlSize);
using (var bs = new BinaryStream(rmlBuffer))
{
var rmlData = new NomadRmlSerializer();
var rml = rmlData.Deserialize(bs);
result.Children.Add(rml);
}
stream.Position = (rmlBase + next);
}
else
{
var nAttrs = DescriptorTag.Read(stream, ReferenceType.Index);
for (int i = 0; i < nAttrs; i++)
{
ReadAttribute_FmtA(stream, result);
}
for (int i = 0; i < nChildren; i++)
{
var cP = (int)stream.Position;
var cD = DescriptorTag.Read(stream, ReferenceType.Index);
// rip
if (cD.IsIndex)
{
var idx = cD.Value;
var childRef = Context.GetRefByIdx(idx) as NomadObject;
result.Children.Add(childRef);
}
else
{
// move back
stream.Position = cP;
ReadObject_FmtA(stream, result);
}
}
}
if (parent != null)
{
parent.Children.Add(result);
}
return(result);
}
public override NomadObject Deserialize(Stream stream)
{
if (Context.State == ContextStateType.End)
{
Context.Reset();
}
var _stream = (stream as BinaryStream)
?? new BinaryStream(stream);
if (_stream.ReadByte() != 0)
{
throw new InvalidOperationException("Invalid RML data.");
}
Reserved = (byte)_stream.ReadByte();
var sD = DescriptorTag.Read(_stream, ReferenceType.Index);
var strTableLen = sD.Value;
var strTablePtr = (_stream.Length - strTableLen);
var nElems = DescriptorTag.Read(_stream, ReferenceType.Index);
var nAttrs = DescriptorTag.Read(_stream, ReferenceType.Index);
// save position so we can parse strings first
var rmlDataPtr = (int)_stream.Position;
// move to beginning of string table
_stream.Position = strTablePtr;
var strPtr = 0;
// read in all strings and store them by their relative offset
// TODO: convert to more efficient reads from buffer
while (strPtr < strTableLen)
{
var str = "";
var strLen = 1; // include null-terminator
char c;
while ((c = _stream.ReadChar()) != '\0')
{
str += c;
++strLen;
}
_strings.Add(strPtr, str);
strPtr += strLen;
}
// parse RML data
_stream.Position = rmlDataPtr;
var rmlData = ReadRmlObject(_stream);
var result = new NomadObject(true)
{
Id = "RML_DATA"
};
result.Children.Add(rmlData);
Context.State = ContextStateType.End;
return(result);
}
public override void Serialize(BinaryStream stream)
{
Offset = (int)stream.Position;
var nChildren = Children.Count;
var nAttributes = Attributes.Count;
var writeData = true;
if (Size > 16)
{
if (WriteCache.IsCached(this))
{
var cache = WriteCache.GetData(this);
var obj = cache.Object as NodeClass;
if ((obj != null) && obj.Equals(this))
{
Debug.WriteLine($">> [Class:{Offset:X8}] Instance cached @ {cache.Offset:X8} with key {cache.Checksum:X8}");
var nD = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset);
nD.WriteTo(stream);
writeData = false;
}
else
{
Debug.WriteLine($">> [Class:{Offset:X8}] !!! FALSE POSITIVE !!!");
}
}
else
{
Debug.WriteLine($">> [Class:{Offset:X8}] Caching new instance with key {GetHashCode():X8}");
WriteCache.Cache(Offset, this);
}
}
if (writeData)
{
var nD = DescriptorTag.Create(nChildren);
nD.WriteTo(stream);
stream.Write(Hash);
// skip size parameter for now
stream.Position += 2;
var attrsPtr = stream.Position;
if (nAttributes > 0)
{
WriteAttributeHashes(stream);
// write attribute data
foreach (var attribute in Attributes)
{
attribute.Serialize(stream);
}
}
else
{
// no attributes to write!
stream.WriteByte(0);
}
var childrenPtr = stream.Position;
var attrsSize = (int)(childrenPtr - attrsPtr);
if (attrsSize > 65535)
{
throw new InvalidOperationException("Attribute data too large.");
}
// write attributes size
stream.Position = (attrsPtr - 2);
stream.Write((short)attrsSize);
// now write the children out
stream.Position = childrenPtr;
foreach (var child in Children)
{
child.Serialize(stream);
}
}
}
