protected NomadValue ReadRmlAttribute(BinaryStream _stream, NomadObject parent = null) { Context.State = ContextStateType.Member; Context.ObjectIndex++; var unk = (byte)_stream.ReadByte(); if (unk != 0) { throw new InvalidOperationException("Invalid RML attribute data."); } var nameIdx = DescriptorTag.Read(_stream, ReferenceType.Index); var valIdx = DescriptorTag.Read(_stream, ReferenceType.Index); var buffer = Utils.GetStringBuffer(_strings[valIdx]); var result = new NomadValue(DataType.RML, buffer) { Id = _strings[nameIdx], }; if (parent != null) { parent.Attributes.Add(result); } return(result); }
public override void Serialize(BinaryStream stream) { Offset = (int)stream.Position; var nChildren = Children.Count; var nAttributes = Attributes.Count; var nD = DescriptorTag.Create(nChildren); nD.WriteTo(stream); stream.Write(Hash); if (nAttributes > 0) { // write attributes foreach (var attribute in Attributes) { attribute.Serialize(stream, true); // hash attribute.Serialize(stream, false); // data } } else { // no attributes to write! stream.WriteByte(0); } // now write the children out foreach (var child in Children) { child.Serialize(stream); } }
public void Serialize(BinaryStream stream, int baseOffset = 0) { if (Type == DataType.RML) { throw new InvalidOperationException("Cannot serialize RML data directly!"); } var ptr = (int)stream.Position; var oldSize = Size; if (Size > 0) { var writeData = true; if (CanBeCached) { if (WriteCache.IsCached(this)) { var cache = WriteCache.GetData(this); if (cache.Size == Size) { stream.Position = (cache.Offset + 1); var buf = stream.ReadBytes(cache.Size); var key = Memory.GetCRC32(Buffer); var bufKey = Memory.GetCRC32(buf); stream.Position = ptr; // slow as f**k, but there's no room for error if (key == bufKey) { var nD = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset); nD.WriteTo(stream, baseOffset); writeData = false; } } } else { WriteCache.Cache(ptr, this); } } if (writeData) { var nD = DescriptorTag.Create(Size); nD.WriteTo(stream, baseOffset); stream.Write(Buffer); } } else { // nothing to write! stream.WriteByte(0); } }
protected void WriteAttributeRmlData(BinaryStream stream, NomadValue attr) { var rmlBuffer = attr.Data.Buffer; var rmlSize = DescriptorTag.Create(rmlBuffer.Length); rmlSize.WriteTo(stream); stream.Write(rmlBuffer); }
protected void WriteAttributeData(BinaryStream stream, NomadValue attr, int baseOffset = 0) { var ptr = (int)stream.Position + baseOffset; var data = attr.Data; var type = data.Type; var size = data.Size; var buffer = data.Buffer; if (data.Type == DataType.RML) { throw new InvalidOperationException("Cannot serialize RML data directly!"); } var oldSize = size; var attrData = (Format != FormatType.Resource) ? Utils.GetAttributeDataMiniBuffer(buffer, type) : Utils.GetAttributeDataBuffer(buffer, type); size = attrData.Length; var writeData = true; if (size > 4) { // return cached instance, else cache it and return empty var cache = WriteCache.PreCache(ptr, attrData, size); if (!cache.IsEmpty) { // sizes must match if (cache.Size == size) { var nD = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset); nD.WriteTo(stream, baseOffset); writeData = false; } } } if (writeData) { var nD = DescriptorTag.Create(size); nD.WriteTo(stream); stream.Write(attrData); } }
protected void WriteObject_FmtA(BinaryStream stream, NomadObject obj) { Context.State = ContextStateType.Object; Context.ObjectIndex++; var ptr = (int)stream.Position; var idx = NomadCache.Find(obj); if (idx != -1) { var cached = NomadCache.Refs[idx]; var tag = DescriptorTag.CreateReference(Context.GetIdx(cached), ReferenceType.Index); tag.WriteTo(stream); } else { var nChildren = DescriptorTag.Create(obj.Children.Count); var nAttributes = DescriptorTag.Create(obj.Attributes.Count); Context.AddRef(obj, ptr); nChildren.WriteTo(stream); stream.Write(obj.Id.Hash); if (obj.IsRml) { WriteRmlData(stream, obj); } else { nAttributes.WriteTo(stream); Context.State = ContextStateType.Member; foreach (var attr in obj.Attributes) { WriteAttribute_FmtA(stream, attr); } foreach (var child in obj.Children) { WriteObject_FmtA(stream, child); } } } }
protected void WriteAttributesData_FmtB(BinaryStream stream, List <NomadValue> attributes, int baseOffset) { var ptr = (int)stream.Position + baseOffset; var nAttrs = attributes.Count; if (nAttrs > 0) { var attrData = new byte[nAttrs * 4]; using (var bs = new BinaryStream(attrData)) { foreach (var attr in attributes) { bs.Write(attr.Id.Hash); } } var cache = WriteCache.PreCache(ptr, attrData, nAttrs); if (!cache.IsEmpty) { var ndAttrs = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset); ndAttrs.WriteTo(stream, baseOffset); } else { var count = DescriptorTag.Create(nAttrs); count.WriteTo(stream); stream.Write(attrData); } foreach (var attr in attributes) { WriteAttribute_FmtB(stream, attr, baseOffset); } } else { // nothing to write stream.WriteByte(0); } }
protected void WriteRmlData(BinaryStream stream, NomadObject data) { byte[] rmlBuffer = null; using (var bs = new BinaryStream(1024)) { var rmlData = new NomadRmlSerializer(); rmlData.Serialize(bs, data); rmlBuffer = bs.ToArray(); } var size = DescriptorTag.Create(rmlBuffer.Length); var next = DescriptorTag.Create(size); next.WriteTo(stream); size.WriteTo(stream); stream.Write(rmlBuffer, 0, rmlBuffer.Length); }
public void Deserialize(BinaryStream stream) { var ptr = (int)stream.Position; var nD = DescriptorTag.Read(stream, ReferenceType.Offset); if (nD.IsOffset) { stream.Position = nD.Value; Deserialize(stream); // move past offset stream.Position = (ptr + nD.Size); } else { var size = nD.Value; Buffer = new byte[size]; stream.Read(Buffer, 0, size); } }
private void WriteAttributeHashes(BinaryStream stream) { var ptr = (int)stream.Position; var nAttrs = Attributes.Count; if (nAttrs > 0) { var attrHBuf = new byte[(nAttrs * 4) + 1]; using (var buf = new BinaryStream(attrHBuf)) { buf.WriteByte(nAttrs); foreach (var attr in Attributes) { attr.Serialize(buf, true); } } if (WriteCache.IsCached(attrHBuf, nAttrs)) { var cache = WriteCache.GetData(attrHBuf, nAttrs); var nhD = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset); nhD.WriteTo(stream); } else { WriteCache.Cache(ptr, attrHBuf, nAttrs); stream.Write(attrHBuf); } } else { // nothing to write stream.WriteByte(0); } }
protected NomadObject ReadRmlObject(BinaryStream _stream, NomadObject parent = null) { Context.State = ContextStateType.Object; Context.ObjectIndex++; var nameIdx = DescriptorTag.Read(_stream, ReferenceType.Index); var valIdx = DescriptorTag.Read(_stream, ReferenceType.Index); var nAttrs = DescriptorTag.Read(_stream, ReferenceType.Index); var nElems = DescriptorTag.Read(_stream, ReferenceType.Index); _attrCount += nAttrs; _elemCount += nElems; var result = new NomadObject(true) { Id = _strings[nameIdx], Tag = _strings[valIdx], }; if (parent != null) { parent.Children.Add(result); } for (int n = 0; n < nAttrs; n++) { ReadRmlAttribute(_stream, result); } for (int o = 0; o < nElems; o++) { ReadRmlObject(_stream, result); } return(result); }
protected void WriteObject_FmtB(BinaryStream stream, NomadObject obj) { Context.State = ContextStateType.Object; Context.ObjectIndex++; var ptr = (int)stream.Position; var idx = NomadCache.Find(obj); if (idx != -1) { var cached = NomadCache.Refs[idx]; var offset = (int)Context.GetPtr(cached); var tag = DescriptorTag.CreateReference(offset, ReferenceType.Offset); tag.WriteTo(stream); } else { var reference = new NomadReference(obj); var cached = reference.Get(); Context.AddRef(cached, ptr); var count = DescriptorTag.Create(obj.Children.Count); count.WriteTo(stream); stream.Write(obj.Id.Hash); WriteAttributesList_FmtB(stream, obj); foreach (var child in obj.Children) { WriteObject_FmtB(stream, child); } } }
public override void Deserialize(BinaryStream stream) { var ptr = (int)stream.Position; var nD = DescriptorTag.Read(stream, ReferenceType.Offset); if (nD.IsOffset) { stream.Position = nD.Value; Deserialize(stream); stream.Position = (ptr + nD.Size); } else { Offset = ptr; var nChildren = nD.Value; Children = new List <NodeClass>(nChildren); var hash = stream.ReadInt32(); var size = stream.ReadInt16(); var name = StringHasher.ResolveHash(hash); if (name != null) { Name = name; } else { Hash = hash; } var attrsPtr = (int)stream.Position; var next = (attrsPtr + size); if (size != 0) { var nhD = DescriptorTag.Read(stream, ReferenceType.Offset); var adjustPtr = false; if (nhD.IsOffset) { stream.Position = nhD.Value; // read again nhD = DescriptorTag.Read(stream, ReferenceType.Offset); if (nhD.IsOffset) { throw new InvalidOperationException("Cannot have nested offsets!"); } // adjust ptr to attributes attrsPtr += nhD.Size; adjustPtr = true; } var nAttrs = nhD.Value; Attributes = new List <NodeAttribute>(nAttrs); for (int i = 0; i < nAttrs; i++) { var attr = new NodeAttribute(stream, Name); Attributes.Add(attr); } // move to the attributes if needed if (adjustPtr) { stream.Position = attrsPtr; } // deserialize attribute data foreach (var attr in Attributes) { attr.Deserialize(stream); } } else { throw new NotImplementedException("Zero-length nodes are not covered under TrumpCare™."); } if (stream.Position != next) { throw new InvalidOperationException("You dun f****d up, son!"); } // read children for (int n = 0; n < nChildren; n++) { var child = new NodeClass(stream); Children.Add(child); } } }
public override void Serialize(Stream stream, NomadObject data) { if (Context.State == ContextStateType.End) { Context.Reset(); } if (data.Id != "RML_DATA") { throw new InvalidOperationException("RML data wasn't prepared before initializing."); } if ((data.Children.Count != 1) || (data.Attributes.Count != 0)) { throw new InvalidOperationException("RML data is malformed and cannot be serialized properly."); } var _stream = (stream as BinaryStream) ?? new BinaryStream(stream); var rmlRoot = data.Children[0]; if (!rmlRoot.IsRml) { throw new InvalidOperationException("You can't serialize non-RML data as RML data, dumbass!"); } _strings.Clear(); var strLookup = new Dictionary <string, int>(); var strPtr = 0; var getStrIdx = new Func <string, int>((str) => { var ptr = 0; if (str == null) { str = String.Empty; } if (strLookup.ContainsKey(str)) { ptr = strLookup[str]; } else { // add to lookup ptr = strPtr; strLookup.Add(str, strPtr); // add to string table _strings.Add(strPtr, str); // must have null-terminator! var strLen = 1; if (str != null) { strLen += str.Length; } strPtr += strLen; } return(ptr); }); var entries = new List <NomadData>(); var elemsCount = 1; var attrsCount = 0; entries.Add(rmlRoot); // iterates through attributes then children (and children's children, etc.) foreach (var nd in rmlRoot) { if (!nd.IsRml) { throw new InvalidOperationException("Can't serialize non-RML data!"); } if (nd.IsAttribute) { ++attrsCount; } else if (nd.IsObject) { ++elemsCount; } entries.Add(nd); } // rough size estimate var rmlSize = ((elemsCount * 4) + (attrsCount * 2)); var strTableLen = -1; byte[] rmlBuffer = null; using (var ms = new BinaryStream(rmlSize)) { var writeInt = new Action <int>((ptr) => { var nD = DescriptorTag.Create(ptr); nD.WriteTo(ms); }); var writeRml = new Action <NomadData>((nd) => { var nameIdx = getStrIdx(nd.Id); var valIdx = -1; if (nd.IsObject) { Context.State = ContextStateType.Object; Context.ObjectIndex++; var obj = (NomadObject)nd; valIdx = getStrIdx(obj.Tag); writeInt(nameIdx); writeInt(valIdx); writeInt(obj.Attributes.Count); writeInt(obj.Children.Count); } else if (nd.IsAttribute) { Context.State = ContextStateType.Member; Context.MemberIndex++; var attr = (NomadValue)nd; valIdx = getStrIdx(attr.Data); // required for attributes ms.WriteByte(0); writeInt(nameIdx); writeInt(valIdx); } }); writeRml(rmlRoot); // enumerates attributes, then children (+ nested children) foreach (var rml in rmlRoot) { writeRml(rml); } // setup string table size strTableLen = strPtr; // write out string table foreach (var kv in _strings) { var str = kv.Value; var strLen = (str != null) ? str.Length : 0; var strBuf = new byte[strLen + 1]; if (strLen > 0) { Encoding.UTF8.GetBytes(str, 0, strLen, strBuf, 0); } ms.Write(strBuf); } // commit buffer rmlBuffer = ms.ToArray(); rmlSize = rmlBuffer.Length; } var bufSize = 5; // header + 3 small ints // expand size as needed if (strTableLen >= 254) { bufSize += 4; } if (elemsCount >= 254) { bufSize += 4; } if (attrsCount >= 254) { bufSize += 4; } // calculate the final size (hopefully) bufSize += rmlSize; byte[] result = null; using (var ms = new BinaryStream(bufSize)) { ms.WriteByte(0); ms.WriteByte(Reserved); DescriptorTag[] descriptors = { DescriptorTag.Create(strTableLen), DescriptorTag.Create(elemsCount), DescriptorTag.Create(attrsCount), }; foreach (var desc in descriptors) { desc.WriteTo(ms); } // write RML data (+ string table) ms.Write(rmlBuffer); // profit!!! result = ms.ToArray(); } _stream.Write(result, 0, result.Length); Context.State = ContextStateType.End; }
public void Deserialize(BinaryStream stream, List <NodeObject> objRefs) { Offset = (int)stream.Position; // define reference type just in case we f**k up somehow var nD = DescriptorTag.Read(stream, ReferenceType.Index); if (nD.Type == DescriptorType.Reference) { throw new InvalidOperationException("Cannot deserialize an object reference directly!"); } var nChildren = nD.Value; Children = new List <NodeObject>(nChildren); var hash = stream.ReadInt32(); var name = StringHasher.ResolveHash(hash); if (name != null) { Name = name; } else { Hash = hash; } // add a reference to this object objRefs.Add(this); var aD = DescriptorTag.Read(stream, ReferenceType.Index); var nAttrs = aD.Value; Attributes = new List <NodeAttribute>(nAttrs); if (nAttrs > 0) { for (int i = 0; i < nAttrs; i++) { // hash and data inline var attr = new NodeAttribute(stream, Name); attr.Deserialize(stream); Attributes.Add(attr); } } if (nChildren > 0) { // read children for (int n = 0; n < nChildren; n++) { var cP = (int)stream.Position; var cD = DescriptorTag.Read(stream, ReferenceType.Index); // rip if (cD.IsIndex) { var idx = cD.Value; var childRef = objRefs[idx]; Children.Add(childRef); } else { // move back stream.Position = cP; var child = new NodeObject(stream, objRefs); Children.Add(child); } } } }
protected NomadObject ReadObject_FmtB(BinaryStream stream, NomadObject parent = null) { Context.State = ContextStateType.Object; Context.ObjectIndex++; var ptr = (int)stream.Position; var nD = DescriptorTag.Read(stream, ReferenceType.Offset); NomadObject result = null; if (nD.IsOffset) { result = Context.GetRefByPtr(nD.Value) as NomadObject; // this should never happen if (result == null) { throw new InvalidDataException("Malformed data!"); } } else { var nChildren = nD.Value; var hash = stream.ReadInt32(); var size = stream.ReadInt16(); if (size == 0) { throw new NotImplementedException("Zero-length nodes are not covered under TrumpCare(tm)."); } var id = StringId.Parse(hash); result = new NomadObject(id); Context.AddRef(result, ptr); var attrsPtr = (int)stream.Position; var next = (attrsPtr + size); var nhD = DescriptorTag.Read(stream, ReferenceType.Offset); var adjustPtr = false; if (nhD.IsOffset) { stream.Position = nhD.Value; // adjust ptr to attributes attrsPtr += nhD.Size; adjustPtr = true; // read again nhD = DescriptorTag.Read(stream, ReferenceType.Offset); if (nhD.IsOffset) { throw new InvalidOperationException("Cannot have nested offsets!"); } } var nAttrs = nhD.Value; var hashes = new int[nAttrs]; // read attribute hash list for (int i = 0; i < nAttrs; i++) { hashes[i] = stream.ReadInt32(); } // move to the attributes if needed if (adjustPtr) { stream.Position = attrsPtr; } // deserialize attributes if (nAttrs > 0) { ReadAttributes_FmtB(stream, result, hashes); } if (stream.Position != next) { Context.LogDebug($"Something went wrong when reading attributes for '{result.Id}':"); Context.LogDebug($" - Expected to read {size} bytes but only read {stream.Position - attrsPtr}"); foreach (var attr in result.Attributes) { Context.LogDebug($" - '{attr.Id}' : {attr.Data.Type} ({attr.Data.Size} bytes)"); } stream.Position = next; } // read children for (int n = 0; n < nChildren; n++) { ReadObject_FmtB(stream, result); } } if (parent != null) { parent.Children.Add(result); } return(result); }
protected NomadObject ReadObject_FmtA(BinaryStream stream, NomadObject parent = null) { Context.State = ContextStateType.Object; Context.ObjectIndex++; var ptr = (int)stream.Position; var nChildren = DescriptorTag.Read(stream, ReferenceType.Index); if (nChildren.Type == DescriptorType.Reference) { throw new InvalidOperationException("Cannot deserialize an object reference directly!"); } var hash = stream.ReadInt32(); var id = StringId.Parse(hash); var result = new NomadObject(id); Context.AddRef(result, ptr); if (result.IsRml) { var next = DescriptorTag.Read(stream, ReferenceType.Index); var rmlBase = (int)stream.Position; var rmlSize = stream.ReadInt32(); var rmlBuffer = stream.ReadBytes(rmlSize); using (var bs = new BinaryStream(rmlBuffer)) { var rmlData = new NomadRmlSerializer(); var rml = rmlData.Deserialize(bs); result.Children.Add(rml); } stream.Position = (rmlBase + next); } else { var nAttrs = DescriptorTag.Read(stream, ReferenceType.Index); for (int i = 0; i < nAttrs; i++) { ReadAttribute_FmtA(stream, result); } for (int i = 0; i < nChildren; i++) { var cP = (int)stream.Position; var cD = DescriptorTag.Read(stream, ReferenceType.Index); // rip if (cD.IsIndex) { var idx = cD.Value; var childRef = Context.GetRefByIdx(idx) as NomadObject; result.Children.Add(childRef); } else { // move back stream.Position = cP; ReadObject_FmtA(stream, result); } } } if (parent != null) { parent.Children.Add(result); } return(result); }
public override NomadObject Deserialize(Stream stream) { if (Context.State == ContextStateType.End) { Context.Reset(); } var _stream = (stream as BinaryStream) ?? new BinaryStream(stream); if (_stream.ReadByte() != 0) { throw new InvalidOperationException("Invalid RML data."); } Reserved = (byte)_stream.ReadByte(); var sD = DescriptorTag.Read(_stream, ReferenceType.Index); var strTableLen = sD.Value; var strTablePtr = (_stream.Length - strTableLen); var nElems = DescriptorTag.Read(_stream, ReferenceType.Index); var nAttrs = DescriptorTag.Read(_stream, ReferenceType.Index); // save position so we can parse strings first var rmlDataPtr = (int)_stream.Position; // move to beginning of string table _stream.Position = strTablePtr; var strPtr = 0; // read in all strings and store them by their relative offset // TODO: convert to more efficient reads from buffer while (strPtr < strTableLen) { var str = ""; var strLen = 1; // include null-terminator char c; while ((c = _stream.ReadChar()) != '\0') { str += c; ++strLen; } _strings.Add(strPtr, str); strPtr += strLen; } // parse RML data _stream.Position = rmlDataPtr; var rmlData = ReadRmlObject(_stream); var result = new NomadObject(true) { Id = "RML_DATA" }; result.Children.Add(rmlData); Context.State = ContextStateType.End; return(result); }
public override void Serialize(BinaryStream stream) { Offset = (int)stream.Position; var nChildren = Children.Count; var nAttributes = Attributes.Count; var writeData = true; if (Size > 16) { if (WriteCache.IsCached(this)) { var cache = WriteCache.GetData(this); var obj = cache.Object as NodeClass; if ((obj != null) && obj.Equals(this)) { Debug.WriteLine($">> [Class:{Offset:X8}] Instance cached @ {cache.Offset:X8} with key {cache.Checksum:X8}"); var nD = DescriptorTag.CreateReference(cache.Offset, ReferenceType.Offset); nD.WriteTo(stream); writeData = false; } else { Debug.WriteLine($">> [Class:{Offset:X8}] !!! FALSE POSITIVE !!!"); } } else { Debug.WriteLine($">> [Class:{Offset:X8}] Caching new instance with key {GetHashCode():X8}"); WriteCache.Cache(Offset, this); } } if (writeData) { var nD = DescriptorTag.Create(nChildren); nD.WriteTo(stream); stream.Write(Hash); // skip size parameter for now stream.Position += 2; var attrsPtr = stream.Position; if (nAttributes > 0) { WriteAttributeHashes(stream); // write attribute data foreach (var attribute in Attributes) { attribute.Serialize(stream); } } else { // no attributes to write! stream.WriteByte(0); } var childrenPtr = stream.Position; var attrsSize = (int)(childrenPtr - attrsPtr); if (attrsSize > 65535) { throw new InvalidOperationException("Attribute data too large."); } // write attributes size stream.Position = (attrsPtr - 2); stream.Write((short)attrsSize); // now write the children out stream.Position = childrenPtr; foreach (var child in Children) { child.Serialize(stream); } } }