public NsaIndex(Stream stream) { using (var memStream = new MemoryStream()) using (var memReader = new ExtendedBinaryReader(memStream)) { stream.CopyTo(memStream);//reading all bytes in stream to memStream memStream.Position = 0; Assembly = (GenomeAssembly)memReader.ReadByte(); Version = DataSourceVersion.Read(memReader); JsonKey = memReader.ReadAsciiString(); MatchByAllele = memReader.ReadBoolean(); IsArray = memReader.ReadBoolean(); SchemaVersion = memReader.ReadOptInt32(); IsPositional = memReader.ReadBoolean(); var chromCount = memReader.ReadOptInt32(); _chromBlocks = new Dictionary <ushort, List <NsaIndexBlock> >(chromCount); for (var i = 0; i < chromCount; i++) { var chromIndex = memReader.ReadOptUInt16(); var chunkCount = memReader.ReadOptInt32(); _chromBlocks[chromIndex] = new List <NsaIndexBlock>(chunkCount); for (var j = 0; j < chunkCount; j++) { _chromBlocks[chromIndex].Add(new NsaIndexBlock(memReader)); } } } }
public ChunkedIndex(Stream stream) { //reading the index in one shot var buffer = new byte[1048576]; var indexLength = stream.Read(buffer, 0, 1048576); using (var memStream = new MemoryStream(buffer, 0, indexLength)) using (var memReader = new ExtendedBinaryReader(memStream)) { Assembly = (GenomeAssembly)memReader.ReadByte(); Version = DataSourceVersion.Read(memReader); JsonKey = memReader.ReadAsciiString(); MatchByAllele = memReader.ReadBoolean(); IsArray = memReader.ReadBoolean(); SchemaVersion = memReader.ReadOptInt32(); IsPositional = memReader.ReadBoolean(); var chromCount = memReader.ReadOptInt32(); _chromChunks = new Dictionary <ushort, List <Chunk> >(chromCount); for (var i = 0; i < chromCount; i++) { var chromIndex = memReader.ReadOptUInt16(); var chunkCount = memReader.ReadOptInt32(); _chromChunks[chromIndex] = new List <Chunk>(chunkCount); for (var j = 0; j < chunkCount; j++) { _chromChunks[chromIndex].Add(new Chunk(memReader)); } } } }
private static object ReadValueType(Type valueType, ExtendedBinaryReader br, bool noAlign = false) { if (!noAlign) { br.AlignStream(); } if (valueType == typeof(string)) { return(br.ReadAlignedString()); } else if (valueType == typeof(Int32)) { return(br.ReadInt32()); } else if (valueType == typeof(UInt32)) { return(br.ReadUInt32()); } else if (valueType == typeof(Int64)) { return(br.ReadInt64()); } else if (valueType == typeof(UInt64)) { return(br.ReadUInt64()); } else if (valueType == typeof(Int16)) { return(br.ReadInt16()); } else if (valueType == typeof(UInt16)) { return(br.ReadUInt16()); } else if (valueType == typeof(Byte)) { return(br.ReadByte()); } else if (valueType == typeof(SByte)) { return(br.ReadSByte()); } else if (valueType == typeof(Boolean)) { return(br.ReadBoolean()); } else if (valueType == typeof(Double)) { return(br.ReadDouble()); } else if (valueType == typeof(Single)) { return(br.ReadSingle()); } else { throw new ArgumentException($"{valueType} is not a value type"); } }
private static (IDataSourceVersion Version, string JsonKey, bool IsArray) ReadHeader(Stream stream) { IDataSourceVersion version; string jsonKey; bool isArray; using (var reader = new ExtendedBinaryReader(stream, Encoding.UTF8, true)) { string identifier = reader.ReadString(); if (identifier != SaCommon.NgaIdentifier) { throw new InvalidDataException($"Expected the NGA identifier ({SaCommon.NgaIdentifier}), but found another value: ({identifier})"); } version = DataSourceVersion.Read(reader); jsonKey = reader.ReadString(); isArray = reader.ReadBoolean(); ushort schemaVersion = reader.ReadUInt16(); if (schemaVersion != SaCommon.SchemaVersion) { throw new UserErrorException($"Expected the schema version {SaCommon.SchemaVersion}, but found another value: ({schemaVersion}) for {jsonKey}"); } uint guard = reader.ReadUInt32(); if (guard != SaCommon.GuardInt) { throw new InvalidDataException($"Expected a guard integer ({SaCommon.GuardInt}), but found another value: ({guard})"); } } return(version, jsonKey, isArray); }
public static ReferenceMetadata Read(ExtendedBinaryReader reader) { var ucscName = reader.ReadAsciiString(); var ensemblName = reader.ReadAsciiString(); reader.ReadBoolean(); // TODO: Remove this when we update the reference file format return(new ReferenceMetadata(ucscName, ensemblName)); }
public static ReferenceMetadata Read(ExtendedBinaryReader reader) { string ucscName = reader.ReadAsciiString(); string ensemblName = reader.ReadAsciiString(); reader.ReadBoolean(); return(new ReferenceMetadata(ucscName, ensemblName)); }
/// <summary> /// reads the gene data from the binary reader /// </summary> public static Gene Read(ExtendedBinaryReader reader) { ushort referenceIndex = reader.ReadUInt16(); int start = reader.ReadOptInt32(); int end = reader.ReadOptInt32(); bool onReverseStrand = reader.ReadBoolean(); string symbol = reader.ReadAsciiString(); int hgncId = reader.ReadOptInt32(); var entrezId = CompactId.Read(reader); var ensemblId = CompactId.Read(reader); int mimNumber = reader.ReadOptInt32(); return(new Gene(referenceIndex, start, end, onReverseStrand, symbol, hgncId, entrezId, ensemblId, mimNumber)); }
// Methods public void Read(ExtendedBinaryReader reader) { NameOffset = reader.ReadInt64(); DataOffset = reader.ReadInt64(); ChildIDTableOffset = reader.ReadInt64(); ParentIndex = reader.ReadInt32(); GlobalIndex = reader.ReadInt32(); DataIndex = reader.ReadInt32(); ChildCount = reader.ReadUInt16(); HasData = reader.ReadBoolean(); FullPathSize = reader.ReadByte(); // Not counting this node in. }
// Methods public static void Read(HavokFile h, ExtendedBinaryReader reader) { // Header reader.JumpAhead(0x10); // A lot of this was gathered from the Max Havok exporter. byte bytesInPointer = reader.ReadByte(); reader.IsBigEndian = !reader.ReadBoolean(); byte reusePaddingOpt = reader.ReadByte(); byte emptyBaseClassOpt = reader.ReadByte(); // We jump around a lot here, but there's not really a much cleaner way to do it. reader.JumpTo(8, true); h.UserTag = reader.ReadUInt32(); h.ClassVersion = reader.ReadInt32(); reader.JumpAhead(4); uint sectionsCount = reader.ReadUInt32(); uint unknown1 = reader.ReadUInt32(); ulong padding1 = reader.ReadUInt64(); uint unknown2 = reader.ReadUInt32(); h.ContentsVersion = reader.ReadNullTerminatedString(); reader.JumpAhead(9); // Seems to be padding // Sections for (uint i = 0; i < sectionsCount; ++i) { string sectionName = new string(reader.ReadChars(0x10)); sectionName = sectionName.Replace("\0", string.Empty); // TODO reader.JumpAhead(0x20); } // Padding Checks if (padding1 != 0) { Console.WriteLine($"WARNING: Padding1 != 0 ({padding1})"); } // TODO throw new NotImplementedException(); }
public static object ReadByType(ExtendedBinaryReader reader, ArgumentType type) { switch (type) { case AT_Bool: return(reader.ReadBoolean()); case AT_Byte: return(reader.ReadByte()); case AT_Int16: return(reader.ReadInt16()); case AT_Int32: return(reader.ReadInt32()); case AT_Float: return(reader.ReadSingle()); case AT_String: return(reader.ReadStringElsewhere()); case AT_StringPtr: return(reader.ReadNullTerminatedStringPointer()); case AT_CodePointer: return(reader.ReadInt32()); case AT_DataReference: return(reader.ReadUInt32()); case AT_DataBlock: long position = reader.ReadUInt32(); long length = reader.ReadUInt32(); if (reader.Offset > position) { reader.Offset = (uint)position; } return(new StreamBlock(position, length)); default: return(null); } }
public CustomItem(ExtendedBinaryReader reader) { Id = reader.ReadAsciiString(); AnnotationType = reader.ReadAsciiString(); SaAltAllele = reader.ReadAsciiString(); AltAllele = SupplementaryAnnotationUtilities.ReverseSaReducedAllele(SaAltAllele); IsPositional = reader.ReadBoolean(); var numStringFields = reader.ReadOptInt32(); if (numStringFields > 0) { StringFields = new Dictionary <string, string>(); for (var i = 0; i < numStringFields; i++) { var key = reader.ReadAsciiString(); var value = reader.ReadAsciiString(); StringFields[key] = value; } } else { StringFields = null; } var numBoolFields = reader.ReadOptInt32(); if (numBoolFields > 0) { BooleanFields = new List <string>(); for (var i = 0; i < numBoolFields; i++) { BooleanFields.Add(reader.ReadAsciiString()); } } else { BooleanFields = null; } }
public NgaReader(Stream stream) { _nsaStream = stream; // read the whole file. Currently they are well under 2MB var compressedBytes = new byte[2 * 1024 * 1024]; var decompressedBytes = new byte[20 * 1024 * 1024]; var compressedSize = _nsaStream.Read(compressedBytes, 0, compressedBytes.Length); var zstd = new Zstandard(); var decompressedSize = zstd.Decompress(compressedBytes, compressedSize, decompressedBytes, decompressedBytes.Length); _memStream = new MemoryStream(decompressedBytes, 0, decompressedSize); _reader = new ExtendedBinaryReader(_memStream); Version = DataSourceVersion.Read(_reader); JsonKey = _reader.ReadAsciiString(); _isArray = _reader.ReadBoolean(); ushort schemaVersion = _reader.ReadOptUInt16(); if (schemaVersion != SaCommon.SchemaVersion) { throw new UserErrorException($"Expected schema version: {SaCommon.SchemaVersion}, observed: {schemaVersion} for {JsonKey}"); } }
/* * public TypeTree(TypeTree parent, uint format, int version, bool isArray, int size, int index, int flags, string type, string name) * { * this.parent = parent; * this.format = format; * * this.version = version; * this.isArray = isArray; * this.size = size; * this.index = index; * this.flags = flags; * this.type = type; * this.name = name; * } */ public static TypeTreeNode ReadTypeTree(uint format, ExtendedBinaryReader buf) { TypeTreeNode root = new TypeTreeNode(); if (format == 10 || format >= 12) { int nodesCount = buf.ReadInt32(); int stringBufferBytes = buf.ReadInt32(); int nodesize = format >= 19 ? 32 : 24; buf.BaseStream.Seek(nodesize * nodesCount, SeekOrigin.Current); byte[] stringData = buf.ReadBytes(stringBufferBytes); buf.BaseStream.Seek(-(nodesize * nodesCount + stringBufferBytes), SeekOrigin.Current); Stack <TypeTreeNode> stack = new Stack <TypeTreeNode>(); stack.Push(root); using (var stringReader = new ExtendedBinaryReader(new MemoryStream(stringData))) { for (int i = 0; i < nodesCount; i++) { short version = buf.ReadInt16(); byte depth = buf.ReadByte(); bool isArray = buf.ReadBoolean(); ushort typeIndex = buf.ReadUInt16(); string typeStr; if (buf.ReadUInt16() == 0) { stringReader.BaseStream.Position = typeIndex; typeStr = stringReader.ReadNullTerminatedString(); } else { typeStr = baseStrings.ContainsKey(typeIndex) ? baseStrings[typeIndex] : typeIndex.ToString(); } ushort nameIndex = buf.ReadUInt16(); string nameStr; if (buf.ReadUInt16() == 0) { stringReader.BaseStream.Position = nameIndex; nameStr = stringReader.ReadNullTerminatedString(); } else { nameStr = baseStrings.ContainsKey(nameIndex) ? baseStrings[nameIndex] : nameIndex.ToString(); } int size = buf.ReadInt32(); int index = buf.ReadInt32(); int flags = buf.ReadInt32(); ulong refTypeHash = 0; if (format >= 19) { refTypeHash = buf.ReadUInt64(); } TypeTreeNode t; if (depth == 0) { t = root; } else { while (stack.Count > depth) { stack.Pop(); } t = new TypeTreeNode(); stack.Peek().children.Add(t); stack.Push(t); } t.version = version; t.isArray = isArray; t.type = typeStr; t.name = nameStr; t.size = size; t.index = index; t.flags = flags; t.refTypeHash = refTypeHash; } } buf.BaseStream.Seek(stringBufferBytes, SeekOrigin.Current); } else { root.type = buf.ReadNullTerminatedString(); root.name = buf.ReadNullTerminatedString(); root.size = buf.ReadInt32(); root.index = buf.ReadInt32(); root.isArray = buf.ReadBoolean(); root.version = buf.ReadInt32(); root.flags = buf.ReadInt32(); int childCount = buf.ReadInt32(); for (int i = 0; i < childCount; i++) { root.children.Add(TypeTreeNode.ReadTypeTree(format, buf)); } } return(root); }
public TypeMetaData(uint format, ExtendedBinaryReader buf) { if (format >= 13) { hasTypeTrees = buf.ReadBoolean(); int types_count = buf.ReadInt32(); for (int i = 0; i < types_count; i++) { int classID = buf.ReadInt32(); int scriptID; if (format >= 17) { byte unk0 = buf.ReadByte(); scriptID = -1 - buf.ReadInt16(); } else { scriptID = classID; } ClassIDs.Add(new ClassInfo { scriptID = scriptID, classID = classID }); byte[] hash; if ((format < 16 && classID < 0) || (format >= 16 && classID == 114)) { hash = buf.ReadBytes(0x20); } else { hash = buf.ReadBytes(0x10); } Hashes[classID] = hash; if (hasTypeTrees) { TypeTrees.Add(classID, TypeTreeNode.ReadTypeTree(format, buf)); if (format >= 21) { int dependenciesCount = buf.ReadInt32(); var TypeDependencies = (int[])buf.ReadValueArray <int>(dependenciesCount); } } } } else { int fieldsCount = buf.ReadInt32(); for (int i = 0; i < fieldsCount; i++) { int classID = buf.ReadInt32(); TypeTrees.Add(classID, TypeTreeNode.ReadTypeTree(format, buf)); } } }
public override void Deserialize(ExtendedBinaryReader reader, Action <bool> valueFixup) { valueFixup(reader.ReadBoolean()); }
public override void Load(Stream fileStream) { // Header var reader = new ExtendedBinaryReader(fileStream, true); var entries = new List <FileEntry>(); uint sig = reader.ReadUInt32(); if (sig != Signature) { throw new InvalidSignatureException( Signature.ToString("X"), sig.ToString("X")); } uint fileTableOffset = reader.ReadUInt32(); uint fileTableLength = reader.ReadUInt32(); uint fileDataOffset = reader.ReadUInt32(); uint unknown5 = reader.ReadUInt32(); uint unknown6 = reader.ReadUInt32(); uint unknown7 = reader.ReadUInt32(); uint unknown8 = reader.ReadUInt32(); // File/Directory Entries long fileTableEnd = (fileTableOffset + fileTableLength); reader.JumpTo(fileTableOffset); while (fileStream.Position < fileTableEnd) { bool isDirectory = reader.ReadBoolean(); byte padding1 = reader.ReadByte(); if (padding1 != 0) { fileStream.Position -= 2; // go bacc break; } var fileEntry = new FileEntry() { IsDirectory = isDirectory, FileNameOffset = reader.ReadUInt16(), DataOffset = reader.ReadUInt32(), DataLength = reader.ReadUInt32(), DataUncompressedSize = reader.ReadUInt32() }; entries.Add(fileEntry); } // Entry Names uint fileNamesOffset = (uint)fileStream.Position; foreach (var entry in entries) { reader.JumpTo(fileNamesOffset + entry.FileNameOffset); entry.Name = reader.ReadNullTerminatedString(); } // Entry Data ArchiveDirectory dir = null; for (int i = 0; i < entries.Count; ++i) { var entry = entries[i]; // Directory if (entry.IsDirectory) { // Generate a HedgeLib directory entry if (i == 0) { continue; } if (entry.DataOffset >= i) { throw new Exception("Entry DataOffset is invalid"); } var dirEntry = new ArchiveDirectory(entry.Name); if (entry.DataOffset != 0) { var parentEntry = entries[(int)entry.DataOffset]; dirEntry.Parent = parentEntry.Directory; } if (dirEntry.Parent == null) { Data.Add(dirEntry); } else { dirEntry.Parent.Data.Add(dirEntry); } dir = entry.Directory = dirEntry; } // File else { if (dir == null) { throw new Exception("Tried to read a file not in a directory!"); } var data = new byte[entry.DataUncompressedSize]; int offset = 0, size = (int)entry.DataUncompressedSize; reader.JumpTo(entry.DataOffset + 2); using (var gzipStream = new DeflateStream(fileStream, CompressionMode.Decompress, true)) { while (offset < size) { offset += gzipStream.Read(data, offset, (size - offset)); } } // Generate a HedgeLib file entry dir.Data.Add(new ArchiveFile(entry.Name, data)); } } }
/// <summary> /// Reads a SEAnim from a stream /// </summary> /// <param name="Stream">The stream to read from</param> /// <returns>A SEAnim if successful, otherwise throws an error and returns null</returns> public static SEModel Read(Stream Stream) { // Create a new model var model = new SEModel(); // Setup a new reader using (ExtendedBinaryReader readFile = new ExtendedBinaryReader(Stream)) { // Magic var Magic = readFile.ReadChars(7); // Version var Version = readFile.ReadInt16(); // Header size var HeaderSize = readFile.ReadInt16(); // Check magic if (!Magic.SequenceEqual(new char[] { 'S', 'E', 'M', 'o', 'd', 'e', 'l' })) { // Bad file throw new Exception("Bad SEModel file, magic was invalid"); } // Data present flags var DataPresentFlags = readFile.ReadByte(); // Bone data present flags var BoneDataPresentFlags = readFile.ReadByte(); // Mesh data present flags var MeshDataPresentFlags = readFile.ReadByte(); // Read counts var BoneCount = readFile.ReadInt32(); var MeshCount = readFile.ReadInt32(); var MatCount = readFile.ReadInt32(); // Skip 3 reserved bytes readFile.BaseStream.Position += 3; // Read bone tag names List <string> BoneNames = new List <string>(); // Loop for (int i = 0; i < BoneCount; i++) { BoneNames.Add(readFile.ReadNullTermString()); } // Loop and read bones for (int i = 0; i < BoneCount; i++) { // Read bone flags (unused) var BoneFlags = readFile.ReadByte(); // Read bone index var ParentIndex = readFile.ReadInt32(); // Check for global matricies Vector3 GlobalPosition = Vector3.Zero; Quaternion GlobalRotation = Quaternion.Identity; // Check if (Convert.ToBoolean(BoneDataPresentFlags & (byte)SEModel_BoneDataPresenceFlags.SEMODEL_PRESENCE_GLOBAL_MATRIX)) { GlobalPosition = new Vector3(readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle()); GlobalRotation = new Quaternion(readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle()); } // Check for local matricies Vector3 LocalPosition = Vector3.Zero; Quaternion LocalRotation = Quaternion.Identity; // Check if (Convert.ToBoolean(BoneDataPresentFlags & (byte)SEModel_BoneDataPresenceFlags.SEMODEL_PRESENCE_LOCAL_MATRIX)) { LocalPosition = new Vector3(readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle()); LocalRotation = new Quaternion(readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle()); } // Check for scales Vector3 Scale = Vector3.One; // Check if (Convert.ToBoolean(BoneDataPresentFlags & (byte)SEModel_BoneDataPresenceFlags.SEMODEL_PRESENCE_SCALES)) { Scale = new Vector3(readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle()); } // Add the bone model.AddBone(BoneNames[i], ParentIndex, GlobalPosition, GlobalRotation, LocalPosition, LocalRotation, Scale); } // Loop and read meshes for (int i = 0; i < MeshCount; i++) { // Make a new submesh var mesh = new SEModelMesh(); // Read mesh flags (unused) var MeshFlags = readFile.ReadByte(); // Read counts var MatIndiciesCount = readFile.ReadByte(); var MaxSkinInfluenceCount = readFile.ReadByte(); var VertexCount = readFile.ReadInt32(); var FaceCount = readFile.ReadInt32(); // Loop and read positions for (int v = 0; v < VertexCount; v++) { mesh.AddVertex(new SEModelVertex() { Position = new Vector3(readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle()) }); } // Read uvlayers if (Convert.ToBoolean(MeshDataPresentFlags & (byte)SEModel_MeshDataPresenceFlags.SEMODEL_PRESENCE_UVSET)) { for (int v = 0; v < VertexCount; v++) { for (int l = 0; l < MatIndiciesCount; l++) { mesh.Verticies[v].UVSets.Add(new Vector2(readFile.ReadSingle(), readFile.ReadSingle())); } } } // Read normals if (Convert.ToBoolean(MeshDataPresentFlags & (byte)SEModel_MeshDataPresenceFlags.SEMODEL_PRESENCE_NORMALS)) { // Loop and read vertex normals for (int v = 0; v < VertexCount; v++) { mesh.Verticies[v].VertexNormal = new Vector3(readFile.ReadSingle(), readFile.ReadSingle(), readFile.ReadSingle()); } } // Read colors if (Convert.ToBoolean(MeshDataPresentFlags & (byte)SEModel_MeshDataPresenceFlags.SEMODEL_PRESENCE_COLOR)) { // Loop and read colors for (int v = 0; v < VertexCount; v++) { mesh.Verticies[v].VertexColor = new Color(readFile.ReadByte(), readFile.ReadByte(), readFile.ReadByte(), readFile.ReadByte()); } } // Read weights if (Convert.ToBoolean(MeshDataPresentFlags & (byte)SEModel_MeshDataPresenceFlags.SEMODEL_PRESENCE_WEIGHTS)) { for (int v = 0; v < VertexCount; v++) { // Read IDs and Values for (int l = 0; l < MaxSkinInfluenceCount; l++) { if (BoneCount <= 0xFF) { mesh.Verticies[v].Weights.Add(new SEModelWeight() { BoneIndex = readFile.ReadByte(), BoneWeight = readFile.ReadSingle() }); } else if (BoneCount <= 0xFFFF) { mesh.Verticies[v].Weights.Add(new SEModelWeight() { BoneIndex = readFile.ReadUInt16(), BoneWeight = readFile.ReadSingle() }); } else { mesh.Verticies[v].Weights.Add(new SEModelWeight() { BoneIndex = readFile.ReadUInt32(), BoneWeight = readFile.ReadSingle() }); } } } } // Loop and read faces for (int f = 0; f < FaceCount; f++) { if (VertexCount <= 0xFF) { mesh.AddFace(readFile.ReadByte(), readFile.ReadByte(), readFile.ReadByte()); } else if (VertexCount <= 0xFFFF) { mesh.AddFace(readFile.ReadUInt16(), readFile.ReadUInt16(), readFile.ReadUInt16()); } else { mesh.AddFace(readFile.ReadUInt32(), readFile.ReadUInt32(), readFile.ReadUInt32()); } } // Read material reference indicies for (int f = 0; f < MatIndiciesCount; f++) { mesh.AddMaterialIndex(readFile.ReadInt32()); } // Add the mesh model.AddMesh(mesh); } // Loop and read materials for (int m = 0; m < MatCount; m++) { var mat = new SEModelMaterial(); // Read the name mat.Name = readFile.ReadNullTermString(); // Read IsSimpleMaterial var IsSimpleMaterial = readFile.ReadBoolean(); // Read the material if (IsSimpleMaterial) { mat.MaterialData = new SEModelSimpleMaterial() { DiffuseMap = readFile.ReadNullTermString(), NormalMap = readFile.ReadNullTermString(), SpecularMap = readFile.ReadNullTermString() }; } // Add the material model.AddMaterial(mat); } } // Return result return(model); }
public override void Read(ExtendedBinaryReader reader, Action <bool> synchronizationCallback) { synchronizationCallback(reader.ReadBoolean()); }