internal unsafe MatNode(MatReader matReader, BinaryReader reader, long offset, MatDataTag tag, bool lazy) { // TODO: Completely refactor this method. this.matReader = matReader; // int originalBytes = tag.NumberOfBytes; contents = new Dictionary <string, MatNode>(); this.startOffset = offset; this.reader = reader; if (tag.DataType == MatDataType.miCOMPRESSED) { compressed = true; // Read zlib's streams with Deflate using a little trick // http://george.chiramattel.com/blog/2007/09/deflatestream-block-length-does-not-match.html reader.ReadBytes(2); // ignore zlib headers reader = new BinaryReader(new DeflateStream(reader.BaseStream, CompressionMode.Decompress, true)); readBytes += 8; if (!reader.Read(out tag)) { throw new NotSupportedException("Invalid reader at position " + readBytes + "."); } } if (tag.DataType != MatDataType.miMATRIX) { throw new NotSupportedException("Unexpected data type at position " + readBytes + "."); } readBytes += 8; MatDataTag flagsTag; if (!reader.Read(out flagsTag)) { throw new NotSupportedException("Invalid flags tag at position " + readBytes + "."); } if (flagsTag.DataType != MatDataType.miUINT32) { throw new NotSupportedException("Unexpected flags data type at position " + readBytes + "."); } readBytes += 8; ArrayFlags flagsElement; if (!reader.Read(out flagsElement)) { throw new NotSupportedException("Invalid flags element at position " + readBytes + "."); } if (flagsElement.Class == MatArrayType.mxOBJECT_CLASS) { throw new NotSupportedException("Unexpected object class flag at position " + readBytes + "."); } readBytes += 8; MatDataTag dimensionsTag; if (!reader.Read(out dimensionsTag)) { throw new NotSupportedException("Invalid dimensions tag at position " + readBytes + "."); } if (dimensionsTag.DataType != MatDataType.miINT32) { throw new NotSupportedException("Invalid dimensions data type at position " + readBytes + "."); } int numberOfDimensions = (int)dimensionsTag.NumberOfBytes / 4; dimensions = new int[numberOfDimensions]; for (int i = dimensions.Length - 1; i >= 0; i--) { dimensions[i] = reader.ReadInt32(); } readBytes += dimensions.Length * 4; readBytes += 8; MatDataTag nameTag; if (!reader.Read(out nameTag)) { throw new NotSupportedException("Invalid name tag at position " + readBytes + "."); } if (nameTag.DataType != MatDataType.miINT8) { throw new NotSupportedException("Invalid name data type at position " + readBytes + "."); } if (nameTag.IsSmallFormat) { Name = new String((sbyte *)nameTag.SmallData_Value, 0, nameTag.SmallData_NumberOfBytes); } else { readBytes += nameTag.NumberOfBytes; Name = new String(reader.ReadChars((int)nameTag.NumberOfBytes)); align(reader, nameTag.NumberOfBytes); } Name = Name.Trim(); if (flagsElement.Class == MatArrayType.mxSPARSE_CLASS) { readBytes += 8; MatDataTag irTag; if (!reader.Read(out irTag)) { throw new NotSupportedException("Invalid sparse row tag at position " + readBytes + "."); } // read ir int[] ir = new int[irTag.NumberOfBytes / 4]; for (int i = 0; i < ir.Length; i++) { ir[i] = reader.ReadInt32(); } align(reader, irTag.NumberOfBytes); readBytes += 8; MatDataTag icTag; if (!reader.Read(out icTag)) { throw new NotSupportedException("Invalid sparse column tag at position " + readBytes + "."); } // read ic int[] ic = new int[icTag.NumberOfBytes / 4]; for (int i = 0; i < ic.Length; i++) { ic[i] = reader.ReadInt32(); } align(reader, icTag.NumberOfBytes); // read values readBytes += 8; MatDataTag valuesTag; if (!reader.Read(out valuesTag)) { throw new NotSupportedException("Invalid values tag at position " + readBytes + "."); } MatDataType matType = valuesTag.DataType; type = MatReader.Translate(matType); typeSize = Marshal.SizeOf(type); length = valuesTag.NumberOfBytes / typeSize; bytes = valuesTag.NumberOfBytes; byte[] rawData = reader.ReadBytes(bytes); align(reader, rawData.Length); if (matType == MatDataType.miINT64 || matType == MatDataType.miUINT64) { for (int i = 7; i < rawData.Length; i += 8) { byte b = rawData[i]; bool bit = (b & (1 << 6)) != 0; if (bit) { rawData[i] |= 1 << 7; } else { rawData[i] = (byte)(b & ~(1 << 7)); } } } Array array = Array.CreateInstance(type, length); Buffer.BlockCopy(rawData, 0, array, 0, rawData.Length); value = new MatSparse(ir, ic, array); } else if (flagsElement.Class == MatArrayType.mxCELL_CLASS) { int readBytes2 = 0; int toRead = tag.NumberOfBytes - readBytes; int cellI = 0; while (readBytes2 < toRead) { // Read first MAT data element MatDataTag elementTag; if (!reader.Read(out elementTag)) { throw new NotSupportedException("Invalid element tag at position " + readBytes + "."); } // Create a new node from the current position MatNode node = new MatNode(matReader, reader, offset, elementTag, false); node.Name = (cellI++).ToString(); contents.Add(node.Name, node); readBytes2 += elementTag.NumberOfBytes + 8; } } else if (flagsElement.Class == MatArrayType.mxSTRUCT_CLASS) { MatDataTag fieldNameLengthTag; if (!reader.Read(out fieldNameLengthTag)) { throw new NotSupportedException("Invalid struct field name length tag at position " + readBytes + "."); } if (!fieldNameLengthTag.IsSmallFormat) { throw new NotSupportedException("Small format struct field name length is not supported at position " + readBytes + "."); } int fieldNameLength = *(int *)fieldNameLengthTag.SmallData_Value; if (fieldNameLengthTag.DataType != MatDataType.miINT32) { throw new NotSupportedException("Unexpected struct field name length data type at position " + readBytes + "."); } MatDataTag fieldNameTag; if (!reader.Read(out fieldNameTag)) { throw new NotSupportedException("Invalid struct field name at position " + readBytes + "."); } if (fieldNameTag.DataType != MatDataType.miINT8) { throw new NotSupportedException("Unexpected struct field name data type at position " + readBytes + "."); } int fields = fieldNameTag.NumberOfBytes / fieldNameLength; string[] names = new string[fields]; for (int i = 0; i < names.Length; i++) { char[] charNames = reader.ReadChars(fieldNameLength); int terminator = Array.IndexOf(charNames, '\0'); names[i] = new String(charNames, 0, terminator); } align(reader, fieldNameTag.NumberOfBytes); for (int i = 0; i < names.Length; i++) { Debug.WriteLine("reading " + names[i]); // Read first MAT data element MatDataTag elementTag; if (!reader.Read(out elementTag)) { throw new NotSupportedException("Invalid struct element at position " + readBytes + "."); } if (elementTag.DataType == MatDataType.miINT32) { throw new NotSupportedException("Unexpected struct element data type at position " + readBytes + "."); } // Create a new node from the current position MatNode node = new MatNode(matReader, reader, offset, elementTag, false); node.Name = names[i]; contents.Add(node.Name, node); } } else { readBytes += 8; MatDataTag contentsTag; if (!reader.Read(out contentsTag)) { throw new NotSupportedException("Invalid contents tag at position " + readBytes + "."); } if (contentsTag.IsSmallFormat) { matType = contentsTag.SmallData_Type; if (matType == MatDataType.miUTF8) { value = new String((sbyte *)contentsTag.SmallData_Value, 0, contentsTag.SmallData_NumberOfBytes); } else { type = MatReader.Translate(matType); typeSize = Marshal.SizeOf(type); length = 1; for (int i = 0; i < dimensions.Length; i++) { length *= dimensions[i]; } var array = Array.CreateInstance(type, dimensions); byte[] rawData = new byte[4]; for (int i = 0; i < rawData.Length; i++) { rawData[i] = contentsTag.SmallData_Value[i]; } Buffer.BlockCopy(rawData, 0, array, 0, length); if (matReader.Transpose) { array = array.Transpose(Matrix.Indices(dimensions.Length, 0)); } value = array; } } else { matType = contentsTag.DataType; if (matType == MatDataType.miMATRIX) { // Create a new node from the current position value = new MatNode(matReader, reader, offset, contentsTag, false); } else if (matType == MatDataType.miUTF8) { char[] utf8 = reader.ReadChars(contentsTag.NumberOfBytes); value = new String(utf8); align(reader, utf8.Length); } else { type = MatReader.Translate(matType); typeSize = Marshal.SizeOf(type); length = contentsTag.NumberOfBytes / typeSize; bytes = contentsTag.NumberOfBytes; if (!lazy) { value = read(reader); } } } } if (!compressed && lazy) { matrixOffset = reader.BaseStream.Position; } }
public static DataTable Load(Stream stream, TableFormat format) { switch (format) { case TableFormat.SerializedXml: { XmlSerializer serializer = new XmlSerializer(typeof(DataTable)); return((DataTable)serializer.Deserialize(stream)); } case TableFormat.SerializedBin: { BinaryFormatter serializer = new BinaryFormatter(); return((DataTable)serializer.Deserialize(stream)); } case TableFormat.OctaveMatFile: { MatReader reader = new MatReader(stream); return(reader.Fields.First().Value.GetValue <double[, ]>().ToTable()); } case TableFormat.OpenDocument: { ExcelReader reader = new ExcelReader(stream, true); string ws = reader.GetWorksheetList().First(); return(reader.GetWorksheet(ws)); } case TableFormat.OlderExcel: { ExcelReader reader = new ExcelReader(stream, false); string ws = reader.GetWorksheetList().First(); return(reader.GetWorksheet(ws)); } case TableFormat.Csv: { CsvReader reader = new CsvReader(new StreamReader(stream), true); return(reader.ToTable()); } case TableFormat.Tsv: { CsvReader reader = new CsvReader(new StreamReader(stream), true, '\t'); return(reader.ToTable()); } case TableFormat.LibSVM: { SparseReader reader = new SparseReader(stream); /* int label = 0; * string description; * double[] sample; * * // Read a single sample from the file * sample = reader.ReadDense(out label, out description); * * // Read all other samples from the file * while (!reader.EndOfStream) * { * sample = reader.ReadDense(out label, out description); * } */ return(DataTable.Load(reader)); } case TableFormat.Idx: { IdxReader reader = new IdxReader(stream); return((DataTable)reader.ReadToEndAsVectors().ToTable()); //throw new NotSupportedException(); // return new DataTable(); } case TableFormat.CSharp: throw new NotSupportedException(); // return new DataTable(); } }