public SkimMatrix Read(string filename, int field, float scale) { var file = new FileInfo(Path.Combine(_path, filename)); Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field); Global.PrintFile.WriteFileInfo(file, true); if (!file.Exists) { throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName)); } var count = _mapping.Count; var matrix = new ushort[count][]; for (var i = 0; i < count; i++) { matrix[i] = new ushort[count]; } using (var reader = new BinaryReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read))) { var position = 0L; var length = reader.BaseStream.Length; for (var origin = 0; origin < count; origin++) { for (var destination = 0; destination < count; destination++) { if (position >= length) { goto end; } var value = reader.ReadUInt16(); // binary matrices are already mapped to consecutive zone indices, matching the zone index file matrix[origin][destination] = value; position += sizeof(ushort); } } } end: var skimMatrix = new SkimMatrix(matrix); return(skimMatrix); }
public SkimMatrix Read(string filename, int field, float scale) { var file = new FileInfo(Path.Combine(_path, filename)); Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field); Global.PrintFile.WriteFileInfo(file, true); if (!file.Exists) { throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName)); } var count = _mapping.Count; var matrix = new ushort[count][]; for (var i = 0; i < count; i++) { matrix[i] = new ushort[count]; } // List<float[]> rows; List <double[]> rows; // 20150703 JLB if (!_cache.TryGetValue(file.Name, out rows)) { // rows = new List<float[]>(); rows = new List <double[]>(); // 20150703 JLB using (var reader = new StreamReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read))) { string line; var current = 0; if (Global.TextSkimFilesContainHeaderRecord) { reader.ReadLine(); } while ((line = reader.ReadLine()) != null) { current++; try { char delimiter = Global.SkimDelimiter; var p1 = line.Split(new[] { delimiter }, StringSplitOptions.RemoveEmptyEntries); // var row = p1.Select(Convert.ToSingle).ToArray(); var row = p1.Select(Convert.ToDouble).ToArray(); // 20150703 JLB rows.Add(row); } catch (Exception e) { throw new InvalidSkimRowException(string.Format("Error parsing row in {0}.\nError in row {1}.\n{{{2}}}", file.FullName, current, line), e); } } } _cache.Add(file.Name, rows); } for (var i = 0; i < rows.Count; i++) { var row = rows[i]; try { // test code var index = Convert.ToInt32(row[0]); var origin = _mapping[index]; //var origin = _mapping[Convert.ToInt32(row[0])]; index = Convert.ToInt32(row[1]); var destination = _mapping[index]; //var destination = _mapping[Convert.ToInt32(row[1])]; var rawValue = Convert.ToSingle(row[field + FIELD_OFFSET]); if (rawValue > short.MaxValue / scale) { rawValue = short.MaxValue / (float)scale; } else if (rawValue < 0) { rawValue = 0; } var value = Convert.ToUInt16(rawValue * scale); matrix[origin][destination] = value; } catch (Exception e) { throw new ErrorReadingSkimFileException(string.Format("Error reading row {0}, {1}", i, string.Join(" ", row)), e); } } var skimMatrix = new SkimMatrix(matrix); return(skimMatrix); }
public SkimMatrix Read(string filename, int field, float scale) { Console.WriteLine("Reading {0}", filename); int hdf5NameEnd = filename.IndexOf("/"); // the first part of the name in the roster file is the hdf5 file: string HDFName = filename.Substring(0, hdf5NameEnd); //rename filename to be only the name of the skim inside of the time period file filename = filename.Substring(hdf5NameEnd); string hdfFile = _path + "\\" + HDFName; var dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY); var dataSet = H5D.open(dataFile, filename); var space = H5D.getSpace(dataSet); var size2 = H5S.getSimpleExtentDims(space); long nRows = size2[0]; long nCols = size2[1]; long numZones = _mapping.Count(); var dataArray = new double[nRows, nCols]; var wrapArray = new H5Array <double>(dataArray); H5DataTypeId tid1 = H5D.getType(dataSet); H5D.read(dataSet, tid1, wrapArray); // if the count in the hdf5 file is larger than the number of // tazs in the mapping, ignore the values over the total number //of tazs in the mapping because these are not valid zones. _matrix = new ushort[numZones][]; for (var i = 0; i < numZones; i++) { _matrix[i] = new ushort[numZones]; } //leave as is for PSRC. Values are already scaled integers and matrices already condensed if (Global.Configuration.PSRC) { for (var i = 0; i < numZones; i++) { for (var j = 0; j < numZones; j++) { _matrix[i][j] = (ushort)dataArray[i, j]; } } } else { for (var row = 0; row < nRows; row++) { if (_mapping.ContainsKey(row + 1)) { for (var col = 0; col < nCols; col++) { if (_mapping.ContainsKey(col + 1)) { var value = dataArray[row, col] * scale; if (value > 0) { if (value > short.MaxValue) { value = short.MaxValue; } _matrix[_mapping[row + 1]][_mapping[col + 1]] = (ushort)value; } } } } } } var skimMatrix = new SkimMatrix(_matrix); return(skimMatrix); }
public SkimMatrix Read(string filename, int field, float scale) { var count = _mapping.Count; var matrix = new ushort[count][]; for (var i = 0; i < count; i++) { matrix[i] = new ushort[count]; } if (filename == "null") { return(new SkimMatrix(matrix)); } var file = new FileInfo(Path.Combine(_path, filename)); Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field); Global.PrintFile.WriteFileInfo(file, true); if (!file.Exists) { throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName)); } using (var b = new BinaryReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read))) { var position = 0L; var length = b.BaseStream.Length; //Length of file ID... int k = b.ReadInt16(); string idv = new string(b.ReadChars(k)); //Print the file ID ... $BK, $BI etc. //Get header length... int l = b.ReadInt16(); //Read file header... (bunch of characters) string header = new string(b.ReadChars(l)); //Print the header... //Transport value int transportValue = b.ReadInt32(); //Start time float startTime = b.ReadSingle(); //End time float endTime = b.ReadSingle(); //Factor float factor = b.ReadSingle(); //Rows int rows = b.ReadInt32(); //Data type Int16 dataType = b.ReadInt16(); int binflag = 101; //Check rounding procedure byte d = b.ReadByte(); if (d == '\x01') { binflag = 1; } else if (d == '\x00') { binflag = 0; } else { Console.WriteLine("Binary flag is missing!"); } int[] zonenums; int[] zonenumscol; if (idv == "$BI") { //Matrix type - BI zonenums = new int[rows * 2]; for (int z = 0; z < rows * 2; z++) { zonenums[z] = b.ReadInt32(); } } else { // Matrix type not BI.. //Number of columns int colnumber = b.ReadInt32(); zonenums = new int[rows]; //Read zone numbers for (int z = 0; z < rows; z++) { zonenums[z] = b.ReadInt32(); //Console.WriteLine(zonenums[z].ToString()); } //Read zone numbers on columns zonenumscol = new int[rows]; for (int z = 0; z < rows; z++) { zonenumscol[z] = b.ReadInt32(); //Console.WriteLine(zonenumscol[z].ToString()); } if (idv == "$BK") { //Read the row and column names... int itemleng; string zonename; for (int z = 0; z < colnumber * 2; z++) { itemleng = b.ReadInt32(); //Console.WriteLine(itemleng.ToString()); if (itemleng > 0) { zonename = new string(b.ReadChars(itemleng * 2)); //Console.WriteLine(zonename); } } } } //Check if all values are zero (null matrix) d = b.ReadByte(); if (d == '\x01') { binflag = 1; //this var is not really needed.. but used just for clarity //Console.WriteLine("This is a zero matrix!"); } else if (d == '\x00') { binflag = 0; double diagsum = b.ReadDouble(); Console.WriteLine("Diagonal sum: " + diagsum.ToString()); //Variable for compressed length int compresslength; for (int vl = 0; vl < rows; vl++) { //Get length of compressed bytes compresslength = b.ReadInt32(); //Create memory streams for reading the zipped data and to store inflated data using (MemoryStream inStream = new MemoryStream(b.ReadBytes(compresslength))) using (MemoryStream decompressedFileStream = new MemoryStream()) using (DeflateStream decompressionStream = new DeflateStream(inStream, CompressionMode.Decompress)) { //Why? //Because DelfateStream is same as zlib but zlib has two extra bytes as headers so we pop those inStream.ReadByte(); inStream.ReadByte(); decompressionStream.CopyTo(decompressedFileStream); decompressedFileStream.Position = 0; var sr = new BinaryReader(decompressedFileStream); //Console.Write("Matrix values for row " + vl.ToString() + " >> "); for (int vi = 0; vi < rows; vi++) { double rawValue = sr.ReadDouble(); if (rawValue > short.MaxValue / scale) { rawValue = short.MaxValue / scale; } else if (rawValue < 0) { rawValue = 0; } var value = Convert.ToUInt16(rawValue * scale); matrix[_mapping[zonenums[vl]]][_mapping[zonenums[vi]]] = value; //matrix[vl][vi] = value; //Console.Write(sr.ReadDouble().ToString()); //Console.Write(","); } //Console.WriteLine(); inStream.Dispose(); decompressedFileStream.Dispose(); decompressionStream.Dispose(); sr.Dispose(); } //Get the Row and Column totals. //b.ReadDouble().ToString(); //b.ReadDouble().ToString(); double rowSum = b.ReadDouble(); double columnSum = b.ReadDouble(); } } } /* * * * for (var origin = 0; origin < count; origin++) * { * for (var destination = 0; destination < count; destination++) * { * if (position >= length) * { * goto end; * } * * var value = reader.ReadUInt16(); * * // binary matrices are already mapped to consecutive zone indices, matching the zone index file * matrix[origin][destination] = value; * * position += sizeof(ushort); * } * } * } * * end:*/ var skimMatrix = new SkimMatrix(matrix); return(skimMatrix); }
public virtual void LoadSkimMatrices(IEnumerable <RosterEntry> entries, Dictionary <int, int> zoneMapping, Dictionary <int, int> transitStopAreaMapping, Dictionary <int, int> microzoneMapping) { SkimMatrices = new SkimMatrix[MatrixKeys.Length]; // var cache = new Dictionary<string, List<float[]>>(); var cache = new Dictionary <string, List <double[]> >(); // 20150703 JLB var currentFileName = ""; foreach (var entry in entries.Where(x => x.FileType != null).Select(x => new { x.Name, x.Field, x.FileType, x.MatrixIndex, x.Scaling, x.Length }).Distinct().OrderBy(x => x.Name)) { ISkimFileReader skimFileReader = null; //Issue #40 -- caching is to prevent same file from being read in multiple times. Can clear cache when we change files since group by name if (!entry.Name.Equals(currentFileName)) { cache.Clear(); currentFileName = entry.Name; } IFileReaderCreator creator = Global.Kernel.Get <SkimFileReaderFactory>().GetFileReaderCreator(entry.FileType); /*switch (entry.FileType) { * case "text_ij": * skimFileReader = new TextIJSkimFileReader(cache, _path, mapping); * break; * case "bin": * skimFileReader = new BinarySkimFileReader(_path, mapping); * break; * }*/ if (creator == null) { if (entry.FileType == "deferred") { continue; } throw new SkimFileTypeNotSupportedException(string.Format("The specified skim file type of \"{0}\" is not supported.", entry.FileType)); } Dictionary <int, int> mapping = zoneMapping; bool useTransitStopAreaMapping = (entry.Length == "transitstop"); if (useTransitStopAreaMapping) { mapping = transitStopAreaMapping; } bool useMicrozoneMapping = (entry.Length == "microzone"); if (useMicrozoneMapping) { mapping = microzoneMapping; } skimFileReader = creator.CreateReader(cache, _path, mapping); var skimMatrix = skimFileReader.Read(entry.Name, entry.Field, (float)entry.Scaling); SkimMatrices[entry.MatrixIndex] = skimMatrix; } foreach ( var entry in entries.Where(x => x.FileType == null) .Select(x => new { x.Name, x.Field, x.FileType, x.MatrixIndex, x.Scaling, x.Length }) .Distinct() .OrderBy(x => x.Name)) { var skimMatrix = new SkimMatrix(null); SkimMatrices[entry.MatrixIndex] = skimMatrix; } }