private static SkimValue GetValue(int origin, int destination, RosterEntry entry, int minute) { if (entry.Name == null) { return(new SkimValue()); } SkimMatrix skimMatrix = _skimMatrices[entry.MatrixIndex]; if (skimMatrix.IsEmpty()) { return(new SkimValue { Variable = 0, BlendVariable = 0 }); } if (skimMatrix == null) { throw new SkimMatrixNotFoundException(string.Format("There is not a skim matrix defined for the combination of variable: {0}, mode: {1}, path type: {2}, and minute: {3}. Please adjust the roster accordingly.", entry.Variable, entry.Mode, entry.PathType, minute)); } SkimValue skimValue = new SkimValue { Variable = entry.Transpose ? skimMatrix.GetValue(destination, origin) : skimMatrix.GetValue(origin, destination) }; skimValue.Variable = skimValue.Variable / entry.Scaling; skimValue.Variable = skimValue.Variable * entry.Factor; return(skimValue); }
public SkimMatrix Read(string filename, int field, float scale) { FileInfo file = new FileInfo(Path.Combine(_path, filename)); Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field); Global.PrintFile.WriteFileInfo(file, true); if (!file.Exists) { throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName)); } int count = _mapping.Count; ushort[][] matrix = new ushort[count][]; for (int i = 0; i < count; i++) { matrix[i] = new ushort[count]; } using (BinaryReader reader = new BinaryReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read))) { long position = 0L; long length = reader.BaseStream.Length; for (int origin = 0; origin < count; origin++) { for (int destination = 0; destination < count; destination++) { if (position >= length) { goto end; } ushort value = reader.ReadUInt16(); // binary matrices are already mapped to consecutive zone indices, matching the zone index file matrix[origin][destination] = value; position += sizeof(ushort); } } } end: SkimMatrix skimMatrix = new SkimMatrix(matrix); return(skimMatrix); }
public SkimMatrix Read(string filename, int field, float scale) { var count = _mapping.Count; var matrix = new ushort[count][]; for (var i = 0; i < count; i++) { matrix[i] = new ushort[count]; } if (filename == "null") { return(new SkimMatrix(matrix)); } var file = new FileInfo(Path.Combine(_path, filename)); Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field); Global.PrintFile.WriteFileInfo(file, true); if (!file.Exists) { throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName)); } using (var b = new BinaryReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read))) { var position = 0L; var length = b.BaseStream.Length; //Length of file ID... int k = b.ReadInt16(); string idv = new string(b.ReadChars(k)); //Print the file ID ... $BK, $BI etc. //Get header length... int l = b.ReadInt16(); //Read file header... (bunch of characters) string header = new string(b.ReadChars(l)); //Print the header... //Transport value int transportValue = b.ReadInt32(); //Start time float startTime = b.ReadSingle(); //End time float endTime = b.ReadSingle(); //Factor float factor = b.ReadSingle(); //Rows int rows = b.ReadInt32(); //Data type Int16 dataType = b.ReadInt16(); int binflag = 101; //Check rounding procedure byte d = b.ReadByte(); if (d == '\x01') { binflag = 1; } else if (d == '\x00') { binflag = 0; } else { Console.WriteLine("Binary flag is missing!"); } int[] zonenums; int[] zonenumscol; if (idv == "$BI") //Matrix type - BI { zonenums = new int[rows * 2]; for (int z = 0; z < rows * 2; z++) { zonenums[z] = b.ReadInt32(); } } else // Matrix type not BI.. //Number of columns { int colnumber = b.ReadInt32(); zonenums = new int[rows]; //Read zone numbers for (int z = 0; z < rows; z++) { zonenums[z] = b.ReadInt32(); //Console.WriteLine(zonenums[z].ToString()); } //Read zone numbers on columns zonenumscol = new int[rows]; for (int z = 0; z < rows; z++) { zonenumscol[z] = b.ReadInt32(); //Console.WriteLine(zonenumscol[z].ToString()); } if (idv == "$BK") //Read the row and column names... { int itemleng; string zonename; for (int z = 0; z < colnumber * 2; z++) { itemleng = b.ReadInt32(); //Console.WriteLine(itemleng.ToString()); if (itemleng > 0) { zonename = new string(b.ReadChars(itemleng * 2)); //Console.WriteLine(zonename); } } } } //Check if all values are zero (null matrix) d = b.ReadByte(); if (d == '\x01') { binflag = 1; //this var is not really needed.. but used just for clarity //Console.WriteLine("This is a zero matrix!"); } else if (d == '\x00') { binflag = 0; double diagsum = b.ReadDouble(); Console.WriteLine("Diagonal sum: " + diagsum.ToString()); //Variable for compressed length int compresslength; for (int vl = 0; vl < rows; vl++) { //Get length of compressed bytes compresslength = b.ReadInt32(); //Create memory streams for reading the zipped data and to store inflated data using (MemoryStream inStream = new MemoryStream(b.ReadBytes(compresslength))) using (MemoryStream decompressedFileStream = new MemoryStream()) using (DeflateStream decompressionStream = new DeflateStream(inStream, CompressionMode.Decompress)) { //Why? //Because DelfateStream is same as zlib but zlib has two extra bytes as headers so we pop those inStream.ReadByte(); inStream.ReadByte(); decompressionStream.CopyTo(decompressedFileStream); decompressedFileStream.Position = 0; var sr = new BinaryReader(decompressedFileStream); //Console.Write("Matrix values for row " + vl.ToString() + " >> "); for (int vi = 0; vi < rows; vi++) { double rawValue = sr.ReadDouble() * scale; if (vi == vl && rawValue / scale > 999990) { rawValue = 0; } if (rawValue > ushort.MaxValue - 1) { rawValue = ushort.MaxValue - 1; } else if (rawValue < 0) { rawValue = 0; } var value = Convert.ToUInt16(rawValue); matrix[_mapping[zonenums[vl]]][_mapping[zonenums[vi]]] = value; //matrix[vl][vi] = value; //Console.Write(sr.ReadDouble().ToString()); //Console.Write(","); } //Console.WriteLine(); inStream.Dispose(); decompressedFileStream.Dispose(); decompressionStream.Dispose(); sr.Dispose(); } //Get the Row and Column totals. //b.ReadDouble().ToString(); //b.ReadDouble().ToString(); double rowSum = b.ReadDouble(); double columnSum = b.ReadDouble(); } } } /* * * * for (var origin = 0; origin < count; origin++) * { * for (var destination = 0; destination < count; destination++) * { * if (position >= length) * { * goto end; * } * * var value = reader.ReadUInt16(); * * // binary matrices are already mapped to consecutive zone indices, matching the zone index file * matrix[origin][destination] = value; * * position += sizeof(ushort); * } * } * } * * end:*/ var skimMatrix = new SkimMatrix(matrix); return(skimMatrix); }
public SkimMatrix Read(string filename, int field, float scale) { Console.WriteLine("Reading {0}", filename); int hdf5NameEnd = filename.IndexOf("/"); // the first part of the name in the roster file is the omx/hdf5 file string HDFName = filename.Substring(0, hdf5NameEnd); //rename filename to be only the name of the skim matrix inside of the skim file //skims are stored in the "data" folder within the omx/hdf5 file filename = filename.Substring(hdf5NameEnd); string hdfFile = _path + "\\" + HDFName; var dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY); var dataSet = H5D.open(dataFile, filename); var space = H5D.getSpace(dataSet); var size2 = H5S.getSimpleExtentDims(space); long nRows = size2[0]; long nCols = size2[1]; long numZones = _mapping.Count(); // if the count in the hdf5 file is larger than the number of // tazs in the mapping, ignore the values over the total number //of tazs in the mapping because these are not valid zones. _matrix = new ushort[numZones][]; for (var i = 0; i < numZones; i++) { _matrix[i] = new ushort[numZones]; } //OMX is a square matrix of doubles //In addition to the data folder for matrices, an OMX file has a lookup folder //with a zone mapping vector. However, this is ignored since DaySim also has one. //Therefore, it is assumed the OMX matrix does not skip rows/cols and every row/col //corresponds to an actual zone in the DaySim zone mapping file by index //Scaling should be set to TRUE since OMX stores doubles (not scaled integers) var dataArray = new double[nRows, nCols]; var wrapArray = new H5Array <double>(dataArray); H5DataTypeId tid1 = H5D.getType(dataSet); H5D.read(dataSet, tid1, wrapArray); for (var row = 0; row < nRows; row++) { if (_mapping.ContainsKey(row + 1)) { for (var col = 0; col < nCols; col++) { if (_mapping.ContainsKey(col + 1)) { var value = dataArray[row, col] * scale; if (value > 0) { if (value > ushort.MaxValue - 1) { value = ushort.MaxValue - 1; } _matrix[_mapping[row + 1]][_mapping[col + 1]] = (ushort)value; } } } } } var skimMatrix = new SkimMatrix(_matrix); return(skimMatrix); }
public SkimMatrix Read(string fileNameAndGroupAndDataTable, int field, float scale) { //mb merged these changes to local develop before pushing #210 //hdf5 filename contain "filename/group/skim" //get the index of group int hdf5GroupEnd = fileNameAndGroupAndDataTable.LastIndexOf("/"); string fileNameAndGroup = hdf5GroupEnd > 0 ? fileNameAndGroupAndDataTable.Substring(0, hdf5GroupEnd) : fileNameAndGroupAndDataTable; //get the index of filename int hdf5NameEnd = hdf5GroupEnd > 0 ? fileNameAndGroup.LastIndexOf("/") : -1; string groupName = fileNameAndGroup.Substring(hdf5NameEnd + 1); //get the omx/hdf5 filename string HDFName = fileNameAndGroup.Substring(0, hdf5NameEnd); string groupAndDataTable = fileNameAndGroupAndDataTable.Substring(hdf5NameEnd); string hdfFile = Path.Combine(_path, HDFName); FileInfo file = new FileInfo(hdfFile); if (!file.Exists) { throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName)); } H5FileId dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY); H5DataSetId dataSet = H5D.open(dataFile, groupAndDataTable); H5DataSpaceId space = H5D.getSpace(dataSet); long[] size2 = H5S.getSimpleExtentDims(space); long nRows = size2[0]; long nCols = size2[1]; Debug.Assert(nRows == nCols); long numZones = _mapping.Count(); int[] lookupMap = null; string lookupMapName = null; string lookupGroupName = "lookup"; H5GroupId luGroup = H5G.open(dataFile, lookupGroupName); if (H5G.getNumObjects(luGroup) == 1L) { lookupMapName = H5G.getObjectNameByIndex(luGroup, 0); H5DataSetId lookupDataSet = H5D.open(dataFile, string.Concat(lookupGroupName, "/", lookupMapName)); H5DataTypeId lookupMapType = H5D.getType(lookupDataSet); H5DataSpaceId lookupMapSpace = H5D.getSpace(lookupDataSet); long lookupMapSize = H5S.getSimpleExtentDims(lookupMapSpace)[0]; lookupMap = new int[lookupMapSize]; H5Array <int> lookupWrapArray = new H5Array <int>(lookupMap); H5D.read(lookupDataSet, lookupMapType, lookupWrapArray); } if (lookupMap != null) { if (lookupMap.Length != nRows) { Global.PrintFile.WriteLine(string.Format("DATA WARNING: skim file: {0} has a lookup map named {1} but its length ({2}) is different than the matrix size ({3}) in group table {4}", hdfFile, lookupMapName, lookupMap.Length, nRows, groupAndDataTable)); lookupMap = null; } } // if the count in the hdf5 file is larger than the number of // tazs in the mapping, ignore the values over the total number //of tazs in the mapping because these are not valid zones. _matrix = new ushort[numZones][]; for (int i = 0; i < numZones; i++) { _matrix[i] = new ushort[numZones]; } //OMX is a square matrix of doubles //In addition to the data folder for matrices, an OMX file has a lookup folder //with a zone mapping vector. However, this is ignored since DaySim also has one. //Therefore, it is assumed the OMX matrix does not skip rows/cols and every row/col //corresponds to an actual zone in the DaySim zone mapping file by index double[,] dataArray = new double[nRows, nCols]; H5Array <double> wrapArray = new H5Array <double>(dataArray); H5DataTypeId tid1 = H5D.getType(dataSet); H5D.read(dataSet, tid1, wrapArray); long numSuccessfuLookups = 0; for (int row = 0; row < nRows; row++) { int mappingKey = (lookupMap == null) ? (row + 1) : lookupMap[row]; if (_mapping.TryGetValue(mappingKey, out int mappedRow)) { for (int col = 0; col < nCols; col++) { mappingKey = (lookupMap == null) ? (col + 1) : lookupMap[col]; if (_mapping.TryGetValue(mappingKey, out int mappedCol)) { double value = dataArray[row, col] * scale; ++numSuccessfuLookups; if (value > 0) { if (value > ushort.MaxValue - 1) { value = ushort.MaxValue - 1; } _matrix[mappedRow][mappedCol] = (ushort)value; //bug #208 deferred but this will eventually changed be Convert.ToUInt16 to avoid 0.57*100=56 bug } } } //for col } } //for row Global.PrintFile.WriteLine(string.Format("Loaded skim file: {0} dataset: {1}, scale={6} with input matrix size={2} and DaySim matrix size={3} zones. Percentage successful mappings {4}. {5}", hdfFile, groupAndDataTable, nRows, numZones, (numSuccessfuLookups / (nRows * nCols)) * 100.0, lookupMap != null ? string.Format("Using lookupMap {0}.", lookupMapName) : "No lookupMap.", scale), true); SkimMatrix skimMatrix = new SkimMatrix(_matrix); return(skimMatrix); }
public SkimMatrix Read(string filename, int field, float scale) { var file = new FileInfo(Path.Combine(_path, filename)); Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field); Global.PrintFile.WriteFileInfo(file, true); if (!file.Exists) { throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName)); } var count = _mapping.Count; var matrix = new ushort[count][]; for (var i = 0; i < count; i++) { matrix[i] = new ushort[count]; } // List<float[]> rows; List <double[]> rows; // 20150703 JLB if (!_cache.TryGetValue(file.Name, out rows)) { // rows = new List<float[]>(); rows = new List <double[]>(); // 20150703 JLB using (var reader = new CountingReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read))) { string line; if (Global.TextSkimFilesContainHeaderRecord) { reader.ReadLine(); } while ((line = reader.ReadLine()) != null) { try { char delimiter = Global.SkimDelimiter; var p1 = line.Split(new[] { delimiter }, StringSplitOptions.RemoveEmptyEntries); // var row = p1.Select(Convert.ToSingle).ToArray(); var row = p1.Select(Convert.ToDouble).ToArray(); // 20150703 JLB rows.Add(row); } catch (Exception e) { throw new InvalidSkimRowException(string.Format("Error parsing row in {0}.\nError in row {1}.\n{{{2}}}", file.FullName, reader.LineNumber, line), e); } } } _cache.Add(file.Name, rows); } for (var i = 0; i < rows.Count; i++) { var row = rows[i]; try { // test code var index = Convert.ToInt32(row[0]); var origin = _mapping[index]; //var origin = _mapping[Convert.ToInt32(row[0])]; index = Convert.ToInt32(row[1]); var destination = _mapping[index]; //var destination = _mapping[Convert.ToInt32(row[1])]; var rawValue = Convert.ToSingle(row[field + FIELD_OFFSET]) * scale; if (rawValue > ushort.MaxValue - 1) { rawValue = ushort.MaxValue - 1; } else if (rawValue < 0) { rawValue = 0; } var value = Convert.ToUInt16(rawValue); matrix[origin][destination] = value; } catch (Exception e) { throw new ErrorReadingSkimFileException(string.Format("Error reading row {0}, {1}", i, string.Join(" ", row)), e); } } var skimMatrix = new SkimMatrix(matrix); return(skimMatrix); }
public SkimMatrix Read(string filename, int field, float scale) { Console.WriteLine("Reading {0}", filename); int hdf5NameEnd = filename.IndexOf("/"); // the first part of the name in the roster file is the hdf5 file: string HDFName = filename.Substring(0, hdf5NameEnd); //rename filename to be only the name of the skim inside of the time period file filename = filename.Substring(hdf5NameEnd); string hdfFile = _path + "\\" + HDFName; H5FileId dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY); H5DataSetId dataSet = H5D.open(dataFile, filename); H5DataSpaceId space = H5D.getSpace(dataSet); long[] size2 = H5S.getSimpleExtentDims(space); long nRows = size2[0]; long nCols = size2[1]; long numZones = _mapping.Count(); // if the count in the hdf5 file is larger than the number of // tazs in the mapping, ignore the values over the total number //of tazs in the mapping because these are not valid zones. _matrix = new ushort[numZones][]; for (int i = 0; i < numZones; i++) { _matrix[i] = new ushort[numZones]; } //leave as is for PSRC. Values are already scaled integers and matrices already condensed if (Global.Configuration.HDF5SkimScaledAndCondensed) { ushort[,] dataArray = new ushort[nRows, nCols]; H5Array <ushort> wrapArray = new H5Array <ushort>(dataArray); H5DataTypeId tid1 = H5D.getType(dataSet); H5D.read(dataSet, tid1, wrapArray); for (int i = 0; i < numZones; i++) { for (int j = 0; j < numZones; j++) { _matrix[i][j] = dataArray[i, j]; } } } else { double[,] dataArray = new double[nRows, nCols]; H5Array <double> wrapArray = new H5Array <double>(dataArray); H5DataTypeId tid1 = H5D.getType(dataSet); H5D.read(dataSet, tid1, wrapArray); for (int row = 0; row < nRows; row++) { if (_mapping.ContainsKey(row + 1)) { for (int col = 0; col < nCols; col++) { if (_mapping.ContainsKey(col + 1)) { double value = dataArray[row, col] * scale; if (value > 0) { if (value > ushort.MaxValue - 1) { value = ushort.MaxValue - 1; } _matrix[_mapping[row + 1]][_mapping[col + 1]] = (ushort)value; } } } } } } SkimMatrix skimMatrix = new SkimMatrix(_matrix); return(skimMatrix); }
public virtual void LoadSkimMatrices(IEnumerable <RosterEntry> entries, Dictionary <int, int> zoneMapping, Dictionary <int, int> transitStopAreaMapping, Dictionary <int, int> microzoneMapping) { SkimMatrices = new SkimMatrix[MatrixKeys.Length]; // var cache = new Dictionary<string, List<float[]>>(); var cache = new Dictionary <string, List <double[]> >(); // 20150703 JLB var currentFileName = ""; foreach (var entry in entries.Where(x => x.FileType != null).Select(x => new { x.Name, x.Field, x.FileType, x.MatrixIndex, x.Scaling, x.Length }).Distinct().OrderBy(x => x.Name)) { ISkimFileReader skimFileReader = null; //Issue #40 -- caching is to prevent same file from being read in multiple times. Can clear cache when we change files since group by name if (!entry.Name.Equals(currentFileName)) { cache.Clear(); currentFileName = entry.Name; } IFileReaderCreator creator = Global.ContainerDaySim.GetInstance <SkimFileReaderFactory>().GetFileReaderCreator(entry.FileType); /*switch (entry.FileType) { * case "text_ij": * skimFileReader = new TextIJSkimFileReader(cache, _path, mapping); * break; * case "bin": * skimFileReader = new BinarySkimFileReader(_path, mapping); * break; * }*/ if (creator == null) { if (entry.FileType == "deferred") { continue; } throw new SkimFileTypeNotSupportedException(string.Format("The specified skim file type of \"{0}\" is not supported.", entry.FileType)); } Dictionary <int, int> mapping = zoneMapping; bool useTransitStopAreaMapping = (entry.Length == "transitstop"); if (useTransitStopAreaMapping) { mapping = transitStopAreaMapping; } bool useMicrozoneMapping = (entry.Length == "microzone"); if (useMicrozoneMapping) { mapping = microzoneMapping; } skimFileReader = creator.CreateReader(cache, _path, mapping); var scaleFactor = (float)entry.Scaling; var skimMatrix = skimFileReader.Read(entry.Name, entry.Field, scaleFactor); SkimMatrices[entry.MatrixIndex] = skimMatrix; //new code - report a summary of the skim matrix data var numZones = mapping.Count; var numPositive = 0; double avgPositive = 0; for (var row = 0; row < numZones; row++) { for (var col = 0; col < numZones; col++) { var value = skimMatrix.GetValue(row, col); if (value > Constants.EPSILON) { numPositive++; avgPositive += value / scaleFactor; } } } var pctPositive = Math.Round(numPositive * 100.0 / (numZones * numZones)); var avgValue = avgPositive / Math.Max(numPositive, 1); Global.PrintFile.WriteLine("Skim File {0}, {1} percent of values are positive, with average value {2}.", entry.Name, pctPositive, avgValue); } foreach ( var entry in entries.Where(x => x.FileType == null) .Select(x => new { x.Name, x.Field, x.FileType, x.MatrixIndex, x.Scaling, x.Length }) .Distinct() .OrderBy(x => x.Name)) { var skimMatrix = new SkimMatrix(null); SkimMatrices[entry.MatrixIndex] = skimMatrix; } }
public SkimMatrix Read(string filename, int field, float scale) { //hdf5 filename contain "filename/group/skim" //get the index of group int hdf5GroupEnd = filename.LastIndexOf("/"); //get the index of filename int hdf5NameEnd = hdf5GroupEnd > 0 ? filename.LastIndexOf("/", hdf5GroupEnd - 1) : -1; //get the omx/hdf5 filename string HDFName = filename.Substring(0, hdf5NameEnd); //rename filename to be only the name of the skim matrix inside of the skim file filename = filename.Substring(hdf5NameEnd); string hdfFile = Path.Combine(_path, HDFName); Console.WriteLine("Loading skim file: {0}.", hdfFile); FileInfo file = new FileInfo(hdfFile); if (!file.Exists) { throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName)); } H5FileId dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY); H5DataSetId dataSet = H5D.open(dataFile, filename); H5DataSpaceId space = H5D.getSpace(dataSet); long[] size2 = H5S.getSimpleExtentDims(space); long nRows = size2[0]; long nCols = size2[1]; long numZones = _mapping.Count(); // if the count in the hdf5 file is larger than the number of // tazs in the mapping, ignore the values over the total number //of tazs in the mapping because these are not valid zones. _matrix = new ushort[numZones][]; for (int i = 0; i < numZones; i++) { _matrix[i] = new ushort[numZones]; } //OMX is a square matrix of doubles //In addition to the data folder for matrices, an OMX file has a lookup folder //with a zone mapping vector. However, this is ignored since DaySim also has one. //Therefore, it is assumed the OMX matrix does not skip rows/cols and every row/col //corresponds to an actual zone in the DaySim zone mapping file by index //Scaling should be set to TRUE since OMX stores doubles (not scaled integers) double[,] dataArray = new double[nRows, nCols]; H5Array <double> wrapArray = new H5Array <double>(dataArray); H5DataTypeId tid1 = H5D.getType(dataSet); H5D.read(dataSet, tid1, wrapArray); for (int row = 0; row < nRows; row++) { if (_mapping.ContainsKey(row + 1)) { for (int col = 0; col < nCols; col++) { if (_mapping.ContainsKey(col + 1)) { double value = dataArray[row, col] * scale; if (value > 0) { if (value > ushort.MaxValue - 1) { value = ushort.MaxValue - 1; } _matrix[_mapping[row + 1]][_mapping[col + 1]] = (ushort)value; } } } } } SkimMatrix skimMatrix = new SkimMatrix(_matrix); return(skimMatrix); }