public SkimMatrix Read(string filename, int field, float scale)
{
var file = new FileInfo(Path.Combine(_path, filename));
Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field);
Global.PrintFile.WriteFileInfo(file, true);
if (!file.Exists)
{
throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName));
}
var count = _mapping.Count;
var matrix = new ushort[count][];
for (var i = 0; i < count; i++)
{
matrix[i] = new ushort[count];
}
using (var reader = new BinaryReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read))) {
var position = 0L;
var length = reader.BaseStream.Length;
for (var origin = 0; origin < count; origin++)
{
for (var destination = 0; destination < count; destination++)
{
if (position >= length)
{
goto end;
}
var value = reader.ReadUInt16();
// binary matrices are already mapped to consecutive zone indices, matching the zone index file
matrix[origin][destination] = value;
position += sizeof(ushort);
}
}
}
end:
var skimMatrix = new SkimMatrix(matrix);
return(skimMatrix);
}
public SkimMatrix Read(string filename, int field, float scale)
{
var file = new FileInfo(Path.Combine(_path, filename));
Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field);
Global.PrintFile.WriteFileInfo(file, true);
if (!file.Exists)
{
throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName));
}
var count = _mapping.Count;
var matrix = new ushort[count][];
for (var i = 0; i < count; i++)
{
matrix[i] = new ushort[count];
}
// List<float[]> rows;
List <double[]> rows; // 20150703 JLB
if (!_cache.TryGetValue(file.Name, out rows))
{
// rows = new List<float[]>();
rows = new List <double[]>(); // 20150703 JLB
using (var reader = new StreamReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read))) {
string line;
var current = 0;
if (Global.TextSkimFilesContainHeaderRecord)
{
reader.ReadLine();
}
while ((line = reader.ReadLine()) != null)
{
current++;
try
{
char delimiter = Global.SkimDelimiter;
var p1 = line.Split(new[] { delimiter }, StringSplitOptions.RemoveEmptyEntries);
// var row = p1.Select(Convert.ToSingle).ToArray();
var row = p1.Select(Convert.ToDouble).ToArray(); // 20150703 JLB
rows.Add(row);
}
catch (Exception e) {
throw new InvalidSkimRowException(string.Format("Error parsing row in {0}.\nError in row {1}.\n{{{2}}}", file.FullName, current, line), e);
}
}
}
_cache.Add(file.Name, rows);
}
for (var i = 0; i < rows.Count; i++)
{
var row = rows[i];
try {
// test code
var index = Convert.ToInt32(row[0]);
var origin = _mapping[index];
//var origin = _mapping[Convert.ToInt32(row[0])];
index = Convert.ToInt32(row[1]);
var destination = _mapping[index];
//var destination = _mapping[Convert.ToInt32(row[1])];
var rawValue = Convert.ToSingle(row[field + FIELD_OFFSET]);
if (rawValue > short.MaxValue / scale)
{
rawValue = short.MaxValue / (float)scale;
}
else if (rawValue < 0)
{
rawValue = 0;
}
var value = Convert.ToUInt16(rawValue * scale);
matrix[origin][destination] = value;
}
catch (Exception e) {
throw new ErrorReadingSkimFileException(string.Format("Error reading row {0}, {1}", i, string.Join(" ", row)), e);
}
}
var skimMatrix = new SkimMatrix(matrix);
return(skimMatrix);
}
public SkimMatrix Read(string filename, int field, float scale)
{
Console.WriteLine("Reading {0}", filename);
int hdf5NameEnd = filename.IndexOf("/");
// the first part of the name in the roster file is the hdf5 file:
string HDFName = filename.Substring(0, hdf5NameEnd);
//rename filename to be only the name of the skim inside of the time period file
filename = filename.Substring(hdf5NameEnd);
string hdfFile = _path + "\\" + HDFName;
var dataFile = H5F.open(hdfFile, H5F.OpenMode.ACC_RDONLY);
var dataSet = H5D.open(dataFile, filename);
var space = H5D.getSpace(dataSet);
var size2 = H5S.getSimpleExtentDims(space);
long nRows = size2[0];
long nCols = size2[1];
long numZones = _mapping.Count();
var dataArray = new double[nRows, nCols];
var wrapArray = new H5Array <double>(dataArray);
H5DataTypeId tid1 = H5D.getType(dataSet);
H5D.read(dataSet, tid1, wrapArray);
// if the count in the hdf5 file is larger than the number of
// tazs in the mapping, ignore the values over the total number
//of tazs in the mapping because these are not valid zones.
_matrix = new ushort[numZones][];
for (var i = 0; i < numZones; i++)
{
_matrix[i] = new ushort[numZones];
}
//leave as is for PSRC. Values are already scaled integers and matrices already condensed
if (Global.Configuration.PSRC)
{
for (var i = 0; i < numZones; i++)
{
for (var j = 0; j < numZones; j++)
{
_matrix[i][j] = (ushort)dataArray[i, j];
}
}
}
else
{
for (var row = 0; row < nRows; row++)
{
if (_mapping.ContainsKey(row + 1))
{
for (var col = 0; col < nCols; col++)
{
if (_mapping.ContainsKey(col + 1))
{
var value = dataArray[row, col] * scale;
if (value > 0)
{
if (value > short.MaxValue)
{
value = short.MaxValue;
}
_matrix[_mapping[row + 1]][_mapping[col + 1]] = (ushort)value;
}
}
}
}
}
}
var skimMatrix = new SkimMatrix(_matrix);
return(skimMatrix);
}
public SkimMatrix Read(string filename, int field, float scale)
{
var count = _mapping.Count;
var matrix = new ushort[count][];
for (var i = 0; i < count; i++)
{
matrix[i] = new ushort[count];
}
if (filename == "null")
{
return(new SkimMatrix(matrix));
}
var file = new FileInfo(Path.Combine(_path, filename));
Console.WriteLine("Loading skim file: {0}, field: {1}.", file.Name, field);
Global.PrintFile.WriteFileInfo(file, true);
if (!file.Exists)
{
throw new FileNotFoundException(string.Format("The skim file {0} could not be found.", file.FullName));
}
using (var b = new BinaryReader(file.Open(FileMode.Open, FileAccess.Read, FileShare.Read)))
{
var position = 0L;
var length = b.BaseStream.Length;
//Length of file ID...
int k = b.ReadInt16();
string idv = new string(b.ReadChars(k));
//Print the file ID ... $BK, $BI etc.
//Get header length...
int l = b.ReadInt16();
//Read file header... (bunch of characters)
string header = new string(b.ReadChars(l));
//Print the header...
//Transport value
int transportValue = b.ReadInt32();
//Start time
float startTime = b.ReadSingle();
//End time
float endTime = b.ReadSingle();
//Factor
float factor = b.ReadSingle();
//Rows
int rows = b.ReadInt32();
//Data type
Int16 dataType = b.ReadInt16();
int binflag = 101;
//Check rounding procedure
byte d = b.ReadByte();
if (d == '\x01')
{
binflag = 1;
}
else if (d == '\x00')
{
binflag = 0;
}
else
{
Console.WriteLine("Binary flag is missing!");
}
int[] zonenums;
int[] zonenumscol;
if (idv == "$BI")
{ //Matrix type - BI
zonenums = new int[rows * 2];
for (int z = 0; z < rows * 2; z++)
{
zonenums[z] = b.ReadInt32();
}
}
else
{ // Matrix type not BI..
//Number of columns
int colnumber = b.ReadInt32();
zonenums = new int[rows];
//Read zone numbers
for (int z = 0; z < rows; z++)
{
zonenums[z] = b.ReadInt32();
//Console.WriteLine(zonenums[z].ToString());
}
//Read zone numbers on columns
zonenumscol = new int[rows];
for (int z = 0; z < rows; z++)
{
zonenumscol[z] = b.ReadInt32();
//Console.WriteLine(zonenumscol[z].ToString());
}
if (idv == "$BK")
{ //Read the row and column names...
int itemleng;
string zonename;
for (int z = 0; z < colnumber * 2; z++)
{
itemleng = b.ReadInt32();
//Console.WriteLine(itemleng.ToString());
if (itemleng > 0)
{
zonename = new string(b.ReadChars(itemleng * 2));
//Console.WriteLine(zonename);
}
}
}
}
//Check if all values are zero (null matrix)
d = b.ReadByte();
if (d == '\x01')
{
binflag = 1; //this var is not really needed.. but used just for clarity
//Console.WriteLine("This is a zero matrix!");
}
else if (d == '\x00')
{
binflag = 0;
double diagsum = b.ReadDouble();
Console.WriteLine("Diagonal sum: " + diagsum.ToString());
//Variable for compressed length
int compresslength;
for (int vl = 0; vl < rows; vl++)
{
//Get length of compressed bytes
compresslength = b.ReadInt32();
//Create memory streams for reading the zipped data and to store inflated data
using (MemoryStream inStream = new MemoryStream(b.ReadBytes(compresslength)))
using (MemoryStream decompressedFileStream = new MemoryStream())
using (DeflateStream decompressionStream = new DeflateStream(inStream, CompressionMode.Decompress))
{
//Why?
//Because DelfateStream is same as zlib but zlib has two extra bytes as headers so we pop those
inStream.ReadByte();
inStream.ReadByte();
decompressionStream.CopyTo(decompressedFileStream);
decompressedFileStream.Position = 0;
var sr = new BinaryReader(decompressedFileStream);
//Console.Write("Matrix values for row " + vl.ToString() + " >> ");
for (int vi = 0; vi < rows; vi++)
{
double rawValue = sr.ReadDouble();
if (rawValue > short.MaxValue / scale)
{
rawValue = short.MaxValue / scale;
}
else if (rawValue < 0)
{
rawValue = 0;
}
var value = Convert.ToUInt16(rawValue * scale);
matrix[_mapping[zonenums[vl]]][_mapping[zonenums[vi]]] = value;
//matrix[vl][vi] = value;
//Console.Write(sr.ReadDouble().ToString());
//Console.Write(",");
}
//Console.WriteLine();
inStream.Dispose();
decompressedFileStream.Dispose();
decompressionStream.Dispose();
sr.Dispose();
}
//Get the Row and Column totals.
//b.ReadDouble().ToString();
//b.ReadDouble().ToString();
double rowSum = b.ReadDouble();
double columnSum = b.ReadDouble();
}
}
}
/*
*
*
* for (var origin = 0; origin < count; origin++)
* {
* for (var destination = 0; destination < count; destination++)
* {
* if (position >= length)
* {
* goto end;
* }
*
* var value = reader.ReadUInt16();
*
* // binary matrices are already mapped to consecutive zone indices, matching the zone index file
* matrix[origin][destination] = value;
*
* position += sizeof(ushort);
* }
* }
* }
*
* end:*/
var skimMatrix = new SkimMatrix(matrix);
return(skimMatrix);
}
public virtual void LoadSkimMatrices(IEnumerable <RosterEntry> entries, Dictionary <int, int> zoneMapping, Dictionary <int, int> transitStopAreaMapping, Dictionary <int, int> microzoneMapping)
{
SkimMatrices = new SkimMatrix[MatrixKeys.Length];
// var cache = new Dictionary<string, List<float[]>>();
var cache = new Dictionary <string, List <double[]> >(); // 20150703 JLB
var currentFileName = "";
foreach (var entry in entries.Where(x => x.FileType != null).Select(x => new { x.Name, x.Field, x.FileType, x.MatrixIndex, x.Scaling, x.Length }).Distinct().OrderBy(x => x.Name))
{
ISkimFileReader skimFileReader = null;
//Issue #40 -- caching is to prevent same file from being read in multiple times. Can clear cache when we change files since group by name
if (!entry.Name.Equals(currentFileName))
{
cache.Clear();
currentFileName = entry.Name;
}
IFileReaderCreator creator = Global.Kernel.Get <SkimFileReaderFactory>().GetFileReaderCreator(entry.FileType);
/*switch (entry.FileType) {
* case "text_ij":
* skimFileReader = new TextIJSkimFileReader(cache, _path, mapping);
* break;
* case "bin":
* skimFileReader = new BinarySkimFileReader(_path, mapping);
* break;
* }*/
if (creator == null)
{
if (entry.FileType == "deferred")
{
continue;
}
throw new SkimFileTypeNotSupportedException(string.Format("The specified skim file type of \"{0}\" is not supported.", entry.FileType));
}
Dictionary <int, int> mapping = zoneMapping;
bool useTransitStopAreaMapping = (entry.Length == "transitstop");
if (useTransitStopAreaMapping)
{
mapping = transitStopAreaMapping;
}
bool useMicrozoneMapping = (entry.Length == "microzone");
if (useMicrozoneMapping)
{
mapping = microzoneMapping;
}
skimFileReader = creator.CreateReader(cache, _path, mapping);
var skimMatrix = skimFileReader.Read(entry.Name, entry.Field, (float)entry.Scaling);
SkimMatrices[entry.MatrixIndex] = skimMatrix;
}
foreach (
var entry in
entries.Where(x => x.FileType == null)
.Select(x => new { x.Name, x.Field, x.FileType, x.MatrixIndex, x.Scaling, x.Length })
.Distinct()
.OrderBy(x => x.Name))
{
var skimMatrix = new SkimMatrix(null);
SkimMatrices[entry.MatrixIndex] = skimMatrix;
}
}
