#pragma warning disable 1591
public override void SetValueOrMissing(string rowKey, string colKey, TValue value)
#pragma warning restore 1591
{
int colIndex = ColSerialNumbers.GetOld(colKey);
ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue;
byte[] byteArray = ValueOrMissingToByteArray(value);
Helper.CheckCondition(byteArray.Length == BytesPerValue, "Expected byteArray to be the right length");
ThreadLocalStream.Write(byteArray, 0, BytesPerValue);
}
#pragma warning disable 1591
public override void SetValueOrMissing(string rowKey, string colKey, TValue value)
#pragma warning restore 1591
{
int colIndex = ColSerialNumbers.GetOld(colKey);
ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue;
byte[] byteArray = ValueOrMissingToByteArray(value);
Helper.CheckCondition(byteArray.Length == BytesPerValue, Properties.Resource.ExpectedByteArrayLengthAndBytesPerValueToBeEqual, byteArray.Length, BytesPerValue);
ThreadLocalStream.Write(byteArray, 0, BytesPerValue);
}
#pragma warning disable 1591
override public bool TryGetValue(string rowKey, string colKey, out TValue value)
#pragma warning restore 1591
{
int colIndex = ColSerialNumbers.GetOld(colKey);
ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue;
byte[] byteArray = new byte[BytesPerValue];
int bytesRead = ThreadLocalStream.Read(byteArray, 0, BytesPerValue);
Helper.CheckCondition(bytesRead == BytesPerValue, Properties.Resource.ExpectedToReadAllBytesOfValue);
value = ByteArrayToValueOrMissing(byteArray);
return(!IsMissing(value));
}
private void ValueTester(string rowKeysStructFileName)
{
//Console.WriteLine("Index loaded. Now testing values");
//Test that can really read values from data file
if (RowCount > 0 && ColCount > 0)
{
try
{
string rowKey = RowKeys[RowCount - 1];
//Check the width of the data
if (RowCount > 1)
{
long rowLength = RowKeyToFilePosition[rowKey] - RowKeyToFilePosition[RowKeys[RowCount - 2]] - rowKey.Length - 1 /*tab*/ - 2 /*newline*/;
Helper.CheckCondition <MatrixFormatException>(rowLength == BytesPerValue * ColCount, () => string.Format(CultureInfo.InvariantCulture, Properties.Resource.ExpectEachDataRowToHaveNCharacters, BytesPerValue * ColCount));
}
//Check the length of the file
long lastPosition = RowKeyToFilePosition[rowKey] + ColCount * BytesPerValue + 2 /*newline*/;
Helper.CheckCondition <MatrixFormatException>(ThreadLocalStream.Length == lastPosition, Properties.Resource.ExpectFileToEndAfterLastValue);
//Check that the last rowKey is where it is expected
ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] - rowKey.Length - 1;
byte[] byteArray = new byte[rowKey.Length];
int bytesRead = ThreadLocalStream.Read(byteArray, 0, rowKey.Length);
Helper.CheckCondition <MatrixFormatException>(bytesRead == rowKey.Length, Properties.Resource.ExpectToReadRowKey);
string asString = System.Text.Encoding.UTF8.GetString(byteArray, 0, byteArray.Length);
Helper.CheckCondition <MatrixFormatException>(asString == rowKey, Properties.Resource.ExpectRowKeyFileAndMainFileToAgreeOnTheRowkeys);
//Read some values
GetValueOrMissing(0, 0);
GetValueOrMissing(RowCount / 2, ColCount / 2);
GetValueOrMissing(RowCount - 1, ColCount - 1);
}
catch (MatrixFormatException)
{
throw;
}
catch (Exception e)
{
throw new MatrixFormatException(Properties.Resource.RowKeyIndexSeemsToReferToAFileOfTheWrongFormat + rowKeysStructFileName, e);
}
}
//Console.WriteLine("Values tested. Done");
}
private void ValueTester(string rowKeysStructFileName)
{
Console.WriteLine("Index loaded. Now testing values");
//Test that can really read values from data file
if (RowCount > 0 && ColCount > 0)
{
try
{
string rowKey = RowKeys[RowCount - 1];
//Check the width of the data
if (RowCount > 1)
{
long rowLength = RowKeyToFilePosition[rowKey] - RowKeyToFilePosition[RowKeys[RowCount - 2]] - rowKey.Length - 1 /*tab*/ - 2 /*newline*/;
Helper.CheckCondition <MatrixFormatException>(rowLength == BytesPerValue * ColCount, "Expect each data row to have {0} characters.", BytesPerValue * ColCount);
}
//Check the length of the file
long lastPosition = RowKeyToFilePosition[rowKey] + ColCount * BytesPerValue + 2 /*newline*/;
Helper.CheckCondition <MatrixFormatException>(ThreadLocalStream.Length == lastPosition, "Expect file to end after last value");
//Check that the last rowKey is where it is expected
ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] - rowKey.Length - 1;
byte[] byteArray = new byte[rowKey.Length];
int bytesRead = ThreadLocalStream.Read(byteArray, 0, rowKey.Length);
Helper.CheckCondition <MatrixFormatException>(bytesRead == rowKey.Length, "Expect to read rowKey");
string asString = System.Text.Encoding.UTF8.GetString(byteArray, 0, byteArray.Length);
Helper.CheckCondition <MatrixFormatException>(asString == rowKey, "Expect rowKey file and main file to agree on the rowkeys");
//Read some values
GetValueOrMissing(0, 0);
GetValueOrMissing(RowCount / 2, ColCount / 2);
GetValueOrMissing(RowCount - 1, ColCount - 1);
}
catch (MatrixFormatException)
{
throw;
}
catch (Exception e)
{
throw new MatrixFormatException("RowKeyIndex seems to refer to a file of the wrong format. " + rowKeysStructFileName, e);
}
}
Console.WriteLine("Values tested. Done");
}
protected void GetInstanceFromDenseStructFileNameInternal(string denseStructFileName, ParallelOptions parallelOptions, FileAccess fileAccess = FileAccess.Read, FileShare fileShare = FileShare.Read)
{
// parallelOptions is not currently used, but it is need so that this method will have the same signature as other, similar methods.
lock (this)
{
using (FileStream fileStream = File.Open(denseStructFileName, FileMode.Open, fileAccess, fileShare))
{
using (TextReader textReader = new StreamReader(fileStream))
{
string firstLineOrNull = textReader.ReadLine();
Helper.CheckCondition(null != firstLineOrNull, Properties.Resource.ExpectedFileToHaveData, denseStructFileName);
Helper.CheckCondition(!firstLineOrNull.StartsWith(FileUtils.CommentHeader, StringComparison.Ordinal), Properties.Resource.ExpectedNoCommentsInRowKeysAnsiFiles, denseStructFileName);
}
}
RowKeyToFilePosition = new Dictionary <string, long>();
DenseStructFileName = denseStructFileName;
FileAccess = fileAccess;
FileShare = fileShare;
long position = 0;
string colKeysLineOrNull = ThreadLocalTextReader.ReadLine();
position += colKeysLineOrNull.Length + 2; //!!!const assuming 2 char newslines
string[] varAndColKeys = colKeysLineOrNull.Split('\t');
if (!varAndColKeys[0].Equals("var"))
{
throw new MatrixFormatException("Expect first row's first value to be 'var'");
}
ColSerialNumbers = new SerialNumbers <string>(varAndColKeys.Skip(1));
_rowKeys = new List <string>();
if (null == colKeysLineOrNull)
{
throw new MatrixFormatException("Surprised by empty file. " + denseStructFileName);
}
CounterWithMessages counterWithMessages = new CounterWithMessages("Reading data file to find location of rows, #{0}", 10000, null);
while (true)
{
counterWithMessages.Increment();
ThreadLocalStream.Position = position;
StringBuilder sb = new StringBuilder();
while (true)
{
int i = ThreadLocalStream.ReadByte();
if (-1 == i)
{
goto END;
}
if ('\t' == (char)i)
{
break; // real break, not conintue
}
sb.Append((char)i);
}
string rowKey = sb.ToString();
if (RowKeyToFilePosition.ContainsKey(rowKey))
{
throw new MatrixFormatException(string.Format(CultureInfo.InvariantCulture, "The rowkey {0} appears more than once", rowKey));
}
_rowKeys.Add(rowKey);
position += rowKey.Length + 1;
RowKeyToFilePosition.Add(rowKey, position);
position += ColCount * BytesPerValue + 2;//!!!assumes two char newlines
if (position > ThreadLocalStream.Length)
{
throw new MatrixFormatException("File seems too short");
}
}
END :;
_indexOfRowKey = RowKeys.Select((key, index) => new { key, index }).ToDictionary(keyAndIndex => keyAndIndex.key, keyAndIndex => keyAndIndex.index);
}
}
protected void GetInstanceFromRowKeysStructFileNameInternal(string rowKeysStructFileName, ParallelOptions parallelOptions, FileAccess fileAccess = FileAccess.Read, FileShare fileShare = FileShare.Read)
{
// parallelOptions is not currently used, but it is need so that this method will have the same signature as other, similar methods.
lock (this)
{
string firstLineOrNull = FileUtils.ReadLine(rowKeysStructFileName);
Helper.CheckCondition(null != firstLineOrNull, Properties.Resource.ExpectedFileToHaveData, rowKeysStructFileName);
Helper.CheckCondition(!firstLineOrNull.StartsWith(FileUtils.CommentHeader, StringComparison.Ordinal), Properties.Resource.ExpectedNoCommentsInRowKeysAnsiFiles, rowKeysStructFileName);
RowKeyToFilePosition = new Dictionary <string, long>();
FileAccess = fileAccess;
FileShare = fileShare;
using (TextReader textReader = File.OpenText(rowKeysStructFileName))
{
string colKeysLineOrNull = textReader.ReadLine();
string[] varAndColKeys = colKeysLineOrNull.Split('\t');
if (!varAndColKeys[0].Equals("rowKey"))
{
throw new MatrixFormatException("Expect first row's first value to be 'rowKey'"); //!!!rowKey
}
ColSerialNumbers = new SerialNumbers <string>(varAndColKeys.Skip(1));
_rowKeys = new List <string>();
if (null == colKeysLineOrNull)
{
throw new MatrixFormatException("Surprised by empty file. " + rowKeysStructFileName);
}
//!!!not really thread-safe
string denseStructFileNameInFile = textReader.ReadLine();
DenseStructFileName = Path.Combine(Path.GetDirectoryName(rowKeysStructFileName), denseStructFileNameInFile);
CounterWithMessages counterWithMessages = new CounterWithMessages("Reading rowKey file to find location of rows, #{0}", 10000, null);
string line = null;
while (null != (line = textReader.ReadLine()))
{
counterWithMessages.Increment();
string[] rowKeyAndPosition = line.Split('\t');
if (rowKeyAndPosition.Length != 2)
{
throw new MatrixFormatException("Expect rows to have two columns");
}
string rowKey = rowKeyAndPosition[0];
long position = long.Parse(rowKeyAndPosition[1], CultureInfo.CurrentCulture);
_rowKeys.Add(rowKey);
RowKeyToFilePosition.Add(rowKey, position);
}
}
Console.WriteLine("all lines read from file [{0}]", rowKeysStructFileName);
_indexOfRowKey = RowKeys.Select((key, index) => new { key, index }).ToDictionary(keyAndIndex => keyAndIndex.key, keyAndIndex => keyAndIndex.index);
Console.WriteLine("Dictionary created. Now testing values");
//Test that can really read values from data file
if (RowCount > 0 && ColCount > 0)
{
//!!!kludge - try up to 10 times to get a good value
GetValueOrMissing(0, 0);
//Console.WriteLine("GetValueOrMissing(0,0)={0} tested", value0);
int rowCount = RowCount;
//Console.WriteLine("rowCount is {0}", rowCount);
int colCount = ColCount;
//Console.WriteLine("colCount is {0}", colCount);
string rowKey = RowKeys[rowCount - 1];
//Console.WriteLine("rowKey is {0}", rowKey);
string colKey = ColKeys[colCount - 1];
//Console.WriteLine("colKey is {0}", colKey);
int colIndex = ColSerialNumbers.GetOld(colKey);
//Console.WriteLine("colIndex is {0}", colIndex);
byte[] byteArray = new byte[23]; //C# will init to 0's
//!!!kludge - try up to 10 times to get a good value
for (int i = 0; i < 10; ++i)
{
ThreadLocalStream.Position = 0;
ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue;
//Console.WriteLine("ThreadLocalStream.Position is {0}", ThreadLocalStream.Position);
byteArray = new byte[BytesPerValue];
int bytesRead = ThreadLocalStream.Read(byteArray, 0, BytesPerValue);
//Console.WriteLine("byteArray[0] is {0}", (int)byteArray[0]);
//Console.WriteLine("bytesRead is {0}", bytesRead);
if ((int)byteArray[0] != 0)
{
break;
}
//Console.WriteLine("Read a 0 instead of a 32, going to sleep for 10 seconds");
Thread.Sleep(10000);
Helper.CheckCondition(bytesRead == BytesPerValue, "Expected to read all the bytes of a value");
//Console.WriteLine("expected bytes read");
}
//string asString = System.Text.Encoding.Default.GetString(byteArray);
//Console.WriteLine("bytes to string is {0}", asString);
//TValue valueLast = Parser.Parse<TValue>(asString);
//Console.WriteLine("value is {0}", valueLast);
//Helper.CheckCondition(!valueLast.Equals(MissingValue), "Should not be missing"); //OK to use Equals because double can't be null
GetValueOrMissing(RowCount / 2, ColCount / 2);
//Console.WriteLine("GetValueOrMissing({0}, {1})={2} tested", RowCount / 2, ColCount / 2, valueMiddle);
}
//Console.WriteLine("Values tested. Done");
}
}
private void OldBadValueTester(string rowKeysStructFileName)
{
Console.WriteLine("Dictionary created. Now testing values");
//Test that can really read values from data file
if (RowCount > 0 && ColCount > 0)
{
try
{
//!!!kludge - try up to 10 times to get a good value
for (int i = 0; i < 10; ++i)
{
}
TValue value0 = this[0, 0];
Console.WriteLine("this[0, 0] tested");
int rowCount = RowCount;
Console.WriteLine("rowCount is {0}", rowCount);
int colCount = ColCount;
Console.WriteLine("colCount is {0}", colCount);
string rowKey = RowKeys[rowCount - 1];
Console.WriteLine("rowKey is {0}", rowKey);
string colKey = ColKeys[colCount - 1];
Console.WriteLine("colKey is {0}", colKey);
int colIndex = ColSerialNumbers.GetOld(colKey);
Console.WriteLine("colIndex is {0}", colIndex);
byte[] byteArray = new byte[23]; //C# will init to 0's
//!!!kludge - try up to 10 times to get a good value
for (int i = 0; i < 10; ++i)
{
ThreadLocalStream.Position = 0;
ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue;
Console.WriteLine("ThreadLocalStream.Position is {0}", ThreadLocalStream.Position);
byteArray = new byte[BytesPerValue];
int bytesRead = ThreadLocalStream.Read(byteArray, 0, BytesPerValue);
Console.WriteLine("byteArray[0] is {0}", (int)byteArray[0]);
Console.WriteLine("bytesRead is {0}", bytesRead);
if ((int)byteArray[0] != 0)
{
break;
}
Console.WriteLine("Read a 0 instead of a 32, going to sleep for 10 seconds");
Thread.Sleep(10000);
Helper.CheckCondition(bytesRead == BytesPerValue, "Expected to read all the bytes of a value");
Console.WriteLine("expected bytes read");
}
string asString = System.Text.Encoding.UTF8.GetString(byteArray, 0, byteArray.Length);
Console.WriteLine("bytes to string is ", asString);
//double valueLast = double.Parse(asString);
//Console.WriteLine("value is {0}", valueLast);
//Helper.CheckCondition(!valueLast.Equals(MissingValue), "Should not be missing"); //OK to use Equals because double can't be null
TValue valueMiddle = this[RowCount / 2, ColCount / 2];
Console.WriteLine("this[{0}, {1}] tested", RowCount / 2, ColCount / 2);
}
catch (Exception e)
{
throw new MatrixFormatException("RowKeyIndex seems to refer to a file of the wrong format. " + rowKeysStructFileName, e);
}
}
Console.WriteLine("Values tested. Done");
}
