#pragma warning disable 1591 public override void SetValueOrMissing(string rowKey, string colKey, TValue value) #pragma warning restore 1591 { int colIndex = ColSerialNumbers.GetOld(colKey); ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue; byte[] byteArray = ValueOrMissingToByteArray(value); Helper.CheckCondition(byteArray.Length == BytesPerValue, "Expected byteArray to be the right length"); ThreadLocalStream.Write(byteArray, 0, BytesPerValue); }
#pragma warning disable 1591 public override void SetValueOrMissing(string rowKey, string colKey, TValue value) #pragma warning restore 1591 { int colIndex = ColSerialNumbers.GetOld(colKey); ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue; byte[] byteArray = ValueOrMissingToByteArray(value); Helper.CheckCondition(byteArray.Length == BytesPerValue, Properties.Resource.ExpectedByteArrayLengthAndBytesPerValueToBeEqual, byteArray.Length, BytesPerValue); ThreadLocalStream.Write(byteArray, 0, BytesPerValue); }
#pragma warning disable 1591 override public bool TryGetValue(string rowKey, string colKey, out TValue value) #pragma warning restore 1591 { int colIndex = ColSerialNumbers.GetOld(colKey); ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue; byte[] byteArray = new byte[BytesPerValue]; int bytesRead = ThreadLocalStream.Read(byteArray, 0, BytesPerValue); Helper.CheckCondition(bytesRead == BytesPerValue, Properties.Resource.ExpectedToReadAllBytesOfValue); value = ByteArrayToValueOrMissing(byteArray); return(!IsMissing(value)); }
private void ValueTester(string rowKeysStructFileName) { //Console.WriteLine("Index loaded. Now testing values"); //Test that can really read values from data file if (RowCount > 0 && ColCount > 0) { try { string rowKey = RowKeys[RowCount - 1]; //Check the width of the data if (RowCount > 1) { long rowLength = RowKeyToFilePosition[rowKey] - RowKeyToFilePosition[RowKeys[RowCount - 2]] - rowKey.Length - 1 /*tab*/ - 2 /*newline*/; Helper.CheckCondition <MatrixFormatException>(rowLength == BytesPerValue * ColCount, () => string.Format(CultureInfo.InvariantCulture, Properties.Resource.ExpectEachDataRowToHaveNCharacters, BytesPerValue * ColCount)); } //Check the length of the file long lastPosition = RowKeyToFilePosition[rowKey] + ColCount * BytesPerValue + 2 /*newline*/; Helper.CheckCondition <MatrixFormatException>(ThreadLocalStream.Length == lastPosition, Properties.Resource.ExpectFileToEndAfterLastValue); //Check that the last rowKey is where it is expected ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] - rowKey.Length - 1; byte[] byteArray = new byte[rowKey.Length]; int bytesRead = ThreadLocalStream.Read(byteArray, 0, rowKey.Length); Helper.CheckCondition <MatrixFormatException>(bytesRead == rowKey.Length, Properties.Resource.ExpectToReadRowKey); string asString = System.Text.Encoding.UTF8.GetString(byteArray, 0, byteArray.Length); Helper.CheckCondition <MatrixFormatException>(asString == rowKey, Properties.Resource.ExpectRowKeyFileAndMainFileToAgreeOnTheRowkeys); //Read some values GetValueOrMissing(0, 0); GetValueOrMissing(RowCount / 2, ColCount / 2); GetValueOrMissing(RowCount - 1, ColCount - 1); } catch (MatrixFormatException) { throw; } catch (Exception e) { throw new MatrixFormatException(Properties.Resource.RowKeyIndexSeemsToReferToAFileOfTheWrongFormat + rowKeysStructFileName, e); } } //Console.WriteLine("Values tested. Done"); }
private void ValueTester(string rowKeysStructFileName) { Console.WriteLine("Index loaded. Now testing values"); //Test that can really read values from data file if (RowCount > 0 && ColCount > 0) { try { string rowKey = RowKeys[RowCount - 1]; //Check the width of the data if (RowCount > 1) { long rowLength = RowKeyToFilePosition[rowKey] - RowKeyToFilePosition[RowKeys[RowCount - 2]] - rowKey.Length - 1 /*tab*/ - 2 /*newline*/; Helper.CheckCondition <MatrixFormatException>(rowLength == BytesPerValue * ColCount, "Expect each data row to have {0} characters.", BytesPerValue * ColCount); } //Check the length of the file long lastPosition = RowKeyToFilePosition[rowKey] + ColCount * BytesPerValue + 2 /*newline*/; Helper.CheckCondition <MatrixFormatException>(ThreadLocalStream.Length == lastPosition, "Expect file to end after last value"); //Check that the last rowKey is where it is expected ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] - rowKey.Length - 1; byte[] byteArray = new byte[rowKey.Length]; int bytesRead = ThreadLocalStream.Read(byteArray, 0, rowKey.Length); Helper.CheckCondition <MatrixFormatException>(bytesRead == rowKey.Length, "Expect to read rowKey"); string asString = System.Text.Encoding.UTF8.GetString(byteArray, 0, byteArray.Length); Helper.CheckCondition <MatrixFormatException>(asString == rowKey, "Expect rowKey file and main file to agree on the rowkeys"); //Read some values GetValueOrMissing(0, 0); GetValueOrMissing(RowCount / 2, ColCount / 2); GetValueOrMissing(RowCount - 1, ColCount - 1); } catch (MatrixFormatException) { throw; } catch (Exception e) { throw new MatrixFormatException("RowKeyIndex seems to refer to a file of the wrong format. " + rowKeysStructFileName, e); } } Console.WriteLine("Values tested. Done"); }
protected void GetInstanceFromDenseStructFileNameInternal(string denseStructFileName, ParallelOptions parallelOptions, FileAccess fileAccess = FileAccess.Read, FileShare fileShare = FileShare.Read) { // parallelOptions is not currently used, but it is need so that this method will have the same signature as other, similar methods. lock (this) { using (FileStream fileStream = File.Open(denseStructFileName, FileMode.Open, fileAccess, fileShare)) { using (TextReader textReader = new StreamReader(fileStream)) { string firstLineOrNull = textReader.ReadLine(); Helper.CheckCondition(null != firstLineOrNull, Properties.Resource.ExpectedFileToHaveData, denseStructFileName); Helper.CheckCondition(!firstLineOrNull.StartsWith(FileUtils.CommentHeader, StringComparison.Ordinal), Properties.Resource.ExpectedNoCommentsInRowKeysAnsiFiles, denseStructFileName); } } RowKeyToFilePosition = new Dictionary <string, long>(); DenseStructFileName = denseStructFileName; FileAccess = fileAccess; FileShare = fileShare; long position = 0; string colKeysLineOrNull = ThreadLocalTextReader.ReadLine(); position += colKeysLineOrNull.Length + 2; //!!!const assuming 2 char newslines string[] varAndColKeys = colKeysLineOrNull.Split('\t'); if (!varAndColKeys[0].Equals("var")) { throw new MatrixFormatException("Expect first row's first value to be 'var'"); } ColSerialNumbers = new SerialNumbers <string>(varAndColKeys.Skip(1)); _rowKeys = new List <string>(); if (null == colKeysLineOrNull) { throw new MatrixFormatException("Surprised by empty file. " + denseStructFileName); } CounterWithMessages counterWithMessages = new CounterWithMessages("Reading data file to find location of rows, #{0}", 10000, null); while (true) { counterWithMessages.Increment(); ThreadLocalStream.Position = position; StringBuilder sb = new StringBuilder(); while (true) { int i = ThreadLocalStream.ReadByte(); if (-1 == i) { goto END; } if ('\t' == (char)i) { break; // real break, not conintue } sb.Append((char)i); } string rowKey = sb.ToString(); if (RowKeyToFilePosition.ContainsKey(rowKey)) { throw new MatrixFormatException(string.Format(CultureInfo.InvariantCulture, "The rowkey {0} appears more than once", rowKey)); } _rowKeys.Add(rowKey); position += rowKey.Length + 1; RowKeyToFilePosition.Add(rowKey, position); position += ColCount * BytesPerValue + 2;//!!!assumes two char newlines if (position > ThreadLocalStream.Length) { throw new MatrixFormatException("File seems too short"); } } END :; _indexOfRowKey = RowKeys.Select((key, index) => new { key, index }).ToDictionary(keyAndIndex => keyAndIndex.key, keyAndIndex => keyAndIndex.index); } }
protected void GetInstanceFromRowKeysStructFileNameInternal(string rowKeysStructFileName, ParallelOptions parallelOptions, FileAccess fileAccess = FileAccess.Read, FileShare fileShare = FileShare.Read) { // parallelOptions is not currently used, but it is need so that this method will have the same signature as other, similar methods. lock (this) { string firstLineOrNull = FileUtils.ReadLine(rowKeysStructFileName); Helper.CheckCondition(null != firstLineOrNull, Properties.Resource.ExpectedFileToHaveData, rowKeysStructFileName); Helper.CheckCondition(!firstLineOrNull.StartsWith(FileUtils.CommentHeader, StringComparison.Ordinal), Properties.Resource.ExpectedNoCommentsInRowKeysAnsiFiles, rowKeysStructFileName); RowKeyToFilePosition = new Dictionary <string, long>(); FileAccess = fileAccess; FileShare = fileShare; using (TextReader textReader = File.OpenText(rowKeysStructFileName)) { string colKeysLineOrNull = textReader.ReadLine(); string[] varAndColKeys = colKeysLineOrNull.Split('\t'); if (!varAndColKeys[0].Equals("rowKey")) { throw new MatrixFormatException("Expect first row's first value to be 'rowKey'"); //!!!rowKey } ColSerialNumbers = new SerialNumbers <string>(varAndColKeys.Skip(1)); _rowKeys = new List <string>(); if (null == colKeysLineOrNull) { throw new MatrixFormatException("Surprised by empty file. " + rowKeysStructFileName); } //!!!not really thread-safe string denseStructFileNameInFile = textReader.ReadLine(); DenseStructFileName = Path.Combine(Path.GetDirectoryName(rowKeysStructFileName), denseStructFileNameInFile); CounterWithMessages counterWithMessages = new CounterWithMessages("Reading rowKey file to find location of rows, #{0}", 10000, null); string line = null; while (null != (line = textReader.ReadLine())) { counterWithMessages.Increment(); string[] rowKeyAndPosition = line.Split('\t'); if (rowKeyAndPosition.Length != 2) { throw new MatrixFormatException("Expect rows to have two columns"); } string rowKey = rowKeyAndPosition[0]; long position = long.Parse(rowKeyAndPosition[1], CultureInfo.CurrentCulture); _rowKeys.Add(rowKey); RowKeyToFilePosition.Add(rowKey, position); } } Console.WriteLine("all lines read from file [{0}]", rowKeysStructFileName); _indexOfRowKey = RowKeys.Select((key, index) => new { key, index }).ToDictionary(keyAndIndex => keyAndIndex.key, keyAndIndex => keyAndIndex.index); Console.WriteLine("Dictionary created. Now testing values"); //Test that can really read values from data file if (RowCount > 0 && ColCount > 0) { //!!!kludge - try up to 10 times to get a good value GetValueOrMissing(0, 0); //Console.WriteLine("GetValueOrMissing(0,0)={0} tested", value0); int rowCount = RowCount; //Console.WriteLine("rowCount is {0}", rowCount); int colCount = ColCount; //Console.WriteLine("colCount is {0}", colCount); string rowKey = RowKeys[rowCount - 1]; //Console.WriteLine("rowKey is {0}", rowKey); string colKey = ColKeys[colCount - 1]; //Console.WriteLine("colKey is {0}", colKey); int colIndex = ColSerialNumbers.GetOld(colKey); //Console.WriteLine("colIndex is {0}", colIndex); byte[] byteArray = new byte[23]; //C# will init to 0's //!!!kludge - try up to 10 times to get a good value for (int i = 0; i < 10; ++i) { ThreadLocalStream.Position = 0; ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue; //Console.WriteLine("ThreadLocalStream.Position is {0}", ThreadLocalStream.Position); byteArray = new byte[BytesPerValue]; int bytesRead = ThreadLocalStream.Read(byteArray, 0, BytesPerValue); //Console.WriteLine("byteArray[0] is {0}", (int)byteArray[0]); //Console.WriteLine("bytesRead is {0}", bytesRead); if ((int)byteArray[0] != 0) { break; } //Console.WriteLine("Read a 0 instead of a 32, going to sleep for 10 seconds"); Thread.Sleep(10000); Helper.CheckCondition(bytesRead == BytesPerValue, "Expected to read all the bytes of a value"); //Console.WriteLine("expected bytes read"); } //string asString = System.Text.Encoding.Default.GetString(byteArray); //Console.WriteLine("bytes to string is {0}", asString); //TValue valueLast = Parser.Parse<TValue>(asString); //Console.WriteLine("value is {0}", valueLast); //Helper.CheckCondition(!valueLast.Equals(MissingValue), "Should not be missing"); //OK to use Equals because double can't be null GetValueOrMissing(RowCount / 2, ColCount / 2); //Console.WriteLine("GetValueOrMissing({0}, {1})={2} tested", RowCount / 2, ColCount / 2, valueMiddle); } //Console.WriteLine("Values tested. Done"); } }
private void OldBadValueTester(string rowKeysStructFileName) { Console.WriteLine("Dictionary created. Now testing values"); //Test that can really read values from data file if (RowCount > 0 && ColCount > 0) { try { //!!!kludge - try up to 10 times to get a good value for (int i = 0; i < 10; ++i) { } TValue value0 = this[0, 0]; Console.WriteLine("this[0, 0] tested"); int rowCount = RowCount; Console.WriteLine("rowCount is {0}", rowCount); int colCount = ColCount; Console.WriteLine("colCount is {0}", colCount); string rowKey = RowKeys[rowCount - 1]; Console.WriteLine("rowKey is {0}", rowKey); string colKey = ColKeys[colCount - 1]; Console.WriteLine("colKey is {0}", colKey); int colIndex = ColSerialNumbers.GetOld(colKey); Console.WriteLine("colIndex is {0}", colIndex); byte[] byteArray = new byte[23]; //C# will init to 0's //!!!kludge - try up to 10 times to get a good value for (int i = 0; i < 10; ++i) { ThreadLocalStream.Position = 0; ThreadLocalStream.Position = RowKeyToFilePosition[rowKey] + colIndex * BytesPerValue; Console.WriteLine("ThreadLocalStream.Position is {0}", ThreadLocalStream.Position); byteArray = new byte[BytesPerValue]; int bytesRead = ThreadLocalStream.Read(byteArray, 0, BytesPerValue); Console.WriteLine("byteArray[0] is {0}", (int)byteArray[0]); Console.WriteLine("bytesRead is {0}", bytesRead); if ((int)byteArray[0] != 0) { break; } Console.WriteLine("Read a 0 instead of a 32, going to sleep for 10 seconds"); Thread.Sleep(10000); Helper.CheckCondition(bytesRead == BytesPerValue, "Expected to read all the bytes of a value"); Console.WriteLine("expected bytes read"); } string asString = System.Text.Encoding.UTF8.GetString(byteArray, 0, byteArray.Length); Console.WriteLine("bytes to string is ", asString); //double valueLast = double.Parse(asString); //Console.WriteLine("value is {0}", valueLast); //Helper.CheckCondition(!valueLast.Equals(MissingValue), "Should not be missing"); //OK to use Equals because double can't be null TValue valueMiddle = this[RowCount / 2, ColCount / 2]; Console.WriteLine("this[{0}, {1}] tested", RowCount / 2, ColCount / 2); } catch (Exception e) { throw new MatrixFormatException("RowKeyIndex seems to refer to a file of the wrong format. " + rowKeysStructFileName, e); } } Console.WriteLine("Values tested. Done"); }