private static void Skip(string filePath, string fromDictionaryPath)
{
VerifyFileExists(filePath);
VerifyFileExists(fromDictionaryPath);
using (new ConsoleWatch($"Reading [Skip All] {filePath} ({FileLength.MB(filePath)})..."))
{
if (filePath.EndsWith(".bion", StringComparison.OrdinalIgnoreCase))
{
using (WordCompressor compressor = (fromDictionaryPath == null ? null : WordCompressor.OpenRead(fromDictionaryPath)))
using (BionReader reader = new BionReader(File.OpenRead(filePath), compressor: compressor))
{
reader.Skip();
}
}
else
{
using (JsonTextReader reader = new JsonTextReader(new StreamReader(filePath)))
{
reader.Read();
reader.Skip();
}
}
}
}
static List <Region2> BionDirectToClass(string bionPath)
{
using (BionReader reader = new BionReader(File.OpenRead(bionPath)))
{
return(_bionDirectDeserializer.DeserializeRegion2s(reader));
}
}
private static void Count(string filePath, string fromDictionaryPath)
{
VerifyFileExists(filePath);
VerifyFileExists(fromDictionaryPath);
long tokenCount = 0;
using (new ConsoleWatch($"Reading [Count] {filePath} ({FileLength.MB(filePath)})...",
() => $"Done; {tokenCount:n0} tokens found in file"))
{
if (filePath.EndsWith(".bion", StringComparison.OrdinalIgnoreCase))
{
using (WordCompressor compressor = (fromDictionaryPath == null ? null : WordCompressor.OpenRead(fromDictionaryPath)))
using (BionReader reader = new BionReader(File.OpenRead(filePath), compressor: compressor))
{
while (reader.Read())
{
tokenCount++;
}
}
}
else
{
using (JsonTextReader reader = new JsonTextReader(new StreamReader(filePath)))
{
while (reader.Read())
{
tokenCount++;
}
}
}
}
}
private static void RoundTrip(Action <BionWriter> write, Action <BionReader> readAndVerify)
{
using (MemoryStream stream = new MemoryStream())
{
// Write desired value(s) without closing stream
using (BionWriter writer = new BionWriter(new BufferedWriter(stream)
{
CloseStream = false
}))
{
write(writer);
}
// Track position and seek back
long length = stream.Position;
stream.Seek(0, SeekOrigin.Begin);
// Read, verify, validate position and no more content
using (BionReader reader = new BionReader(new BufferedReader(stream)
{
CloseStream = false
}))
{
readAndVerify(reader);
Assert.AreEqual(length, reader.BytesRead);
Assert.IsFalse(reader.Read());
}
}
}
public void Read(BionReader reader)
{
reader.Read(BionToken.StartArray);
reader.Read(BionToken.Integer);
int wordCount = (int)reader.CurrentInteger();
Words.Clear();
Index.Clear();
Indexed = false;
while (true)
{
reader.Read();
if (reader.TokenType == BionToken.EndArray)
{
break;
}
// NOTE: Not copying word. Must use a 'ReadAll' BufferedReader
reader.Expect(BionToken.String);
String8 value = reader.CurrentString8();
reader.Read(BionToken.Integer);
int count = (int)reader.CurrentInteger();
// Add to List, but not Index. Index will be populated when first needed.
Words.Add(value);
Counts.Add(count);
}
// Set capacity exact to save RAM
Words.SetCapacity(Words.LengthBytes);
}
public static void BionToJson(string bionPath, string jsonPath, string fromDictionaryPath = null)
{
using (WordCompressor compressor = (String.IsNullOrEmpty(fromDictionaryPath) ? null : WordCompressor.OpenRead(fromDictionaryPath)))
using (BionReader reader = new BionReader(File.OpenRead(bionPath), compressor: compressor))
using (JsonTextWriter writer = new JsonTextWriter(new StreamWriter(jsonPath)))
{
writer.Formatting = Formatting.Indented;
BionToJson(reader, writer);
}
}
public BionSearcher(string bionFilePath, int runDepth)
{
_compressor = Memory.Log("Dictionary", () => WordCompressor.OpenRead(Path.ChangeExtension(bionFilePath, ".wdx")));
_containerIndex = Memory.Log("ContainerIndex", () => ContainerIndex.OpenRead(Path.ChangeExtension(bionFilePath, ".cdx")));
_searchIndexReader = Memory.Log("SearchIndex", () => new SearchIndexReader(Path.ChangeExtension(bionFilePath, ".idx")));
_bionReader = Memory.Log("BionReader", () => new BionReader(File.OpenRead(bionFilePath), containerIndex: _containerIndex, compressor: _compressor));
_runDepth = runDepth;
_termPositions = new long[256];
}
public static WordCompressor OpenRead(string dictionaryPath)
{
WordCompressor compressor = new WordCompressor();
using (BionReader reader = new BionReader(File.OpenRead(dictionaryPath)))
{
compressor.Read(reader);
}
return(compressor);
}
private static void Expect(BionReader reader, BionToken requiredToken, string parseContext)
{
if (reader.TokenType == BionToken.None)
{
reader.Read();
}
if (reader.TokenType != requiredToken)
{
throw new IOException($"Reader found invalid token type '{requiredToken}' while parsing {parseContext}.");
}
reader.Read();
}
// -----
public List <Region2> DeserializeRegion2s(BionReader reader)
{
List <Region2> result = new List <Region2>();
Expect(reader, BionToken.StartArray, "List<Region2>");
while (reader.TokenType != BionToken.EndArray)
{
result.Add(DeserializeRegion2(reader));
}
Expect(reader, BionToken.EndArray, "List<Region2>");
return(result);
}
public void Dispose()
{
_compressor?.Dispose();
_compressor = null;
_searchIndexReader?.Dispose();
_searchIndexReader = null;
_containerIndex?.Dispose();
_containerIndex = null;
_bionReader?.Dispose();
_bionReader = null;
}
public static void WriteToken(BionReader reader, JsonTextWriter writer)
{
switch (reader.TokenType)
{
case BionToken.StartObject:
writer.WriteStartObject();
break;
case BionToken.StartArray:
writer.WriteStartArray();
break;
case BionToken.EndObject:
writer.WriteEndObject();
break;
case BionToken.EndArray:
writer.WriteEndArray();
break;
case BionToken.PropertyName:
writer.WritePropertyName(reader.CurrentString());
break;
case BionToken.String:
writer.WriteValue(reader.CurrentString());
break;
case BionToken.Integer:
writer.WriteValue(reader.CurrentInteger());
break;
case BionToken.Float:
writer.WriteValue(reader.CurrentFloat());
break;
case BionToken.True:
case BionToken.False:
writer.WriteValue(reader.CurrentBool());
break;
case BionToken.Null:
writer.WriteNull();
break;
default:
throw new NotImplementedException($"BionToJson not implemented for {reader.TokenType} @{reader.BytesRead}.");
}
}
public static void BionToJson(BionReader reader, JsonTextWriter writer)
{
// 8.6s
//int untilDepth = reader.Depth;
//while (reader.Read())
//{
// WriteToken(reader, writer);
// if (reader.Depth == untilDepth) { break; }
//}
// 11.4s
using (BionDataReader dataReader = new BionDataReader(reader))
{
writer.WriteToken(dataReader);
}
}
public void ContainerIndex_EndToEnd()
{
string jsonFilePath = @"Content\Medium.json";
string bionFilePath = Path.ChangeExtension(jsonFilePath, ".bion");
string dictionaryPath = Path.ChangeExtension(bionFilePath, "dict.bion");
string comparePath = Path.ChangeExtension(jsonFilePath, "compare.json");
JsonBionConverter.JsonToBion(jsonFilePath, bionFilePath, dictionaryPath);
using (WordCompressor compressor = WordCompressor.OpenRead(dictionaryPath))
using (ContainerIndex cIndex = ContainerIndex.OpenRead(Path.ChangeExtension(bionFilePath, ".cdx")))
using (BionReader reader = new BionReader(File.OpenRead(bionFilePath), cIndex, compressor))
{
for (int i = 0; i < cIndex.Count; ++i)
{
ContainerEntry container = cIndex[i];
// Seek to container start
reader.Seek(container.StartByteOffset);
// Verify a container start is there
int depth = reader.Depth;
reader.Read();
bool isObject = (reader.TokenType == BionToken.StartObject);
Assert.AreEqual((isObject ? BionToken.StartObject : BionToken.StartArray), reader.TokenType);
// Read until the depth is back to the same value
while (reader.Depth != depth)
{
reader.Read();
}
// Verify this is the end container position
Assert.AreEqual((isObject ? BionToken.EndObject : BionToken.EndArray), reader.TokenType);
Assert.AreEqual(container.EndByteOffset, reader.BytesRead);
}
}
}
public Region2 DeserializeRegion2(BionReader reader)
{
Region2 result = new Region2();
Expect(reader, BionToken.StartObject, "Region2");
while (reader.TokenType == BionToken.PropertyName)
{
String8 propertyName = reader.CurrentString8();
if (!classFieldParsers.TryGetValue(propertyName, out Action <BionReader, Region2> parser))
{
throw new NotImplementedException($"Unknown property Region.{propertyName}. Stopping.");
}
reader.Read();
parser(reader, result);
reader.Read();
}
Expect(reader, BionToken.EndObject, "Region2");
return(result);
}
private void Read(BionReader reader)
{
_words.Read(reader);
}
public BionDataReader(BionReader reader)
{
_reader = reader;
}
public BionDataReader(Stream stream)
{
_reader = new BionReader(stream);
}
public override void Close()
{
_reader?.Dispose();
_reader = null;
}
private static void Read(string filePath, string fromDictionaryPath)
{
VerifyFileExists(filePath);
VerifyFileExists(fromDictionaryPath);
long tokenCount = 0;
using (new ConsoleWatch($"Reading [Full] {filePath} ({FileLength.MB(filePath)})...",
() => $"Done; {tokenCount:n0} tokens found in file"))
{
if (filePath.EndsWith(".bion", StringComparison.OrdinalIgnoreCase))
{
using (WordCompressor compressor = (fromDictionaryPath == null ? null : WordCompressor.OpenRead(fromDictionaryPath)))
using (BionReader reader = new BionReader(File.OpenRead(filePath), compressor: compressor))
{
while (reader.Read())
{
tokenCount++;
switch (reader.TokenType)
{
case BionToken.PropertyName:
case BionToken.String:
String8 value8 = reader.CurrentString8();
//string valueS = reader.CurrentString();
break;
case BionToken.Integer:
long valueI = reader.CurrentInteger();
break;
case BionToken.Float:
double valueF = reader.CurrentFloat();
break;
}
}
}
}
else
{
using (JsonTextReader reader = new JsonTextReader(new StreamReader(filePath)))
{
while (reader.Read())
{
tokenCount++;
switch (reader.TokenType)
{
case JsonToken.PropertyName:
case JsonToken.String:
string valueS = (string)reader.Value;
break;
case JsonToken.Integer:
long valueI = (long)reader.Value;
break;
case JsonToken.Float:
double valueF = (double)reader.Value;
break;
}
}
}
}
}
}
