public void Read(BionReader reader)
{
reader.Read(BionToken.StartArray);
reader.Read(BionToken.Integer);
int wordCount = (int)reader.CurrentInteger();
Words.Clear();
Index.Clear();
Indexed = false;
while (true)
{
reader.Read();
if (reader.TokenType == BionToken.EndArray)
{
break;
}
// NOTE: Not copying word. Must use a 'ReadAll' BufferedReader
reader.Expect(BionToken.String);
String8 value = reader.CurrentString8();
reader.Read(BionToken.Integer);
int count = (int)reader.CurrentInteger();
// Add to List, but not Index. Index will be populated when first needed.
Words.Add(value);
Counts.Add(count);
}
// Set capacity exact to save RAM
Words.SetCapacity(Words.LengthBytes);
}
public override bool Read()
{
bool success = _reader.Read();
switch (_reader.TokenType)
{
case BionToken.EndArray:
SetToken(JsonToken.EndArray);
break;
case BionToken.EndObject:
SetToken(JsonToken.EndObject);
break;
case BionToken.False:
SetToken(JsonToken.Boolean, false);
break;
case BionToken.Float:
SetToken(JsonToken.Float, _reader.CurrentFloat());
break;
case BionToken.Integer:
SetToken(JsonToken.Integer, (int)_reader.CurrentInteger());
break;
case BionToken.None:
SetToken(JsonToken.None);
break;
case BionToken.Null:
SetToken(JsonToken.Null, null);
break;
case BionToken.PropertyName:
SetToken(JsonToken.PropertyName, _reader.CurrentString());
break;
case BionToken.StartArray:
SetToken(JsonToken.StartArray);
break;
case BionToken.StartObject:
SetToken(JsonToken.StartObject);
break;
case BionToken.String:
SetToken(JsonToken.String, _reader.CurrentString());
break;
case BionToken.True:
SetToken(JsonToken.Boolean, true);
break;
default:
throw new NotImplementedException($"BionDataReader.Convert not implemented for '{_reader.TokenType}'.");
}
return(success);
}
public static void WriteToken(BionReader reader, JsonTextWriter writer)
{
switch (reader.TokenType)
{
case BionToken.StartObject:
writer.WriteStartObject();
break;
case BionToken.StartArray:
writer.WriteStartArray();
break;
case BionToken.EndObject:
writer.WriteEndObject();
break;
case BionToken.EndArray:
writer.WriteEndArray();
break;
case BionToken.PropertyName:
writer.WritePropertyName(reader.CurrentString());
break;
case BionToken.String:
writer.WriteValue(reader.CurrentString());
break;
case BionToken.Integer:
writer.WriteValue(reader.CurrentInteger());
break;
case BionToken.Float:
writer.WriteValue(reader.CurrentFloat());
break;
case BionToken.True:
case BionToken.False:
writer.WriteValue(reader.CurrentBool());
break;
case BionToken.Null:
writer.WriteNull();
break;
default:
throw new NotImplementedException($"BionToJson not implemented for {reader.TokenType} @{reader.BytesRead}.");
}
}
private static void Read(string filePath, string fromDictionaryPath)
{
VerifyFileExists(filePath);
VerifyFileExists(fromDictionaryPath);
long tokenCount = 0;
using (new ConsoleWatch($"Reading [Full] {filePath} ({FileLength.MB(filePath)})...",
() => $"Done; {tokenCount:n0} tokens found in file"))
{
if (filePath.EndsWith(".bion", StringComparison.OrdinalIgnoreCase))
{
using (WordCompressor compressor = (fromDictionaryPath == null ? null : WordCompressor.OpenRead(fromDictionaryPath)))
using (BionReader reader = new BionReader(File.OpenRead(filePath), compressor: compressor))
{
while (reader.Read())
{
tokenCount++;
switch (reader.TokenType)
{
case BionToken.PropertyName:
case BionToken.String:
String8 value8 = reader.CurrentString8();
//string valueS = reader.CurrentString();
break;
case BionToken.Integer:
long valueI = reader.CurrentInteger();
break;
case BionToken.Float:
double valueF = reader.CurrentFloat();
break;
}
}
}
}
else
{
using (JsonTextReader reader = new JsonTextReader(new StreamReader(filePath)))
{
while (reader.Read())
{
tokenCount++;
switch (reader.TokenType)
{
case JsonToken.PropertyName:
case JsonToken.String:
string valueS = (string)reader.Value;
break;
case JsonToken.Integer:
long valueI = (long)reader.Value;
break;
case JsonToken.Float:
double valueF = (double)reader.Value;
break;
}
}
}
}
}
}
