public void Read(BionReader reader) { reader.Read(BionToken.StartArray); reader.Read(BionToken.Integer); int wordCount = (int)reader.CurrentInteger(); Words.Clear(); Index.Clear(); Indexed = false; while (true) { reader.Read(); if (reader.TokenType == BionToken.EndArray) { break; } // NOTE: Not copying word. Must use a 'ReadAll' BufferedReader reader.Expect(BionToken.String); String8 value = reader.CurrentString8(); reader.Read(BionToken.Integer); int count = (int)reader.CurrentInteger(); // Add to List, but not Index. Index will be populated when first needed. Words.Add(value); Counts.Add(count); } // Set capacity exact to save RAM Words.SetCapacity(Words.LengthBytes); }
public override bool Read() { bool success = _reader.Read(); switch (_reader.TokenType) { case BionToken.EndArray: SetToken(JsonToken.EndArray); break; case BionToken.EndObject: SetToken(JsonToken.EndObject); break; case BionToken.False: SetToken(JsonToken.Boolean, false); break; case BionToken.Float: SetToken(JsonToken.Float, _reader.CurrentFloat()); break; case BionToken.Integer: SetToken(JsonToken.Integer, (int)_reader.CurrentInteger()); break; case BionToken.None: SetToken(JsonToken.None); break; case BionToken.Null: SetToken(JsonToken.Null, null); break; case BionToken.PropertyName: SetToken(JsonToken.PropertyName, _reader.CurrentString()); break; case BionToken.StartArray: SetToken(JsonToken.StartArray); break; case BionToken.StartObject: SetToken(JsonToken.StartObject); break; case BionToken.String: SetToken(JsonToken.String, _reader.CurrentString()); break; case BionToken.True: SetToken(JsonToken.Boolean, true); break; default: throw new NotImplementedException($"BionDataReader.Convert not implemented for '{_reader.TokenType}'."); } return(success); }
public static void WriteToken(BionReader reader, JsonTextWriter writer) { switch (reader.TokenType) { case BionToken.StartObject: writer.WriteStartObject(); break; case BionToken.StartArray: writer.WriteStartArray(); break; case BionToken.EndObject: writer.WriteEndObject(); break; case BionToken.EndArray: writer.WriteEndArray(); break; case BionToken.PropertyName: writer.WritePropertyName(reader.CurrentString()); break; case BionToken.String: writer.WriteValue(reader.CurrentString()); break; case BionToken.Integer: writer.WriteValue(reader.CurrentInteger()); break; case BionToken.Float: writer.WriteValue(reader.CurrentFloat()); break; case BionToken.True: case BionToken.False: writer.WriteValue(reader.CurrentBool()); break; case BionToken.Null: writer.WriteNull(); break; default: throw new NotImplementedException($"BionToJson not implemented for {reader.TokenType} @{reader.BytesRead}."); } }
private static void Read(string filePath, string fromDictionaryPath) { VerifyFileExists(filePath); VerifyFileExists(fromDictionaryPath); long tokenCount = 0; using (new ConsoleWatch($"Reading [Full] {filePath} ({FileLength.MB(filePath)})...", () => $"Done; {tokenCount:n0} tokens found in file")) { if (filePath.EndsWith(".bion", StringComparison.OrdinalIgnoreCase)) { using (WordCompressor compressor = (fromDictionaryPath == null ? null : WordCompressor.OpenRead(fromDictionaryPath))) using (BionReader reader = new BionReader(File.OpenRead(filePath), compressor: compressor)) { while (reader.Read()) { tokenCount++; switch (reader.TokenType) { case BionToken.PropertyName: case BionToken.String: String8 value8 = reader.CurrentString8(); //string valueS = reader.CurrentString(); break; case BionToken.Integer: long valueI = reader.CurrentInteger(); break; case BionToken.Float: double valueF = reader.CurrentFloat(); break; } } } } else { using (JsonTextReader reader = new JsonTextReader(new StreamReader(filePath))) { while (reader.Read()) { tokenCount++; switch (reader.TokenType) { case JsonToken.PropertyName: case JsonToken.String: string valueS = (string)reader.Value; break; case JsonToken.Integer: long valueI = (long)reader.Value; break; case JsonToken.Float: double valueF = (double)reader.Value; break; } } } } } }