Ejemplo n.º 1
0
        private static void Skip(string filePath, string fromDictionaryPath)
        {
            VerifyFileExists(filePath);
            VerifyFileExists(fromDictionaryPath);

            using (new ConsoleWatch($"Reading [Skip All] {filePath} ({FileLength.MB(filePath)})..."))
            {
                if (filePath.EndsWith(".bion", StringComparison.OrdinalIgnoreCase))
                {
                    using (WordCompressor compressor = (fromDictionaryPath == null ? null : WordCompressor.OpenRead(fromDictionaryPath)))
                        using (BionReader reader = new BionReader(File.OpenRead(filePath), compressor: compressor))
                        {
                            reader.Skip();
                        }
                }
                else
                {
                    using (JsonTextReader reader = new JsonTextReader(new StreamReader(filePath)))
                    {
                        reader.Read();
                        reader.Skip();
                    }
                }
            }
        }
Ejemplo n.º 2
0
 static List <Region2> BionDirectToClass(string bionPath)
 {
     using (BionReader reader = new BionReader(File.OpenRead(bionPath)))
     {
         return(_bionDirectDeserializer.DeserializeRegion2s(reader));
     }
 }
Ejemplo n.º 3
0
        private static void Count(string filePath, string fromDictionaryPath)
        {
            VerifyFileExists(filePath);
            VerifyFileExists(fromDictionaryPath);
            long tokenCount = 0;

            using (new ConsoleWatch($"Reading [Count] {filePath} ({FileLength.MB(filePath)})...",
                                    () => $"Done; {tokenCount:n0} tokens found in file"))
            {
                if (filePath.EndsWith(".bion", StringComparison.OrdinalIgnoreCase))
                {
                    using (WordCompressor compressor = (fromDictionaryPath == null ? null : WordCompressor.OpenRead(fromDictionaryPath)))
                        using (BionReader reader = new BionReader(File.OpenRead(filePath), compressor: compressor))
                        {
                            while (reader.Read())
                            {
                                tokenCount++;
                            }
                        }
                }
                else
                {
                    using (JsonTextReader reader = new JsonTextReader(new StreamReader(filePath)))
                    {
                        while (reader.Read())
                        {
                            tokenCount++;
                        }
                    }
                }
            }
        }
Ejemplo n.º 4
0
        private static void RoundTrip(Action <BionWriter> write, Action <BionReader> readAndVerify)
        {
            using (MemoryStream stream = new MemoryStream())
            {
                // Write desired value(s) without closing stream
                using (BionWriter writer = new BionWriter(new BufferedWriter(stream)
                {
                    CloseStream = false
                }))
                {
                    write(writer);
                }

                // Track position and seek back
                long length = stream.Position;
                stream.Seek(0, SeekOrigin.Begin);

                // Read, verify, validate position and no more content
                using (BionReader reader = new BionReader(new BufferedReader(stream)
                {
                    CloseStream = false
                }))
                {
                    readAndVerify(reader);

                    Assert.AreEqual(length, reader.BytesRead);
                    Assert.IsFalse(reader.Read());
                }
            }
        }
Ejemplo n.º 5
0
        public void Read(BionReader reader)
        {
            reader.Read(BionToken.StartArray);
            reader.Read(BionToken.Integer);
            int wordCount = (int)reader.CurrentInteger();

            Words.Clear();

            Index.Clear();
            Indexed = false;

            while (true)
            {
                reader.Read();
                if (reader.TokenType == BionToken.EndArray)
                {
                    break;
                }

                // NOTE: Not copying word. Must use a 'ReadAll' BufferedReader
                reader.Expect(BionToken.String);
                String8 value = reader.CurrentString8();

                reader.Read(BionToken.Integer);
                int count = (int)reader.CurrentInteger();

                // Add to List, but not Index. Index will be populated when first needed.
                Words.Add(value);
                Counts.Add(count);
            }

            // Set capacity exact to save RAM
            Words.SetCapacity(Words.LengthBytes);
        }
Ejemplo n.º 6
0
 public static void BionToJson(string bionPath, string jsonPath, string fromDictionaryPath = null)
 {
     using (WordCompressor compressor = (String.IsNullOrEmpty(fromDictionaryPath) ? null : WordCompressor.OpenRead(fromDictionaryPath)))
         using (BionReader reader = new BionReader(File.OpenRead(bionPath), compressor: compressor))
             using (JsonTextWriter writer = new JsonTextWriter(new StreamWriter(jsonPath)))
             {
                 writer.Formatting = Formatting.Indented;
                 BionToJson(reader, writer);
             }
 }
Ejemplo n.º 7
0
        public BionSearcher(string bionFilePath, int runDepth)
        {
            _compressor        = Memory.Log("Dictionary", () => WordCompressor.OpenRead(Path.ChangeExtension(bionFilePath, ".wdx")));
            _containerIndex    = Memory.Log("ContainerIndex", () => ContainerIndex.OpenRead(Path.ChangeExtension(bionFilePath, ".cdx")));
            _searchIndexReader = Memory.Log("SearchIndex", () => new SearchIndexReader(Path.ChangeExtension(bionFilePath, ".idx")));
            _bionReader        = Memory.Log("BionReader", () => new BionReader(File.OpenRead(bionFilePath), containerIndex: _containerIndex, compressor: _compressor));

            _runDepth      = runDepth;
            _termPositions = new long[256];
        }
Ejemplo n.º 8
0
        public static WordCompressor OpenRead(string dictionaryPath)
        {
            WordCompressor compressor = new WordCompressor();

            using (BionReader reader = new BionReader(File.OpenRead(dictionaryPath)))
            {
                compressor.Read(reader);
            }

            return(compressor);
        }
        private static void Expect(BionReader reader, BionToken requiredToken, string parseContext)
        {
            if (reader.TokenType == BionToken.None)
            {
                reader.Read();
            }

            if (reader.TokenType != requiredToken)
            {
                throw new IOException($"Reader found invalid token type '{requiredToken}' while parsing {parseContext}.");
            }

            reader.Read();
        }
Ejemplo n.º 10
0
        // -----

        public List <Region2> DeserializeRegion2s(BionReader reader)
        {
            List <Region2> result = new List <Region2>();

            Expect(reader, BionToken.StartArray, "List<Region2>");

            while (reader.TokenType != BionToken.EndArray)
            {
                result.Add(DeserializeRegion2(reader));
            }

            Expect(reader, BionToken.EndArray, "List<Region2>");
            return(result);
        }
Ejemplo n.º 11
0
        public void Dispose()
        {
            _compressor?.Dispose();
            _compressor = null;

            _searchIndexReader?.Dispose();
            _searchIndexReader = null;

            _containerIndex?.Dispose();
            _containerIndex = null;

            _bionReader?.Dispose();
            _bionReader = null;
        }
Ejemplo n.º 12
0
        public static void WriteToken(BionReader reader, JsonTextWriter writer)
        {
            switch (reader.TokenType)
            {
            case BionToken.StartObject:
                writer.WriteStartObject();
                break;

            case BionToken.StartArray:
                writer.WriteStartArray();
                break;

            case BionToken.EndObject:
                writer.WriteEndObject();
                break;

            case BionToken.EndArray:
                writer.WriteEndArray();
                break;

            case BionToken.PropertyName:
                writer.WritePropertyName(reader.CurrentString());
                break;

            case BionToken.String:
                writer.WriteValue(reader.CurrentString());
                break;

            case BionToken.Integer:
                writer.WriteValue(reader.CurrentInteger());
                break;

            case BionToken.Float:
                writer.WriteValue(reader.CurrentFloat());
                break;

            case BionToken.True:
            case BionToken.False:
                writer.WriteValue(reader.CurrentBool());
                break;

            case BionToken.Null:
                writer.WriteNull();
                break;

            default:
                throw new NotImplementedException($"BionToJson not implemented for {reader.TokenType} @{reader.BytesRead}.");
            }
        }
Ejemplo n.º 13
0
        public static void BionToJson(BionReader reader, JsonTextWriter writer)
        {
            // 8.6s
            //int untilDepth = reader.Depth;

            //while (reader.Read())
            //{
            //    WriteToken(reader, writer);
            //    if (reader.Depth == untilDepth) { break; }
            //}

            // 11.4s
            using (BionDataReader dataReader = new BionDataReader(reader))
            {
                writer.WriteToken(dataReader);
            }
        }
Ejemplo n.º 14
0
        public void ContainerIndex_EndToEnd()
        {
            string jsonFilePath   = @"Content\Medium.json";
            string bionFilePath   = Path.ChangeExtension(jsonFilePath, ".bion");
            string dictionaryPath = Path.ChangeExtension(bionFilePath, "dict.bion");
            string comparePath    = Path.ChangeExtension(jsonFilePath, "compare.json");

            JsonBionConverter.JsonToBion(jsonFilePath, bionFilePath, dictionaryPath);

            using (WordCompressor compressor = WordCompressor.OpenRead(dictionaryPath))
                using (ContainerIndex cIndex = ContainerIndex.OpenRead(Path.ChangeExtension(bionFilePath, ".cdx")))
                    using (BionReader reader = new BionReader(File.OpenRead(bionFilePath), cIndex, compressor))
                    {
                        for (int i = 0; i < cIndex.Count; ++i)
                        {
                            ContainerEntry container = cIndex[i];

                            // Seek to container start
                            reader.Seek(container.StartByteOffset);

                            // Verify a container start is there
                            int depth = reader.Depth;
                            reader.Read();

                            bool isObject = (reader.TokenType == BionToken.StartObject);
                            Assert.AreEqual((isObject ? BionToken.StartObject : BionToken.StartArray), reader.TokenType);

                            // Read until the depth is back to the same value
                            while (reader.Depth != depth)
                            {
                                reader.Read();
                            }

                            // Verify this is the end container position
                            Assert.AreEqual((isObject ? BionToken.EndObject : BionToken.EndArray), reader.TokenType);
                            Assert.AreEqual(container.EndByteOffset, reader.BytesRead);
                        }
                    }
        }
Ejemplo n.º 15
0
        public Region2 DeserializeRegion2(BionReader reader)
        {
            Region2 result = new Region2();

            Expect(reader, BionToken.StartObject, "Region2");

            while (reader.TokenType == BionToken.PropertyName)
            {
                String8 propertyName = reader.CurrentString8();

                if (!classFieldParsers.TryGetValue(propertyName, out Action <BionReader, Region2> parser))
                {
                    throw new NotImplementedException($"Unknown property Region.{propertyName}. Stopping.");
                }

                reader.Read();
                parser(reader, result);
                reader.Read();
            }

            Expect(reader, BionToken.EndObject, "Region2");
            return(result);
        }
Ejemplo n.º 16
0
 private void Read(BionReader reader)
 {
     _words.Read(reader);
 }
Ejemplo n.º 17
0
 public BionDataReader(BionReader reader)
 {
     _reader = reader;
 }
Ejemplo n.º 18
0
 public BionDataReader(Stream stream)
 {
     _reader = new BionReader(stream);
 }
Ejemplo n.º 19
0
 public override void Close()
 {
     _reader?.Dispose();
     _reader = null;
 }
Ejemplo n.º 20
0
        private static void Read(string filePath, string fromDictionaryPath)
        {
            VerifyFileExists(filePath);
            VerifyFileExists(fromDictionaryPath);
            long tokenCount = 0;

            using (new ConsoleWatch($"Reading [Full] {filePath} ({FileLength.MB(filePath)})...",
                                    () => $"Done; {tokenCount:n0} tokens found in file"))
            {
                if (filePath.EndsWith(".bion", StringComparison.OrdinalIgnoreCase))
                {
                    using (WordCompressor compressor = (fromDictionaryPath == null ? null : WordCompressor.OpenRead(fromDictionaryPath)))
                        using (BionReader reader = new BionReader(File.OpenRead(filePath), compressor: compressor))
                        {
                            while (reader.Read())
                            {
                                tokenCount++;

                                switch (reader.TokenType)
                                {
                                case BionToken.PropertyName:
                                case BionToken.String:
                                    String8 value8 = reader.CurrentString8();
                                    //string valueS = reader.CurrentString();
                                    break;

                                case BionToken.Integer:
                                    long valueI = reader.CurrentInteger();
                                    break;

                                case BionToken.Float:
                                    double valueF = reader.CurrentFloat();
                                    break;
                                }
                            }
                        }
                }
                else
                {
                    using (JsonTextReader reader = new JsonTextReader(new StreamReader(filePath)))
                    {
                        while (reader.Read())
                        {
                            tokenCount++;

                            switch (reader.TokenType)
                            {
                            case JsonToken.PropertyName:
                            case JsonToken.String:
                                string valueS = (string)reader.Value;
                                break;

                            case JsonToken.Integer:
                                long valueI = (long)reader.Value;
                                break;

                            case JsonToken.Float:
                                double valueF = (double)reader.Value;
                                break;
                            }
                        }
                    }
                }
            }
        }