private static void CompareUncompressed(string jsonFilePath) { string bionFilePath = Path.ChangeExtension(jsonFilePath, ".bion"); string comparePath = Path.ChangeExtension(jsonFilePath, "compare.json"); JsonBionConverter.JsonToBion(jsonFilePath, bionFilePath); JsonBionConverter.BionToJson(bionFilePath, comparePath); Verify.JsonEqual(jsonFilePath, comparePath); }
private static void ToBion(string jsonPath, string bionPath, string dictionaryPath) { VerifyFileExists(jsonPath); using (new ConsoleWatch($"Converting {jsonPath} to {bionPath}...", () => $"Done. {FileLength.MB(jsonPath)} JSON to {FileLength.MB(bionPath)} BION{(String.IsNullOrEmpty(dictionaryPath) ? "" : $" + {FileLength.MB(dictionaryPath)} dictionary")} ({FileLength.Percentage(jsonPath, bionPath, dictionaryPath)})")) { JsonBionConverter.JsonToBion(jsonPath, bionPath, dictionaryPath); } }
private static void ToJson(string fromPath, string toPath, string dictionaryPath) { VerifyFileExists(fromPath); VerifyFileExists(dictionaryPath); using (new ConsoleWatch($"Converting {fromPath} to {toPath}...", () => $"Done. {FileLength.MB(fromPath)} BION to {FileLength.MB(toPath)} JSON")) { JsonBionConverter.BionToJson(fromPath, toPath, dictionaryPath); } }
private static void Index(string jsonRootPath, string bionRootPath) { using (new ConsoleWatch($"Converting under \"{jsonRootPath}\" to \"{bionRootPath}\"...")) { jsonRootPath = Path.GetFullPath(jsonRootPath); foreach (string jsonFilePath in Directory.EnumerateFiles(jsonRootPath, "*.*", SearchOption.AllDirectories)) { string jsonFilePathUnderRoot = jsonFilePath.Substring(jsonRootPath.Length + 1); string outputPath = Path.ChangeExtension(Path.Combine(bionRootPath, jsonFilePathUnderRoot), ".bion"); if (!File.Exists(outputPath)) { System.Console.WriteLine($" {jsonFilePathUnderRoot}"); Directory.CreateDirectory(Path.GetDirectoryName(outputPath)); JsonBionConverter.JsonToBion(jsonFilePath, outputPath, Path.ChangeExtension(outputPath, ".wdx")); } } } }
public void ContainerIndex_EndToEnd() { string jsonFilePath = @"Content\Medium.json"; string bionFilePath = Path.ChangeExtension(jsonFilePath, ".bion"); string dictionaryPath = Path.ChangeExtension(bionFilePath, "dict.bion"); string comparePath = Path.ChangeExtension(jsonFilePath, "compare.json"); JsonBionConverter.JsonToBion(jsonFilePath, bionFilePath, dictionaryPath); using (WordCompressor compressor = WordCompressor.OpenRead(dictionaryPath)) using (ContainerIndex cIndex = ContainerIndex.OpenRead(Path.ChangeExtension(bionFilePath, ".cdx"))) using (BionReader reader = new BionReader(File.OpenRead(bionFilePath), cIndex, compressor)) { for (int i = 0; i < cIndex.Count; ++i) { ContainerEntry container = cIndex[i]; // Seek to container start reader.Seek(container.StartByteOffset); // Verify a container start is there int depth = reader.Depth; reader.Read(); bool isObject = (reader.TokenType == BionToken.StartObject); Assert.AreEqual((isObject ? BionToken.StartObject : BionToken.StartArray), reader.TokenType); // Read until the depth is back to the same value while (reader.Depth != depth) { reader.Read(); } // Verify this is the end container position Assert.AreEqual((isObject ? BionToken.EndObject : BionToken.EndArray), reader.TokenType); Assert.AreEqual(container.EndByteOffset, reader.BytesRead); } } }
private void WriteRunSubsetStart(JsonTextWriter writer) { int depth = _bionReader.Depth; // Read and copy everything in the run until the end object except the large collections while (_bionReader.Read() && _bionReader.Depth > depth) { if (_bionReader.TokenType == BionToken.PropertyName) { String8 propertyName = _bionReader.CurrentString8(); if (propertyName.Equals(Files) || propertyName.Equals(Results)) { _bionReader.Skip(); continue; } } JsonBionConverter.WriteToken(_bionReader, writer); } // Start a results array and leave open writer.WritePropertyName("results"); writer.WriteStartArray(); }
public int Write(JsonTextWriter writer, ISearchResult wordMatches, int skip = 0, int take = -1) { int matchCount = 0; ContainerEntry lastRun = ContainerEntry.Empty; ContainerEntry lastResult = ContainerEntry.Empty; while (!wordMatches.Done) { int count = wordMatches.Page(ref _termPositions); for (int i = 0; i < count; ++i) { long position = _termPositions[i]; // If this position isn't in the last run's results array, find the run which contains it if (!lastRun.Contains(position)) { // Close previous run, if any if (!lastRun.IsEmpty()) { WriteRunSubsetEnd(writer); } // Find containing run lastRun = FindContainerAtDepth(position, _runDepth); // Write run subset _bionReader.Seek(lastRun.StartByteOffset); WriteRunSubsetStart(writer); } // Find and write result ContainerEntry result = FindContainerAtDepth(position, _runDepth + 2); if (!result.IsEmpty() && !lastResult.Equals(result)) { matchCount++; if (matchCount <= skip) { continue; } _bionReader.Seek(result.StartByteOffset); JsonBionConverter.BionToJson(_bionReader, writer); if (take >= 0 && matchCount >= skip + take) { break; } } } if (take >= 0 && matchCount >= skip + take) { break; } } // Close last run, if any if (!lastRun.IsEmpty()) { WriteRunSubsetEnd(writer); } return(matchCount); }