private static void CompareUncompressed(string jsonFilePath)
{
string bionFilePath = Path.ChangeExtension(jsonFilePath, ".bion");
string comparePath = Path.ChangeExtension(jsonFilePath, "compare.json");
JsonBionConverter.JsonToBion(jsonFilePath, bionFilePath);
JsonBionConverter.BionToJson(bionFilePath, comparePath);
Verify.JsonEqual(jsonFilePath, comparePath);
}
private static void ToBion(string jsonPath, string bionPath, string dictionaryPath)
{
VerifyFileExists(jsonPath);
using (new ConsoleWatch($"Converting {jsonPath} to {bionPath}...",
() => $"Done. {FileLength.MB(jsonPath)} JSON to {FileLength.MB(bionPath)} BION{(String.IsNullOrEmpty(dictionaryPath) ? "" : $" + {FileLength.MB(dictionaryPath)} dictionary")} ({FileLength.Percentage(jsonPath, bionPath, dictionaryPath)})"))
{
JsonBionConverter.JsonToBion(jsonPath, bionPath, dictionaryPath);
}
}
private static void ToJson(string fromPath, string toPath, string dictionaryPath)
{
VerifyFileExists(fromPath);
VerifyFileExists(dictionaryPath);
using (new ConsoleWatch($"Converting {fromPath} to {toPath}...",
() => $"Done. {FileLength.MB(fromPath)} BION to {FileLength.MB(toPath)} JSON"))
{
JsonBionConverter.BionToJson(fromPath, toPath, dictionaryPath);
}
}
private static void Index(string jsonRootPath, string bionRootPath)
{
using (new ConsoleWatch($"Converting under \"{jsonRootPath}\" to \"{bionRootPath}\"..."))
{
jsonRootPath = Path.GetFullPath(jsonRootPath);
foreach (string jsonFilePath in Directory.EnumerateFiles(jsonRootPath, "*.*", SearchOption.AllDirectories))
{
string jsonFilePathUnderRoot = jsonFilePath.Substring(jsonRootPath.Length + 1);
string outputPath = Path.ChangeExtension(Path.Combine(bionRootPath, jsonFilePathUnderRoot), ".bion");
if (!File.Exists(outputPath))
{
System.Console.WriteLine($" {jsonFilePathUnderRoot}");
Directory.CreateDirectory(Path.GetDirectoryName(outputPath));
JsonBionConverter.JsonToBion(jsonFilePath, outputPath, Path.ChangeExtension(outputPath, ".wdx"));
}
}
}
}
public void ContainerIndex_EndToEnd()
{
string jsonFilePath = @"Content\Medium.json";
string bionFilePath = Path.ChangeExtension(jsonFilePath, ".bion");
string dictionaryPath = Path.ChangeExtension(bionFilePath, "dict.bion");
string comparePath = Path.ChangeExtension(jsonFilePath, "compare.json");
JsonBionConverter.JsonToBion(jsonFilePath, bionFilePath, dictionaryPath);
using (WordCompressor compressor = WordCompressor.OpenRead(dictionaryPath))
using (ContainerIndex cIndex = ContainerIndex.OpenRead(Path.ChangeExtension(bionFilePath, ".cdx")))
using (BionReader reader = new BionReader(File.OpenRead(bionFilePath), cIndex, compressor))
{
for (int i = 0; i < cIndex.Count; ++i)
{
ContainerEntry container = cIndex[i];
// Seek to container start
reader.Seek(container.StartByteOffset);
// Verify a container start is there
int depth = reader.Depth;
reader.Read();
bool isObject = (reader.TokenType == BionToken.StartObject);
Assert.AreEqual((isObject ? BionToken.StartObject : BionToken.StartArray), reader.TokenType);
// Read until the depth is back to the same value
while (reader.Depth != depth)
{
reader.Read();
}
// Verify this is the end container position
Assert.AreEqual((isObject ? BionToken.EndObject : BionToken.EndArray), reader.TokenType);
Assert.AreEqual(container.EndByteOffset, reader.BytesRead);
}
}
}
private void WriteRunSubsetStart(JsonTextWriter writer)
{
int depth = _bionReader.Depth;
// Read and copy everything in the run until the end object except the large collections
while (_bionReader.Read() && _bionReader.Depth > depth)
{
if (_bionReader.TokenType == BionToken.PropertyName)
{
String8 propertyName = _bionReader.CurrentString8();
if (propertyName.Equals(Files) || propertyName.Equals(Results))
{
_bionReader.Skip();
continue;
}
}
JsonBionConverter.WriteToken(_bionReader, writer);
}
// Start a results array and leave open
writer.WritePropertyName("results");
writer.WriteStartArray();
}
public int Write(JsonTextWriter writer, ISearchResult wordMatches, int skip = 0, int take = -1)
{
int matchCount = 0;
ContainerEntry lastRun = ContainerEntry.Empty;
ContainerEntry lastResult = ContainerEntry.Empty;
while (!wordMatches.Done)
{
int count = wordMatches.Page(ref _termPositions);
for (int i = 0; i < count; ++i)
{
long position = _termPositions[i];
// If this position isn't in the last run's results array, find the run which contains it
if (!lastRun.Contains(position))
{
// Close previous run, if any
if (!lastRun.IsEmpty())
{
WriteRunSubsetEnd(writer);
}
// Find containing run
lastRun = FindContainerAtDepth(position, _runDepth);
// Write run subset
_bionReader.Seek(lastRun.StartByteOffset);
WriteRunSubsetStart(writer);
}
// Find and write result
ContainerEntry result = FindContainerAtDepth(position, _runDepth + 2);
if (!result.IsEmpty() && !lastResult.Equals(result))
{
matchCount++;
if (matchCount <= skip)
{
continue;
}
_bionReader.Seek(result.StartByteOffset);
JsonBionConverter.BionToJson(_bionReader, writer);
if (take >= 0 && matchCount >= skip + take)
{
break;
}
}
}
if (take >= 0 && matchCount >= skip + take)
{
break;
}
}
// Close last run, if any
if (!lastRun.IsEmpty())
{
WriteRunSubsetEnd(writer);
}
return(matchCount);
}
