public long Commit()
{
if (_ixFilesToProcess.Length == 1)
{
// truncate segments
var ix = BatchInfo.Load(_ixFilesToProcess[0]);
if (Util.IsSegmented(_ixFilesToProcess[0]))
{
return(Truncate(_ixFilesToProcess[0]));
}
else
{
return(-1);
}
}
// merge branches
var branchFileName = _ixFilesToProcess[1];
var dir = Path.GetDirectoryName(branchFileName);
var dataFileName = Path.Combine(dir, Path.GetFileNameWithoutExtension(branchFileName) + ".rdb");
if (File.Exists(dataFileName))
{
return(Merge(_ixFilesToProcess[1]));
}
return(long.Parse(Path.GetFileNameWithoutExtension(_ixFilesToProcess[0])));
}
private long Merge(string srcIxFileName)
{
Log.InfoFormat("merging branch {0} with trunk {1}", _ixFilesToProcess[1], _ixFilesToProcess[0]);
var ix = BatchInfo.Load(srcIxFileName);
var dataFileName = Path.Combine(_directory, ix.VersionId + ".rdb");
long version;
using (var stream = new FileStream(dataFileName, FileMode.Open))
using (var documentStream = new DtblStream(stream, ix))
{
// TODO: instead of rewriting, copy the segments from the branch file into the main data file.
using (var upsert = new UpsertTransaction(
_directory,
_analyzer,
ix.Compression,
documentStream))
{
version = upsert.Write();
upsert.Commit();
}
Log.InfoFormat("{0} merged with {1} creating a segmented index", srcIxFileName, _ixFilesToProcess[0]);
}
Util.RemoveAll(srcIxFileName);
return(version);
}
private long Truncate(string srcIxFileName)
{
Log.InfoFormat("truncating {0}", srcIxFileName);
var srcIx = BatchInfo.Load(srcIxFileName);
var dataFileName = Path.Combine(_directory, srcIx.VersionId + ".rdb");
long version;
using (var stream = new FileStream(dataFileName, FileMode.Open))
using (var documentStream = new DtblStream(stream, srcIx))
{
using (var upsert = new UpsertTransaction(
_directory,
_analyzer,
srcIx.Compression,
documentStream))
{
version = upsert.Write();
upsert.Commit();
}
Log.InfoFormat("ix {0} fully truncated", _ixFilesToProcess[0]);
}
Util.RemoveAll(srcIxFileName);
return(version);
}
public Searcher(string directory, long version, QueryParser parser, IScoringSchemeFactory scorerFactory, IReadSessionFactory sessionFactory = null)
{
_directory = directory;
_parser = parser;
_scorerFactory = scorerFactory;
_versions = new[] { BatchInfo.Load(Path.Combine(directory, version + ".ix")) };
_blockSize = BlockSerializer.SizeOfBlock();
_sessionFactory = sessionFactory ?? new ReadSessionFactory(directory);
}
public Searcher(string directory, QueryParser parser, IScoringSchemeFactory scorerFactory, IReadSessionFactory sessionFactory = null)
{
_directory = directory;
_parser = parser;
_scorerFactory = scorerFactory;
_versions = Util.GetIndexFileNamesInChronologicalOrder(directory)
.Select(f => BatchInfo.Load(f)).ToArray();
_blockSize = BlockSerializer.SizeOfBlock();
_sessionFactory = sessionFactory ?? new ReadSessionFactory(directory);
}
static void Rewrite(string[] args)
{
var take = int.MaxValue;
var skip = 0;
string pk = null;
bool gzip = false;
bool lz = false;
string dir = null;
if (Array.IndexOf(args, "--take") > 0)
{
take = int.Parse(args[Array.IndexOf(args, "--take") + 1]);
}
if (Array.IndexOf(args, "--skip") > 0)
{
skip = int.Parse(args[Array.IndexOf(args, "--skip") + 1]);
}
if (Array.IndexOf(args, "--pk") > 0)
{
pk = args[Array.IndexOf(args, "--pk") + 1];
}
if (Array.IndexOf(args, "--gzip") > 0)
{
gzip = true;
}
if (Array.IndexOf(args, "--lz") > 0)
{
lz = true;
}
if (Array.IndexOf(args, "--dir") > 0)
{
dir = args[Array.IndexOf(args, "--dir") + 1];
}
var compression = gzip ? Compression.GZip : lz ? Compression.Lz : Compression.NoCompression;
var dataFileName = args[Array.IndexOf(args, "--file") + 1];
var ixFileName = Directory.GetFiles(Path.GetDirectoryName(dataFileName), "*.ix")
.OrderBy(s => s).First();
var ix = BatchInfo.Load(ixFileName);
Console.WriteLine("rewriting...");
var writeTimer = new Stopwatch();
writeTimer.Start();
using (var stream = new FileStream(dataFileName, FileMode.Open))
using (var documents = new DtblStream(stream, ix, skip, take))
using (var upsert = new UpsertTransaction(dir, new Analyzer(), compression, documents))
{
upsert.Write();
}
Console.WriteLine("write operation took {0}", writeTimer.Elapsed);
}
static void Export(string[] args)
{
var take = int.MaxValue;
var skip = 0;
if (Array.IndexOf(args, "--take") > 0)
{
take = int.Parse(args[Array.IndexOf(args, "--take") + 1]);
}
if (Array.IndexOf(args, "--skip") > 0)
{
skip = int.Parse(args[Array.IndexOf(args, "--skip") + 1]);
}
var sourceFileName = args[Array.IndexOf(args, "--source-file") + 1];
var targetFileName = args[Array.IndexOf(args, "--target-file") + 1];
var dir = Path.GetDirectoryName(sourceFileName);
var version = Path.GetFileNameWithoutExtension(sourceFileName);
var ix = BatchInfo.Load(Path.Combine(dir, version + ".ix"));
Console.WriteLine("migrating...");
var writeTimer = new Stopwatch();
writeTimer.Start();
using (var sourceStream = new FileStream(sourceFileName, FileMode.Open))
using (var targetStream = new FileStream(targetFileName, FileMode.Create))
using (var jsonWriter = new StreamWriter(targetStream, Encoding.UTF8))
using (var documents = new DtblStream(sourceStream, ix, skip, take))
{
jsonWriter.WriteLine("[");
foreach (var document in documents.ReadSource())
{
var dic = document.Fields.ToDictionary(x => x.Key, y => y.Value.Value);
var json = JsonConvert.SerializeObject(dic, Formatting.None);
jsonWriter.WriteLine(json);
}
jsonWriter.Write("]");
}
Console.WriteLine("write operation took {0}", writeTimer.Elapsed);
}
