public long Commit() { if (_ixFilesToProcess.Length == 1) { // truncate segments var ix = BatchInfo.Load(_ixFilesToProcess[0]); if (Util.IsSegmented(_ixFilesToProcess[0])) { return(Truncate(_ixFilesToProcess[0])); } else { return(-1); } } // merge branches var branchFileName = _ixFilesToProcess[1]; var dir = Path.GetDirectoryName(branchFileName); var dataFileName = Path.Combine(dir, Path.GetFileNameWithoutExtension(branchFileName) + ".rdb"); if (File.Exists(dataFileName)) { return(Merge(_ixFilesToProcess[1])); } return(long.Parse(Path.GetFileNameWithoutExtension(_ixFilesToProcess[0]))); }
private long Merge(string srcIxFileName) { Log.InfoFormat("merging branch {0} with trunk {1}", _ixFilesToProcess[1], _ixFilesToProcess[0]); var ix = BatchInfo.Load(srcIxFileName); var dataFileName = Path.Combine(_directory, ix.VersionId + ".rdb"); long version; using (var stream = new FileStream(dataFileName, FileMode.Open)) using (var documentStream = new DtblStream(stream, ix)) { // TODO: instead of rewriting, copy the segments from the branch file into the main data file. using (var upsert = new UpsertTransaction( _directory, _analyzer, ix.Compression, documentStream)) { version = upsert.Write(); upsert.Commit(); } Log.InfoFormat("{0} merged with {1} creating a segmented index", srcIxFileName, _ixFilesToProcess[0]); } Util.RemoveAll(srcIxFileName); return(version); }
private long Truncate(string srcIxFileName) { Log.InfoFormat("truncating {0}", srcIxFileName); var srcIx = BatchInfo.Load(srcIxFileName); var dataFileName = Path.Combine(_directory, srcIx.VersionId + ".rdb"); long version; using (var stream = new FileStream(dataFileName, FileMode.Open)) using (var documentStream = new DtblStream(stream, srcIx)) { using (var upsert = new UpsertTransaction( _directory, _analyzer, srcIx.Compression, documentStream)) { version = upsert.Write(); upsert.Commit(); } Log.InfoFormat("ix {0} fully truncated", _ixFilesToProcess[0]); } Util.RemoveAll(srcIxFileName); return(version); }
public Searcher(string directory, long version, QueryParser parser, IScoringSchemeFactory scorerFactory, IReadSessionFactory sessionFactory = null) { _directory = directory; _parser = parser; _scorerFactory = scorerFactory; _versions = new[] { BatchInfo.Load(Path.Combine(directory, version + ".ix")) }; _blockSize = BlockSerializer.SizeOfBlock(); _sessionFactory = sessionFactory ?? new ReadSessionFactory(directory); }
public Searcher(string directory, QueryParser parser, IScoringSchemeFactory scorerFactory, IReadSessionFactory sessionFactory = null) { _directory = directory; _parser = parser; _scorerFactory = scorerFactory; _versions = Util.GetIndexFileNamesInChronologicalOrder(directory) .Select(f => BatchInfo.Load(f)).ToArray(); _blockSize = BlockSerializer.SizeOfBlock(); _sessionFactory = sessionFactory ?? new ReadSessionFactory(directory); }
static void Rewrite(string[] args) { var take = int.MaxValue; var skip = 0; string pk = null; bool gzip = false; bool lz = false; string dir = null; if (Array.IndexOf(args, "--take") > 0) { take = int.Parse(args[Array.IndexOf(args, "--take") + 1]); } if (Array.IndexOf(args, "--skip") > 0) { skip = int.Parse(args[Array.IndexOf(args, "--skip") + 1]); } if (Array.IndexOf(args, "--pk") > 0) { pk = args[Array.IndexOf(args, "--pk") + 1]; } if (Array.IndexOf(args, "--gzip") > 0) { gzip = true; } if (Array.IndexOf(args, "--lz") > 0) { lz = true; } if (Array.IndexOf(args, "--dir") > 0) { dir = args[Array.IndexOf(args, "--dir") + 1]; } var compression = gzip ? Compression.GZip : lz ? Compression.Lz : Compression.NoCompression; var dataFileName = args[Array.IndexOf(args, "--file") + 1]; var ixFileName = Directory.GetFiles(Path.GetDirectoryName(dataFileName), "*.ix") .OrderBy(s => s).First(); var ix = BatchInfo.Load(ixFileName); Console.WriteLine("rewriting..."); var writeTimer = new Stopwatch(); writeTimer.Start(); using (var stream = new FileStream(dataFileName, FileMode.Open)) using (var documents = new DtblStream(stream, ix, skip, take)) using (var upsert = new UpsertTransaction(dir, new Analyzer(), compression, documents)) { upsert.Write(); } Console.WriteLine("write operation took {0}", writeTimer.Elapsed); }
static void Export(string[] args) { var take = int.MaxValue; var skip = 0; if (Array.IndexOf(args, "--take") > 0) { take = int.Parse(args[Array.IndexOf(args, "--take") + 1]); } if (Array.IndexOf(args, "--skip") > 0) { skip = int.Parse(args[Array.IndexOf(args, "--skip") + 1]); } var sourceFileName = args[Array.IndexOf(args, "--source-file") + 1]; var targetFileName = args[Array.IndexOf(args, "--target-file") + 1]; var dir = Path.GetDirectoryName(sourceFileName); var version = Path.GetFileNameWithoutExtension(sourceFileName); var ix = BatchInfo.Load(Path.Combine(dir, version + ".ix")); Console.WriteLine("migrating..."); var writeTimer = new Stopwatch(); writeTimer.Start(); using (var sourceStream = new FileStream(sourceFileName, FileMode.Open)) using (var targetStream = new FileStream(targetFileName, FileMode.Create)) using (var jsonWriter = new StreamWriter(targetStream, Encoding.UTF8)) using (var documents = new DtblStream(sourceStream, ix, skip, take)) { jsonWriter.WriteLine("["); foreach (var document in documents.ReadSource()) { var dic = document.Fields.ToDictionary(x => x.Key, y => y.Value.Value); var json = JsonConvert.SerializeObject(dic, Formatting.None); jsonWriter.WriteLine(json); } jsonWriter.Write("]"); } Console.WriteLine("write operation took {0}", writeTimer.Elapsed); }