/// <summary> /// Recursively looks for indexes underneath <code>file</code>, /// and runs checkindex on them. returns true if it found any indexes. /// </summary> public virtual bool CheckIndexes(DirectoryInfo file) { if (file.IsDirectory) { BaseDirectoryWrapper dir = NewFSDirectory(file); dir.CheckIndexOnClose = false; // don't double-checkindex if (DirectoryReader.IndexExists(dir)) { if (VERBOSE) { Console.Error.WriteLine("Checking index: " + file); } // LUCENE-4738: if we crashed while writing first // commit it's possible index will be corrupt (by // design we don't try to be smart about this case // since that too risky): if (SegmentInfos.GetLastCommitGeneration(dir) > 1) { TestUtil.CheckIndex(dir); } dir.Dispose(); return(true); } dir.Dispose(); foreach (FileInfo f in file.ListAll()) { if (CheckIndexes(f)) { return(true); } } } return(false); }
/// <summary> /// Perform the upgrade. </summary> public void Upgrade() { if (!DirectoryReader.IndexExists(dir)) { throw new IndexNotFoundException(dir.ToString()); } if (!deletePriorCommits) { ICollection <IndexCommit> commits = DirectoryReader.ListCommits(dir); if (commits.Count > 1) { throw new System.ArgumentException("this tool was invoked to not delete prior commit points, but the following commits were found: " + commits); } } IndexWriterConfig c = (IndexWriterConfig)iwc.Clone(); c.MergePolicy = new UpgradeIndexMergePolicy(c.MergePolicy); c.IndexDeletionPolicy = new KeepOnlyLastCommitDeletionPolicy(); IndexWriter w = new IndexWriter(dir, c); try { InfoStream infoStream = c.InfoStream; if (infoStream.IsEnabled("IndexUpgrader")) { infoStream.Message("IndexUpgrader", "Upgrading all pre-" + Constants.LUCENE_MAIN_VERSION + " segments of index directory '" + dir + "' to version " + Constants.LUCENE_MAIN_VERSION + "..."); } w.ForceMerge(1); if (infoStream.IsEnabled("IndexUpgrader")) { infoStream.Message("IndexUpgrader", "All segments upgraded to version " + Constants.LUCENE_MAIN_VERSION); } } finally { w.Dispose(); } }
public virtual void Test() { MockDirectoryWrapper dir = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors")); dir.PreventDoubleWrite = false; double rate = Random.NextDouble() * 0.01; //System.out.println("rate=" + rate); dir.RandomIOExceptionRateOnOpen = rate; int iters = AtLeast(20); LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues); IndexReader r = null; DirectoryReader r2 = null; bool any = false; MockDirectoryWrapper dirCopy = null; int lastNumDocs = 0; for (int iter = 0; iter < iters; iter++) { IndexWriter w = null; if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter); } try { MockAnalyzer analyzer = new MockAnalyzer(Random); analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); if (VERBOSE) { // Do this ourselves instead of relying on LTC so // we see incrementing messageID: iwc.SetInfoStream(new TextWriterInfoStream(Console.Out)); } var ms = iwc.MergeScheduler; if (ms is IConcurrentMergeScheduler) { ((IConcurrentMergeScheduler)ms).SetSuppressExceptions(); } w = new IndexWriter(dir, iwc); if (r != null && Random.Next(5) == 3) { if (Random.NextBoolean()) { if (VERBOSE) { Console.WriteLine("TEST: addIndexes IR[]"); } w.AddIndexes(new IndexReader[] { r }); } else { if (VERBOSE) { Console.WriteLine("TEST: addIndexes Directory[]"); } w.AddIndexes(new Directory[] { dirCopy }); } } else { if (VERBOSE) { Console.WriteLine("TEST: addDocument"); } w.AddDocument(docs.NextDoc()); } dir.RandomIOExceptionRateOnOpen = 0.0; w.Dispose(); w = null; // NOTE: this is O(N^2)! Only enable for temporary debugging: //dir.setRandomIOExceptionRateOnOpen(0.0); //TestUtil.CheckIndex(dir); //dir.setRandomIOExceptionRateOnOpen(rate); // Verify numDocs only increases, to catch IndexWriter // accidentally deleting the index: dir.RandomIOExceptionRateOnOpen = 0.0; Assert.IsTrue(DirectoryReader.IndexExists(dir)); if (r2 == null) { r2 = DirectoryReader.Open(dir); } else { DirectoryReader r3 = DirectoryReader.OpenIfChanged(r2); if (r3 != null) { r2.Dispose(); r2 = r3; } } Assert.IsTrue(r2.NumDocs >= lastNumDocs, "before=" + lastNumDocs + " after=" + r2.NumDocs); lastNumDocs = r2.NumDocs; //System.out.println("numDocs=" + lastNumDocs); dir.RandomIOExceptionRateOnOpen = rate; any = true; if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter + ": success"); } } catch (IOException ioe) { if (VERBOSE) { Console.WriteLine("TEST: iter=" + iter + ": exception"); Console.WriteLine(ioe.ToString()); Console.Write(ioe.StackTrace); } if (w != null) { // NOTE: leave random IO exceptions enabled here, // to verify that rollback does not try to write // anything: w.Rollback(); } } if (any && r == null && Random.NextBoolean()) { // Make a copy of a non-empty index so we can use // it to addIndexes later: dir.RandomIOExceptionRateOnOpen = 0.0; r = DirectoryReader.Open(dir); dirCopy = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors.copy")); HashSet <string> files = new HashSet <string>(); foreach (string file in dir.ListAll()) { dir.Copy(dirCopy, file, file, IOContext.DEFAULT); files.Add(file); } dirCopy.Sync(files); // Have IW kiss the dir so we remove any leftover // files ... we can easily have leftover files at // the time we take a copy because we are holding // open a reader: (new IndexWriter(dirCopy, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)))).Dispose(); dirCopy.RandomIOExceptionRate = rate; dir.RandomIOExceptionRateOnOpen = rate; } } if (r2 != null) { r2.Dispose(); } if (r != null) { r.Dispose(); dirCopy.Dispose(); } dir.Dispose(); }
public static void Main(string[] args) { if (args.Length < 5) { Console.Error.WriteLine("Usage: MultiPassIndexSplitter -out <outputDir> -num <numParts> [-seq] <inputIndex1> [<inputIndex2 ...]"); Console.Error.WriteLine("\tinputIndex\tpath to input index, multiple values are ok"); Console.Error.WriteLine("\t-out ouputDir\tpath to output directory to contain partial indexes"); Console.Error.WriteLine("\t-num numParts\tnumber of parts to produce"); Console.Error.WriteLine("\t-seq\tsequential docid-range split (default is round-robin)"); Environment.Exit(-1); } List <IndexReader> indexes = new List <IndexReader>(); string outDir = null; int numParts = -1; bool seq = false; for (int i = 0; i < args.Length; i++) { if (args[i].Equals("-out")) { outDir = args[++i]; } else if (args[i].Equals("-num")) { numParts = Convert.ToInt32(args[++i]); } else if (args[i].Equals("-seq")) { seq = true; } else { DirectoryInfo file = new DirectoryInfo(args[i]); if (!file.Exists) { Console.Error.WriteLine("Invalid input path - skipping: " + file); continue; } Store.Directory dir = FSDirectory.Open(new DirectoryInfo(args[i])); try { if (!DirectoryReader.IndexExists(dir)) { Console.Error.WriteLine("Invalid input index - skipping: " + file); continue; } } catch (Exception) { Console.Error.WriteLine("Invalid input index - skipping: " + file); continue; } indexes.Add(DirectoryReader.Open(dir)); } } if (outDir == null) { throw new Exception("Required argument missing: -out outputDir"); } if (numParts < 2) { throw new Exception("Invalid value of required argument: -num numParts"); } if (indexes.Count == 0) { throw new Exception("No input indexes to process"); } DirectoryInfo @out = new DirectoryInfo(outDir); @out.Create(); if (!new DirectoryInfo(outDir).Exists) { throw new Exception("Can't create output directory: " + @out); } Store.Directory[] dirs = new Store.Directory[numParts]; for (int i = 0; i < numParts; i++) { dirs[i] = FSDirectory.Open(new DirectoryInfo(Path.Combine(@out.FullName, "part-" + i))); } MultiPassIndexSplitter splitter = new MultiPassIndexSplitter(); IndexReader input; if (indexes.Count == 1) { input = indexes[0]; } else { input = new MultiReader(indexes.ToArray()); } #pragma warning disable 612, 618 splitter.Split(LuceneVersion.LUCENE_CURRENT, input, dirs, seq); #pragma warning restore 612, 618 }