private static void buildIndex(bool buildVerbose) { try { string[] si = MyIndex.GetAllIndexDirectory(); Stopwatch sw = new Stopwatch(); CleanIndexIfApplicable(); SearchAutoComplete searchIndex = null; bool flagFirstTime = true; if (si != null && si.Length > 0) { foreach (string directory in si) { if (directory != null) { searchIndex = new SearchAutoComplete(directory); searchIndex.IsFirstTime = flagFirstTime; System.IO.DirectoryInfo di = new System.IO.DirectoryInfo(directory); if (di.Exists) { log.Info(string.Format("Index in directory {0} is building now...", di.Name)); FSDirectory fsd = FSDirectory.Open(di); Lucene.Net.Index.CheckIndex ci = new CheckIndex(fsd); Lucene.Net.Index.CheckIndex.Status st = ci.CheckIndex_Renamed_Method(); if (st.clean) { sw.Start(); searchIndex.BuildAutoCompleteIndex(fsd, MyIndex.GetAutoUpdateIndexDirectory(), buildVerbose); sw.Stop(); TimeSpan ts = sw.Elapsed; string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); log.Info(string.Format("Time taken {0} to build this index {1}", elapsedTime, di.Name)); flagFirstTime = false; } else { log.Info(string.Format("Following Index named {0} is corrupted, please rebuild that index using Sitecore Rebuild Index Wizard", di.Name)); } } } } } } catch (Exception ex) { log.Error("Error Occurred during buildIndex().", ex); } }
public virtual void TestUnsupportedOldIndexes() { for (int i = 0; i < UnsupportedNames.Length; i++) { if (VERBOSE) { Console.WriteLine("TEST: index " + UnsupportedNames[i]); } DirectoryInfo oldIndxeDir = CreateTempDir(UnsupportedNames[i]); using (Stream dataFile = this.GetType().Assembly.GetManifestResourceStream(CURRENT_RESOURCE_DIRECTORY + "unsupported." + UnsupportedNames[i] + ".zip")) { TestUtil.Unzip(dataFile, oldIndxeDir); } BaseDirectoryWrapper dir = NewFSDirectory(oldIndxeDir); // don't checkindex, these are intentionally not supported dir.CheckIndexOnClose = false; IndexReader reader = null; IndexWriter writer = null; try { reader = DirectoryReader.Open(dir); Assert.Fail("DirectoryReader.open should not pass for " + UnsupportedNames[i]); } catch (IndexFormatTooOldException e) { // pass } finally { if (reader != null) { reader.Dispose(); } reader = null; } try { writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); Assert.Fail("IndexWriter creation should not pass for " + UnsupportedNames[i]); } catch (IndexFormatTooOldException e) { // pass if (VERBOSE) { Console.WriteLine("TEST: got expected exc:"); Console.WriteLine(e.StackTrace); } // Make sure exc message includes a path= Assert.IsTrue(e.Message.IndexOf("path=\"") != -1, "got exc message: " + e.Message); } finally { // we should fail to open IW, and so it should be null when we get here. // However, if the test fails (i.e., IW did not fail on open), we need // to close IW. However, if merges are run, IW may throw // IndexFormatTooOldException, and we don't want to mask the Assert.Fail() // above, so close without waiting for merges. if (writer != null) { writer.Dispose(false); } writer = null; } StringBuilder sb = new StringBuilder(1024); CheckIndex checker = new CheckIndex(dir); CheckIndex.Status indexStatus; using (var infoStream = new StringWriter(sb)) { checker.InfoStream = infoStream; indexStatus = checker.DoCheckIndex(); } Assert.IsFalse(indexStatus.Clean); Assert.IsTrue(sb.ToString().Contains(typeof(IndexFormatTooOldException).Name)); dir.Dispose(); TestUtil.Rm(oldIndxeDir); } }
public virtual void TestDeletedDocs() { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(2); Document doc = new Document(); doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); for (int i = 0; i < 19; i++) { writer.AddDocument(doc); } writer.Close(); IndexReader reader = IndexReader.Open(dir, false); reader.DeleteDocument(5); reader.Close(); System.IO.MemoryStream bos = new System.IO.MemoryStream(1024); CheckIndex checker = new CheckIndex(dir); checker.SetInfoStream(new System.IO.StreamWriter(bos)); //checker.setInfoStream(System.out); CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method(); if (indexStatus.clean == false) { System.Console.Out.WriteLine("CheckIndex failed"); char[] tmpChar; byte[] tmpByte; tmpByte = bos.GetBuffer(); tmpChar = new char[bos.Length]; System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length); System.Console.Out.WriteLine(new System.String(tmpChar)); Assert.Fail(); } CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus) indexStatus.segmentInfos[0]; Assert.IsTrue(seg.openReaderPassed); Assert.IsNotNull(seg.diagnostics); Assert.IsNotNull(seg.fieldNormStatus); Assert.IsNull(seg.fieldNormStatus.error); Assert.AreEqual(1, seg.fieldNormStatus.totFields); Assert.IsNotNull(seg.termIndexStatus); Assert.IsNull(seg.termIndexStatus.error); Assert.AreEqual(1, seg.termIndexStatus.termCount); Assert.AreEqual(19, seg.termIndexStatus.totFreq); Assert.AreEqual(18, seg.termIndexStatus.totPos); Assert.IsNotNull(seg.storedFieldStatus); Assert.IsNull(seg.storedFieldStatus.error); Assert.AreEqual(18, seg.storedFieldStatus.docCount); Assert.AreEqual(18, seg.storedFieldStatus.totFields); Assert.IsNotNull(seg.termVectorStatus); Assert.IsNull(seg.termVectorStatus.error); Assert.AreEqual(18, seg.termVectorStatus.docCount); Assert.AreEqual(18, seg.termVectorStatus.totVectors); Assert.IsTrue(seg.diagnostics.Count > 0); List<string> onlySegments = new List<string>(); onlySegments.Add("_0"); Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true); }
public virtual void TestDeletedDocs() { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(2); Document doc = new Document(); doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); for (int i = 0; i < 19; i++) { writer.AddDocument(doc); } writer.Close(); IndexReader reader = IndexReader.Open(dir); reader.DeleteDocument(5); reader.Close(); System.IO.MemoryStream bos = new System.IO.MemoryStream(1024); CheckIndex checker = new CheckIndex(dir); checker.SetInfoStream(new System.IO.StreamWriter(bos)); //checker.setInfoStream(System.out); CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method(); if (indexStatus.clean == false) { System.Console.Out.WriteLine("CheckIndex failed"); char[] tmpChar; byte[] tmpByte; tmpByte = bos.GetBuffer(); tmpChar = new char[bos.Length]; System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length); System.Console.Out.WriteLine(new System.String(tmpChar)); Assert.Fail(); } CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus)indexStatus.segmentInfos[0]; Assert.IsTrue(seg.openReaderPassed); Assert.IsNotNull(seg.diagnostics); Assert.IsNotNull(seg.fieldNormStatus); Assert.IsNull(seg.fieldNormStatus.error); Assert.AreEqual(1, seg.fieldNormStatus.totFields); Assert.IsNotNull(seg.termIndexStatus); Assert.IsNull(seg.termIndexStatus.error); Assert.AreEqual(1, seg.termIndexStatus.termCount); Assert.AreEqual(19, seg.termIndexStatus.totFreq); Assert.AreEqual(18, seg.termIndexStatus.totPos); Assert.IsNotNull(seg.storedFieldStatus); Assert.IsNull(seg.storedFieldStatus.error); Assert.AreEqual(18, seg.storedFieldStatus.docCount); Assert.AreEqual(18, seg.storedFieldStatus.totFields); Assert.IsNotNull(seg.termVectorStatus); Assert.IsNull(seg.termVectorStatus.error); Assert.AreEqual(18, seg.termVectorStatus.docCount); Assert.AreEqual(18, seg.termVectorStatus.totVectors); Assert.IsTrue(seg.diagnostics.Count > 0); System.Collections.IList onlySegments = new System.Collections.ArrayList(); onlySegments.Add("_0"); Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true); }
private static void CheckIndexAndTryToFix(Lucene.Net.Store.Directory directory, IndexDefinition indexDefinition) { startupLog.Warn("Unclean shutdown detected on {0}, checking the index for errors. This may take a while.", indexDefinition.Name); var memoryStream = new MemoryStream(); var stringWriter = new StreamWriter(memoryStream); var checkIndex = new CheckIndex(directory); if (startupLog.IsWarnEnabled) checkIndex.SetInfoStream(stringWriter); var sp = Stopwatch.StartNew(); var status = checkIndex.CheckIndex_Renamed_Method(); sp.Stop(); if (startupLog.IsWarnEnabled) { startupLog.Warn("Checking index {0} took: {1}, clean: {2}", indexDefinition.Name, sp.Elapsed, status.clean); memoryStream.Position = 0; log.Warn(new StreamReader(memoryStream).ReadToEnd()); } if (status.clean) return; startupLog.Warn("Attempting to fix index: {0}", indexDefinition.Name); sp.Restart(); checkIndex.FixIndex(status); startupLog.Warn("Fixed index {0} in {1}", indexDefinition.Name, sp.Elapsed); }
public static void Main(System.String[] args) { bool doFix = false; var onlySegments = new List<string>(); System.String indexPath = null; int i = 0; while (i < args.Length) { if (args[i].Equals("-fix")) { doFix = true; i++; } else if (args[i].Equals("-segment")) { if (i == args.Length - 1) { System.Console.Out.WriteLine("ERROR: missing name for -segment option"); System.Environment.Exit(1); } onlySegments.Add(args[i + 1]); i += 2; } else { if (indexPath != null) { System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'"); System.Environment.Exit(1); } indexPath = args[i]; i++; } } if (indexPath == null) { System.Console.Out.WriteLine("\nERROR: index path not specified"); System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n"); System.Environment.Exit(1); } if (!AssertsOn()) System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled"); if (onlySegments.Count == 0) onlySegments = null; else if (doFix) { System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment"); System.Environment.Exit(1); } System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n"); Directory dir = null; try { dir = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath)); } catch (Exception t) { Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting"); Console.Out.WriteLine(t.StackTrace); Environment.Exit(1); } var checker = new CheckIndex(dir); var tempWriter = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding) {AutoFlush = true}; checker.SetInfoStream(tempWriter); Status result = checker.CheckIndex_Renamed_Method(onlySegments); if (result.missingSegments) { System.Environment.Exit(1); } if (!result.clean) { if (!doFix) { System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n"); } else { Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n"); Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!"); for (var s = 0; s < 5; s++) { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000)); System.Console.Out.WriteLine(" " + (5 - s) + "..."); } Console.Out.WriteLine("Writing..."); checker.FixIndex(result); Console.Out.WriteLine("OK"); Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\""); } } System.Console.Out.WriteLine(""); int exitCode; if (result != null && result.clean == true) exitCode = 0; else exitCode = 1; System.Environment.Exit(exitCode); }
private void CheckIndexAndTryToFix(LuceneDirectory directory) { StartupLog.Warn(string.Format("Unclean shutdown detected on file system '{0}', checking the index for errors. This may take a while.", name)); var memoryStream = new MemoryStream(); var stringWriter = new StreamWriter(memoryStream); var checkIndex = new CheckIndex(directory); if (StartupLog.IsWarnEnabled) checkIndex.SetInfoStream(stringWriter); var sp = Stopwatch.StartNew(); var status = checkIndex.CheckIndex_Renamed_Method(); sp.Stop(); if (StartupLog.IsWarnEnabled) { StartupLog.Warn("Checking index for file system '{0}' took: {1}, clean: {2}", name, sp.Elapsed, status.clean); memoryStream.Position = 0; Log.Warn(new StreamReader(memoryStream).ReadToEnd()); } if (status.clean) return; StartupLog.Warn("Attempting to fix index of file system: '{0}'", name); sp.Restart(); checkIndex.FixIndex(status); StartupLog.Warn("Fixed index of file system '{0}' in {1}", name, sp.Elapsed); }
private bool IsIndexStateValid(LuceneDirectory luceneDirectory) { // 1. If commitData is null it means that there were no commits, so just in case we are resetting to Etag.Empty // 2. If no 'LastEtag' in commitData then we consider it an invalid index // 3. If 'LastEtag' is present (and valid), then resetting to it (if it is lower than lastStoredEtag) var commitData = IndexReader.GetCommitUserData(luceneDirectory); string value; Etag lastEtag = null; if (commitData != null && commitData.TryGetValue("LastEtag", out value)) Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail var lastStoredEtag = GetLastEtagForIndex() ?? Etag.Empty; lastEtag = lastEtag ?? Etag.Empty; if (EtagUtil.IsGreaterThan(lastEtag, lastStoredEtag)) return false; var checkIndex = new CheckIndex(luceneDirectory); var status = checkIndex.CheckIndex_Renamed_Method(); return status.clean; }
public virtual void TestUnsupportedOldIndexes() { for (int i = 0; i < unsupportedNames.Length; i++) { if (Verbose) { Console.WriteLine("TEST: index " + unsupportedNames[i]); } DirectoryInfo oldIndexDir = CreateTempDir(unsupportedNames[i]); using (Stream dataFile = this.GetType().FindAndGetManifestResourceStream("unsupported." + unsupportedNames[i] + ".zip")) { TestUtil.Unzip(dataFile, oldIndexDir); } BaseDirectoryWrapper dir = NewFSDirectory(oldIndexDir); // don't checkindex, these are intentionally not supported dir.CheckIndexOnDispose = false; IndexReader reader = null; IndexWriter writer = null; try { reader = DirectoryReader.Open(dir); Assert.Fail("DirectoryReader.open should not pass for " + unsupportedNames[i]); } #pragma warning disable 168 catch (IndexFormatTooOldException e) #pragma warning restore 168 { // pass } finally { if (reader != null) { reader.Dispose(); } reader = null; } try { writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random))); Assert.Fail("IndexWriter creation should not pass for " + unsupportedNames[i]); } catch (IndexFormatTooOldException e) { // pass if (Verbose) { Console.WriteLine("TEST: got expected exc:"); Console.WriteLine(e.StackTrace); } // Make sure exc message includes a path= Assert.IsTrue(e.Message.IndexOf("path=\"", StringComparison.Ordinal) != -1, "got exc message: " + e.Message); } finally { // we should fail to open IW, and so it should be null when we get here. // However, if the test fails (i.e., IW did not fail on open), we need // to close IW. However, if merges are run, IW may throw // IndexFormatTooOldException, and we don't want to mask the Assert.Fail() // above, so close without waiting for merges. if (writer != null) { writer.Dispose(false); } writer = null; } StringBuilder bos = new StringBuilder(); CheckIndex checker = new CheckIndex(dir); checker.InfoStream = new StringWriter(bos); CheckIndex.Status indexStatus = checker.DoCheckIndex(); Assert.IsFalse(indexStatus.Clean); checker.InfoStream.Flush(); Assert.IsTrue(bos.ToString().Contains(typeof(IndexFormatTooOldException).Name)); dir.Dispose(); } }
public virtual void TestDeletedDocs() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2)); for (int i = 0; i < 19; i++) { Document doc = new Document(); FieldType customType = new FieldType(TextField.TYPE_STORED); customType.StoreTermVectors = true; customType.StoreTermVectorPositions = true; customType.StoreTermVectorOffsets = true; doc.Add(NewField("field", "aaa" + i, customType)); writer.AddDocument(doc); } writer.ForceMerge(1); writer.Commit(); writer.DeleteDocuments(new Term("field", "aaa5")); writer.Dispose(); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); CheckIndex checker = new CheckIndex(dir); checker.InfoStream = new StreamWriter(bos, Encoding.UTF8); if (VERBOSE) { checker.InfoStream = Console.Out; } CheckIndex.Status indexStatus = checker.DoCheckIndex(); if (indexStatus.Clean == false) { Console.WriteLine("CheckIndex failed"); checker.FlushInfoStream(); Console.WriteLine(bos.ToString()); Assert.Fail(); } CheckIndex.Status.SegmentInfoStatus seg = indexStatus.SegmentInfos[0]; Assert.IsTrue(seg.OpenReaderPassed); Assert.IsNotNull(seg.Diagnostics); Assert.IsNotNull(seg.FieldNormStatus); Assert.IsNull(seg.FieldNormStatus.Error); Assert.AreEqual(1, seg.FieldNormStatus.TotFields); Assert.IsNotNull(seg.TermIndexStatus); Assert.IsNull(seg.TermIndexStatus.Error); Assert.AreEqual(18, seg.TermIndexStatus.TermCount); Assert.AreEqual(18, seg.TermIndexStatus.TotFreq); Assert.AreEqual(18, seg.TermIndexStatus.TotPos); Assert.IsNotNull(seg.StoredFieldStatus); Assert.IsNull(seg.StoredFieldStatus.Error); Assert.AreEqual(18, seg.StoredFieldStatus.DocCount); Assert.AreEqual(18, seg.StoredFieldStatus.TotFields); Assert.IsNotNull(seg.TermVectorStatus); Assert.IsNull(seg.TermVectorStatus.Error); Assert.AreEqual(18, seg.TermVectorStatus.DocCount); Assert.AreEqual(18, seg.TermVectorStatus.TotVectors); Assert.IsTrue(seg.Diagnostics.Count > 0); IList <string> onlySegments = new List <string>(); onlySegments.Add("_0"); Assert.IsTrue(checker.DoCheckIndex(onlySegments).Clean == true); dir.Dispose(); }
public static void Main(System.String[] args) { bool doFix = false; System.Collections.IList onlySegments = new System.Collections.ArrayList(); System.String indexPath = null; int i = 0; while (i < args.Length) { if (args[i].Equals("-fix")) { doFix = true; i++; } else if (args[i].Equals("-segment")) { if (i == args.Length - 1) { System.Console.Out.WriteLine("ERROR: missing name for -segment option"); System.Environment.Exit(1); } onlySegments.Add(args[i + 1]); i += 2; } else { if (indexPath != null) { System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'"); System.Environment.Exit(1); } indexPath = args[i]; i++; } } if (indexPath == null) { System.Console.Out.WriteLine("\nERROR: index path not specified"); System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n"); System.Environment.Exit(1); } if (!AssertsOn()) { System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled"); } if (onlySegments.Count == 0) { onlySegments = null; } else if (doFix) { System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment"); System.Environment.Exit(1); } System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n"); Directory dir = null; try { dir = FSDirectory.Open(new System.IO.FileInfo(indexPath)); } catch (System.Exception t) { System.Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting"); System.Console.Out.WriteLine(t.StackTrace); System.Environment.Exit(1); } CheckIndex checker = new CheckIndex(dir); System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; checker.SetInfoStream(temp_writer); Status result = checker.CheckIndex_Renamed_Method(onlySegments); if (result.missingSegments) { System.Environment.Exit(1); } if (!result.clean) { if (!doFix) { System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n"); } else { System.Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n"); System.Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!"); for (int s = 0; s < 5; s++) { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000)); System.Console.Out.WriteLine(" " + (5 - s) + "..."); } System.Console.Out.WriteLine("Writing..."); checker.FixIndex(result); System.Console.Out.WriteLine("OK"); System.Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\""); } } System.Console.Out.WriteLine(""); int exitCode; if (result != null && result.clean == true) { exitCode = 0; } else { exitCode = 1; } System.Environment.Exit(exitCode); }
public virtual void TestDeletesCheckIndexOutput() { Directory dir = NewDirectory(); IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); iwc.SetMaxBufferedDocs(2); IndexWriter w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone()); Document doc = new Document(); doc.Add(NewField("field", "0", StringField.TYPE_NOT_STORED)); w.AddDocument(doc); doc = new Document(); doc.Add(NewField("field", "1", StringField.TYPE_NOT_STORED)); w.AddDocument(doc); w.Commit(); Assert.AreEqual(1, w.SegmentCount); w.DeleteDocuments(new Term("field", "0")); w.Commit(); Assert.AreEqual(1, w.SegmentCount); w.Dispose(); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); //MemoryStream bos = new MemoryStream(1024); CheckIndex checker = new CheckIndex(dir); checker.InfoStream = new StreamWriter(bos, Encoding.UTF8); CheckIndex.Status indexStatus = checker.DoCheckIndex(null); Assert.IsTrue(indexStatus.Clean); checker.FlushInfoStream(); string s = bos.ToString(); // Segment should have deletions: Assert.IsTrue(s.Contains("has deletions"), "string was: " + s); w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone()); w.ForceMerge(1); w.Dispose(); bos = new ByteArrayOutputStream(1024); checker.InfoStream = new StreamWriter(bos, Encoding.UTF8); indexStatus = checker.DoCheckIndex(null); Assert.IsTrue(indexStatus.Clean); checker.FlushInfoStream(); s = bos.ToString(); Assert.IsFalse(s.Contains("has deletions")); dir.Dispose(); }
public virtual void TestDeletedDocs() { Directory dir = NewDirectory(); IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2)); for (int i = 0; i < 19; i++) { Document doc = new Document(); FieldType customType = new FieldType(TextField.TYPE_STORED); customType.StoreTermVectors = true; customType.StoreTermVectorPositions = true; customType.StoreTermVectorOffsets = true; doc.Add(NewField("field", "aaa" + i, customType)); writer.AddDocument(doc); } writer.ForceMerge(1); writer.Commit(); writer.DeleteDocuments(new Term("field", "aaa5")); writer.Dispose(); ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); CheckIndex checker = new CheckIndex(dir); checker.InfoStream = new StreamWriter(bos, Encoding.UTF8); if (VERBOSE) { checker.InfoStream = Console.Out; } CheckIndex.Status indexStatus = checker.DoCheckIndex(); if (indexStatus.Clean == false) { Console.WriteLine("CheckIndex failed"); checker.FlushInfoStream(); Console.WriteLine(bos.ToString()); Assert.Fail(); } CheckIndex.Status.SegmentInfoStatus seg = indexStatus.SegmentInfos[0]; Assert.IsTrue(seg.OpenReaderPassed); Assert.IsNotNull(seg.Diagnostics); Assert.IsNotNull(seg.FieldNormStatus); Assert.IsNull(seg.FieldNormStatus.Error); Assert.AreEqual(1, seg.FieldNormStatus.TotFields); Assert.IsNotNull(seg.TermIndexStatus); Assert.IsNull(seg.TermIndexStatus.Error); Assert.AreEqual(18, seg.TermIndexStatus.TermCount); Assert.AreEqual(18, seg.TermIndexStatus.TotFreq); Assert.AreEqual(18, seg.TermIndexStatus.TotPos); Assert.IsNotNull(seg.StoredFieldStatus); Assert.IsNull(seg.StoredFieldStatus.Error); Assert.AreEqual(18, seg.StoredFieldStatus.DocCount); Assert.AreEqual(18, seg.StoredFieldStatus.TotFields); Assert.IsNotNull(seg.TermVectorStatus); Assert.IsNull(seg.TermVectorStatus.Error); Assert.AreEqual(18, seg.TermVectorStatus.DocCount); Assert.AreEqual(18, seg.TermVectorStatus.TotVectors); Assert.IsTrue(seg.Diagnostics.Count > 0); IList<string> onlySegments = new List<string>(); onlySegments.Add("_0"); Assert.IsTrue(checker.DoCheckIndex(onlySegments).Clean == true); dir.Dispose(); }
public static bool Check(Directory dir, bool doFix, System.Collections.IList onlySegments) { CheckIndex checker = new CheckIndex(dir); Status status = checker.CheckIndex_Renamed_Method(onlySegments); if (doFix && !status.clean) checker.FixIndex(status); return status.clean; }
public virtual void TestUnsupportedOldIndexes() { for (int i = 0; i < UnsupportedNames.Length; i++) { if (VERBOSE) { Console.WriteLine("TEST: index " + UnsupportedNames[i]); } DirectoryInfo oldIndexDir = CreateTempDir(UnsupportedNames[i]); TestUtil.Unzip(GetDataFile("unsupported." + UnsupportedNames[i] + ".zip"), oldIndexDir); BaseDirectoryWrapper dir = NewFSDirectory(oldIndexDir); // don't checkindex, these are intentionally not supported dir.CheckIndexOnClose = false; IndexReader reader = null; IndexWriter writer = null; try { reader = DirectoryReader.Open(dir); Assert.Fail("DirectoryReader.open should not pass for " + UnsupportedNames[i]); } catch (IndexFormatTooOldException e) { // pass } finally { if (reader != null) { reader.Dispose(); } reader = null; } try { writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); Assert.Fail("IndexWriter creation should not pass for " + UnsupportedNames[i]); } catch (IndexFormatTooOldException e) { // pass if (VERBOSE) { Console.WriteLine("TEST: got expected exc:"); Console.WriteLine(e.StackTrace); } // Make sure exc message includes a path= Assert.IsTrue(e.Message.IndexOf("path=\"") != -1, "got exc message: " + e.Message); } finally { // we should fail to open IW, and so it should be null when we get here. // However, if the test fails (i.e., IW did not fail on open), we need // to close IW. However, if merges are run, IW may throw // IndexFormatTooOldException, and we don't want to mask the Assert.Fail() // above, so close without waiting for merges. if (writer != null) { writer.Dispose(false); } writer = null; } MemoryStream bos = new MemoryStream(1024); CheckIndex checker = new CheckIndex(dir); checker.InfoStream = new StreamWriter(bos.ToString(), false, IOUtils.CHARSET_UTF_8); CheckIndex.Status indexStatus = checker.DoCheckIndex(); Assert.IsFalse(indexStatus.Clean); Assert.IsTrue(bos.ToString().Contains(typeof(IndexFormatTooOldException).Name)); dir.Dispose(); } }