As this method checks every byte in the index, on a large index it can take quite a long time to run.
WARNING: make sure you only call this when the index is not opened by any writer.
public static bool Check(Directory dir, bool doFix, System.Collections.IList onlySegments) { CheckIndex checker = new CheckIndex(dir); Status status = checker.CheckIndex_Renamed_Method(onlySegments); if (doFix && !status.clean) { checker.FixIndex(status); } return(status.clean); }
private static void buildIndex(bool buildVerbose) { try { string[] si = MyIndex.GetAllIndexDirectory(); Stopwatch sw = new Stopwatch(); CleanIndexIfApplicable(); SearchAutoComplete searchIndex = null; bool flagFirstTime = true; if (si != null && si.Length > 0) { foreach (string directory in si) { if (directory != null) { searchIndex = new SearchAutoComplete(directory); searchIndex.IsFirstTime = flagFirstTime; System.IO.DirectoryInfo di = new System.IO.DirectoryInfo(directory); if (di.Exists) { log.Info(string.Format("Index in directory {0} is building now...", di.Name)); FSDirectory fsd = FSDirectory.Open(di); Lucene.Net.Index.CheckIndex ci = new CheckIndex(fsd); Lucene.Net.Index.CheckIndex.Status st = ci.CheckIndex_Renamed_Method(); if (st.clean) { sw.Start(); searchIndex.BuildAutoCompleteIndex(fsd, MyIndex.GetAutoUpdateIndexDirectory(), buildVerbose); sw.Stop(); TimeSpan ts = sw.Elapsed; string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); log.Info(string.Format("Time taken {0} to build this index {1}", elapsedTime, di.Name)); flagFirstTime = false; } else { log.Info(string.Format("Following Index named {0} is corrupted, please rebuild that index using Sitecore Rebuild Index Wizard", di.Name)); } } } } } } catch (Exception ex) { log.Error("Error Occurred during buildIndex().", ex); } }
public virtual void TestDeletedDocs() { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(2); Document doc = new Document(); doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); for (int i = 0; i < 19; i++) { writer.AddDocument(doc); } writer.Close(); IndexReader reader = IndexReader.Open(dir, false); reader.DeleteDocument(5); reader.Close(); System.IO.MemoryStream bos = new System.IO.MemoryStream(1024); CheckIndex checker = new CheckIndex(dir); checker.SetInfoStream(new System.IO.StreamWriter(bos)); //checker.setInfoStream(System.out); CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method(); if (indexStatus.clean == false) { System.Console.Out.WriteLine("CheckIndex failed"); char[] tmpChar; byte[] tmpByte; tmpByte = bos.GetBuffer(); tmpChar = new char[bos.Length]; System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length); System.Console.Out.WriteLine(new System.String(tmpChar)); Assert.Fail(); } CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus) indexStatus.segmentInfos[0]; Assert.IsTrue(seg.openReaderPassed); Assert.IsNotNull(seg.diagnostics); Assert.IsNotNull(seg.fieldNormStatus); Assert.IsNull(seg.fieldNormStatus.error); Assert.AreEqual(1, seg.fieldNormStatus.totFields); Assert.IsNotNull(seg.termIndexStatus); Assert.IsNull(seg.termIndexStatus.error); Assert.AreEqual(1, seg.termIndexStatus.termCount); Assert.AreEqual(19, seg.termIndexStatus.totFreq); Assert.AreEqual(18, seg.termIndexStatus.totPos); Assert.IsNotNull(seg.storedFieldStatus); Assert.IsNull(seg.storedFieldStatus.error); Assert.AreEqual(18, seg.storedFieldStatus.docCount); Assert.AreEqual(18, seg.storedFieldStatus.totFields); Assert.IsNotNull(seg.termVectorStatus); Assert.IsNull(seg.termVectorStatus.error); Assert.AreEqual(18, seg.termVectorStatus.docCount); Assert.AreEqual(18, seg.termVectorStatus.totVectors); Assert.IsTrue(seg.diagnostics.Count > 0); List<string> onlySegments = new List<string>(); onlySegments.Add("_0"); Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true); }
public virtual void TestDeletedDocs() { MockRAMDirectory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.SetMaxBufferedDocs(2); Document doc = new Document(); doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); for (int i = 0; i < 19; i++) { writer.AddDocument(doc); } writer.Close(); IndexReader reader = IndexReader.Open(dir); reader.DeleteDocument(5); reader.Close(); System.IO.MemoryStream bos = new System.IO.MemoryStream(1024); CheckIndex checker = new CheckIndex(dir); checker.SetInfoStream(new System.IO.StreamWriter(bos)); //checker.setInfoStream(System.out); CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method(); if (indexStatus.clean == false) { System.Console.Out.WriteLine("CheckIndex failed"); char[] tmpChar; byte[] tmpByte; tmpByte = bos.GetBuffer(); tmpChar = new char[bos.Length]; System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length); System.Console.Out.WriteLine(new System.String(tmpChar)); Assert.Fail(); } CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus)indexStatus.segmentInfos[0]; Assert.IsTrue(seg.openReaderPassed); Assert.IsNotNull(seg.diagnostics); Assert.IsNotNull(seg.fieldNormStatus); Assert.IsNull(seg.fieldNormStatus.error); Assert.AreEqual(1, seg.fieldNormStatus.totFields); Assert.IsNotNull(seg.termIndexStatus); Assert.IsNull(seg.termIndexStatus.error); Assert.AreEqual(1, seg.termIndexStatus.termCount); Assert.AreEqual(19, seg.termIndexStatus.totFreq); Assert.AreEqual(18, seg.termIndexStatus.totPos); Assert.IsNotNull(seg.storedFieldStatus); Assert.IsNull(seg.storedFieldStatus.error); Assert.AreEqual(18, seg.storedFieldStatus.docCount); Assert.AreEqual(18, seg.storedFieldStatus.totFields); Assert.IsNotNull(seg.termVectorStatus); Assert.IsNull(seg.termVectorStatus.error); Assert.AreEqual(18, seg.termVectorStatus.docCount); Assert.AreEqual(18, seg.termVectorStatus.totVectors); Assert.IsTrue(seg.diagnostics.Count > 0); System.Collections.IList onlySegments = new System.Collections.ArrayList(); onlySegments.Add("_0"); Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true); }
private static void CheckIndexAndTryToFix(Lucene.Net.Store.Directory directory, IndexDefinition indexDefinition) { startupLog.Warn("Unclean shutdown detected on {0}, checking the index for errors. This may take a while.", indexDefinition.Name); var memoryStream = new MemoryStream(); var stringWriter = new StreamWriter(memoryStream); var checkIndex = new CheckIndex(directory); if (startupLog.IsWarnEnabled) checkIndex.SetInfoStream(stringWriter); var sp = Stopwatch.StartNew(); var status = checkIndex.CheckIndex_Renamed_Method(); sp.Stop(); if (startupLog.IsWarnEnabled) { startupLog.Warn("Checking index {0} took: {1}, clean: {2}", indexDefinition.Name, sp.Elapsed, status.clean); memoryStream.Position = 0; log.Warn(new StreamReader(memoryStream).ReadToEnd()); } if (status.clean) return; startupLog.Warn("Attempting to fix index: {0}", indexDefinition.Name); sp.Restart(); checkIndex.FixIndex(status); startupLog.Warn("Fixed index {0} in {1}", indexDefinition.Name, sp.Elapsed); }
public static void Main(System.String[] args) { bool doFix = false; var onlySegments = new List<string>(); System.String indexPath = null; int i = 0; while (i < args.Length) { if (args[i].Equals("-fix")) { doFix = true; i++; } else if (args[i].Equals("-segment")) { if (i == args.Length - 1) { System.Console.Out.WriteLine("ERROR: missing name for -segment option"); System.Environment.Exit(1); } onlySegments.Add(args[i + 1]); i += 2; } else { if (indexPath != null) { System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'"); System.Environment.Exit(1); } indexPath = args[i]; i++; } } if (indexPath == null) { System.Console.Out.WriteLine("\nERROR: index path not specified"); System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n"); System.Environment.Exit(1); } if (!AssertsOn()) System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled"); if (onlySegments.Count == 0) onlySegments = null; else if (doFix) { System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment"); System.Environment.Exit(1); } System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n"); Directory dir = null; try { dir = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath)); } catch (Exception t) { Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting"); Console.Out.WriteLine(t.StackTrace); Environment.Exit(1); } var checker = new CheckIndex(dir); var tempWriter = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding) {AutoFlush = true}; checker.SetInfoStream(tempWriter); Status result = checker.CheckIndex_Renamed_Method(onlySegments); if (result.missingSegments) { System.Environment.Exit(1); } if (!result.clean) { if (!doFix) { System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n"); } else { Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n"); Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!"); for (var s = 0; s < 5; s++) { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000)); System.Console.Out.WriteLine(" " + (5 - s) + "..."); } Console.Out.WriteLine("Writing..."); checker.FixIndex(result); Console.Out.WriteLine("OK"); Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\""); } } System.Console.Out.WriteLine(""); int exitCode; if (result != null && result.clean == true) exitCode = 0; else exitCode = 1; System.Environment.Exit(exitCode); }
private void CheckIndexAndTryToFix(LuceneDirectory directory) { StartupLog.Warn(string.Format("Unclean shutdown detected on file system '{0}', checking the index for errors. This may take a while.", name)); var memoryStream = new MemoryStream(); var stringWriter = new StreamWriter(memoryStream); var checkIndex = new CheckIndex(directory); if (StartupLog.IsWarnEnabled) checkIndex.SetInfoStream(stringWriter); var sp = Stopwatch.StartNew(); var status = checkIndex.CheckIndex_Renamed_Method(); sp.Stop(); if (StartupLog.IsWarnEnabled) { StartupLog.Warn("Checking index for file system '{0}' took: {1}, clean: {2}", name, sp.Elapsed, status.clean); memoryStream.Position = 0; Log.Warn(new StreamReader(memoryStream).ReadToEnd()); } if (status.clean) return; StartupLog.Warn("Attempting to fix index of file system: '{0}'", name); sp.Restart(); checkIndex.FixIndex(status); StartupLog.Warn("Fixed index of file system '{0}' in {1}", name, sp.Elapsed); }
private bool IsIndexStateValid(LuceneDirectory luceneDirectory) { // 1. If commitData is null it means that there were no commits, so just in case we are resetting to Etag.Empty // 2. If no 'LastEtag' in commitData then we consider it an invalid index // 3. If 'LastEtag' is present (and valid), then resetting to it (if it is lower than lastStoredEtag) var commitData = IndexReader.GetCommitUserData(luceneDirectory); string value; Etag lastEtag = null; if (commitData != null && commitData.TryGetValue("LastEtag", out value)) Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail var lastStoredEtag = GetLastEtagForIndex() ?? Etag.Empty; lastEtag = lastEtag ?? Etag.Empty; if (EtagUtil.IsGreaterThan(lastEtag, lastStoredEtag)) return false; var checkIndex = new CheckIndex(luceneDirectory); var status = checkIndex.CheckIndex_Renamed_Method(); return status.clean; }
public static void Main(System.String[] args) { bool doFix = false; System.Collections.IList onlySegments = new System.Collections.ArrayList(); System.String indexPath = null; int i = 0; while (i < args.Length) { if (args[i].Equals("-fix")) { doFix = true; i++; } else if (args[i].Equals("-segment")) { if (i == args.Length - 1) { System.Console.Out.WriteLine("ERROR: missing name for -segment option"); System.Environment.Exit(1); } onlySegments.Add(args[i + 1]); i += 2; } else { if (indexPath != null) { System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'"); System.Environment.Exit(1); } indexPath = args[i]; i++; } } if (indexPath == null) { System.Console.Out.WriteLine("\nERROR: index path not specified"); System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n"); System.Environment.Exit(1); } if (!AssertsOn()) { System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled"); } if (onlySegments.Count == 0) { onlySegments = null; } else if (doFix) { System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment"); System.Environment.Exit(1); } System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n"); Directory dir = null; try { dir = FSDirectory.Open(new System.IO.FileInfo(indexPath)); } catch (System.Exception t) { System.Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting"); System.Console.Out.WriteLine(t.StackTrace); System.Environment.Exit(1); } CheckIndex checker = new CheckIndex(dir); System.IO.StreamWriter temp_writer; temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding); temp_writer.AutoFlush = true; checker.SetInfoStream(temp_writer); Status result = checker.CheckIndex_Renamed_Method(onlySegments); if (result.missingSegments) { System.Environment.Exit(1); } if (!result.clean) { if (!doFix) { System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n"); } else { System.Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n"); System.Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!"); for (int s = 0; s < 5; s++) { System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000)); System.Console.Out.WriteLine(" " + (5 - s) + "..."); } System.Console.Out.WriteLine("Writing..."); checker.FixIndex(result); System.Console.Out.WriteLine("OK"); System.Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\""); } } System.Console.Out.WriteLine(""); int exitCode; if (result != null && result.clean == true) { exitCode = 0; } else { exitCode = 1; } System.Environment.Exit(exitCode); }
public static bool Check(Directory dir, bool doFix, System.Collections.IList onlySegments) { CheckIndex checker = new CheckIndex(dir); Status status = checker.CheckIndex_Renamed_Method(onlySegments); if (doFix && !status.clean) checker.FixIndex(status); return status.clean; }