Basic tool and API to check the health of an index and write a new segments file that removes reference to problematic segments.

As this tool checks every byte in the index, on a large index it can take quite a long time to run.

WARNING: this tool and API is new and experimental and is subject to suddenly change in the next release. Please make a complete backup of your index before using this to fix your index!

 private static void buildIndex(bool buildVerbose)
 {
     try
     {
         string[] si = MyIndex.GetAllIndexDirectory();
         Stopwatch sw = new Stopwatch();
         CleanIndexIfApplicable();
         SearchAutoComplete searchIndex = null;
         bool flagFirstTime = true;
         if (si != null && si.Length > 0)
         {
             foreach (string directory in si)
             {
                 if (directory != null)
                 {
                     searchIndex = new SearchAutoComplete(directory);
                     searchIndex.IsFirstTime = flagFirstTime;
                     System.IO.DirectoryInfo di = new System.IO.DirectoryInfo(directory);
                     if (di.Exists)
                     {
                         log.Info(string.Format("Index in directory {0} is building now...", di.Name));
                         FSDirectory fsd = FSDirectory.Open(di);
                         Lucene.Net.Index.CheckIndex ci = new CheckIndex(fsd);
                         Lucene.Net.Index.CheckIndex.Status st = ci.CheckIndex_Renamed_Method();
                         if (st.clean)
                         {
                             sw.Start();
                             searchIndex.BuildAutoCompleteIndex(fsd, MyIndex.GetAutoUpdateIndexDirectory(), buildVerbose);
                             sw.Stop();
                             TimeSpan ts = sw.Elapsed;
                             string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
                             log.Info(string.Format("Time taken {0} to build this index {1}", elapsedTime, di.Name));
                             flagFirstTime = false;
                         }
                         else
                         {
                             log.Info(string.Format("Following Index named {0} is corrupted, please rebuild that index using Sitecore Rebuild Index Wizard", di.Name));
                         }
                     }
                 }
             }
         }
     }
     catch (Exception ex)
     {
         log.Error("Error Occurred during buildIndex().", ex);
     }
 }
        public virtual void TestUnsupportedOldIndexes()
        {
            for (int i = 0; i < UnsupportedNames.Length; i++)
            {
                if (VERBOSE)
                {
                    Console.WriteLine("TEST: index " + UnsupportedNames[i]);
                }
                DirectoryInfo oldIndxeDir = CreateTempDir(UnsupportedNames[i]);
                using (Stream dataFile = this.GetType().Assembly.GetManifestResourceStream(CURRENT_RESOURCE_DIRECTORY + "unsupported." + UnsupportedNames[i] + ".zip"))
                {
                    TestUtil.Unzip(dataFile, oldIndxeDir);
                }
                BaseDirectoryWrapper dir = NewFSDirectory(oldIndxeDir);
                // don't checkindex, these are intentionally not supported
                dir.CheckIndexOnClose = false;

                IndexReader reader = null;
                IndexWriter writer = null;
                try
                {
                    reader = DirectoryReader.Open(dir);
                    Assert.Fail("DirectoryReader.open should not pass for " + UnsupportedNames[i]);
                }
                catch (IndexFormatTooOldException e)
                {
                    // pass
                }
                finally
                {
                    if (reader != null)
                    {
                        reader.Dispose();
                    }
                    reader = null;
                }

                try
                {
                    writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
                    Assert.Fail("IndexWriter creation should not pass for " + UnsupportedNames[i]);
                }
                catch (IndexFormatTooOldException e)
                {
                    // pass
                    if (VERBOSE)
                    {
                        Console.WriteLine("TEST: got expected exc:");
                        Console.WriteLine(e.StackTrace);
                    }
                    // Make sure exc message includes a path=
                    Assert.IsTrue(e.Message.IndexOf("path=\"") != -1, "got exc message: " + e.Message);
                }
                finally
                {
                    // we should fail to open IW, and so it should be null when we get here.
                    // However, if the test fails (i.e., IW did not fail on open), we need
                    // to close IW. However, if merges are run, IW may throw
                    // IndexFormatTooOldException, and we don't want to mask the Assert.Fail()
                    // above, so close without waiting for merges.
                    if (writer != null)
                    {
                        writer.Dispose(false);
                    }
                    writer = null;
                }

                StringBuilder sb = new StringBuilder(1024);
                CheckIndex checker = new CheckIndex(dir);
                CheckIndex.Status indexStatus;
                using (var infoStream = new StringWriter(sb))
                {
                    checker.InfoStream = infoStream;
                    indexStatus = checker.DoCheckIndex();
                }
                Assert.IsFalse(indexStatus.Clean);
                Assert.IsTrue(sb.ToString().Contains(typeof(IndexFormatTooOldException).Name));

                dir.Dispose();
                TestUtil.Rm(oldIndxeDir);
            }
        }
Example #3
0
		public virtual void  TestDeletedDocs()
		{
			MockRAMDirectory dir = new MockRAMDirectory();
			IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
			writer.SetMaxBufferedDocs(2);
			Document doc = new Document();
			doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
			for (int i = 0; i < 19; i++)
			{
				writer.AddDocument(doc);
			}
			writer.Close();
			IndexReader reader = IndexReader.Open(dir, false);
			reader.DeleteDocument(5);
			reader.Close();
			
			System.IO.MemoryStream bos = new System.IO.MemoryStream(1024);
			CheckIndex checker = new CheckIndex(dir);
			checker.SetInfoStream(new System.IO.StreamWriter(bos));
			//checker.setInfoStream(System.out);
			CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method();
			if (indexStatus.clean == false)
			{
				System.Console.Out.WriteLine("CheckIndex failed");
				char[] tmpChar;
				byte[] tmpByte;
				tmpByte = bos.GetBuffer();
				tmpChar = new char[bos.Length];
				System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length);
				System.Console.Out.WriteLine(new System.String(tmpChar));
				Assert.Fail();
			}
			
			CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus) indexStatus.segmentInfos[0];
			Assert.IsTrue(seg.openReaderPassed);
			
			Assert.IsNotNull(seg.diagnostics);
			
			Assert.IsNotNull(seg.fieldNormStatus);
			Assert.IsNull(seg.fieldNormStatus.error);
			Assert.AreEqual(1, seg.fieldNormStatus.totFields);
			
			Assert.IsNotNull(seg.termIndexStatus);
			Assert.IsNull(seg.termIndexStatus.error);
			Assert.AreEqual(1, seg.termIndexStatus.termCount);
			Assert.AreEqual(19, seg.termIndexStatus.totFreq);
			Assert.AreEqual(18, seg.termIndexStatus.totPos);
			
			Assert.IsNotNull(seg.storedFieldStatus);
			Assert.IsNull(seg.storedFieldStatus.error);
			Assert.AreEqual(18, seg.storedFieldStatus.docCount);
			Assert.AreEqual(18, seg.storedFieldStatus.totFields);
			
			Assert.IsNotNull(seg.termVectorStatus);
			Assert.IsNull(seg.termVectorStatus.error);
			Assert.AreEqual(18, seg.termVectorStatus.docCount);
			Assert.AreEqual(18, seg.termVectorStatus.totVectors);
			
			Assert.IsTrue(seg.diagnostics.Count > 0);
			List<string> onlySegments = new List<string>();
			onlySegments.Add("_0");
			
			Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true);
		}
Example #4
0
        public virtual void  TestDeletedDocs()
        {
            MockRAMDirectory dir    = new MockRAMDirectory();
            IndexWriter      writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            writer.SetMaxBufferedDocs(2);
            Document doc = new Document();

            doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
            for (int i = 0; i < 19; i++)
            {
                writer.AddDocument(doc);
            }
            writer.Close();
            IndexReader reader = IndexReader.Open(dir);

            reader.DeleteDocument(5);
            reader.Close();

            System.IO.MemoryStream bos     = new System.IO.MemoryStream(1024);
            CheckIndex             checker = new CheckIndex(dir);

            checker.SetInfoStream(new System.IO.StreamWriter(bos));
            //checker.setInfoStream(System.out);
            CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method();
            if (indexStatus.clean == false)
            {
                System.Console.Out.WriteLine("CheckIndex failed");
                char[] tmpChar;
                byte[] tmpByte;
                tmpByte = bos.GetBuffer();
                tmpChar = new char[bos.Length];
                System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length);
                System.Console.Out.WriteLine(new System.String(tmpChar));
                Assert.Fail();
            }

            CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus)indexStatus.segmentInfos[0];
            Assert.IsTrue(seg.openReaderPassed);

            Assert.IsNotNull(seg.diagnostics);

            Assert.IsNotNull(seg.fieldNormStatus);
            Assert.IsNull(seg.fieldNormStatus.error);
            Assert.AreEqual(1, seg.fieldNormStatus.totFields);

            Assert.IsNotNull(seg.termIndexStatus);
            Assert.IsNull(seg.termIndexStatus.error);
            Assert.AreEqual(1, seg.termIndexStatus.termCount);
            Assert.AreEqual(19, seg.termIndexStatus.totFreq);
            Assert.AreEqual(18, seg.termIndexStatus.totPos);

            Assert.IsNotNull(seg.storedFieldStatus);
            Assert.IsNull(seg.storedFieldStatus.error);
            Assert.AreEqual(18, seg.storedFieldStatus.docCount);
            Assert.AreEqual(18, seg.storedFieldStatus.totFields);

            Assert.IsNotNull(seg.termVectorStatus);
            Assert.IsNull(seg.termVectorStatus.error);
            Assert.AreEqual(18, seg.termVectorStatus.docCount);
            Assert.AreEqual(18, seg.termVectorStatus.totVectors);

            Assert.IsTrue(seg.diagnostics.Count > 0);
            System.Collections.IList onlySegments = new System.Collections.ArrayList();
            onlySegments.Add("_0");

            Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true);
        }
Example #5
0
		private static void CheckIndexAndTryToFix(Lucene.Net.Store.Directory directory, IndexDefinition indexDefinition)
		{
			startupLog.Warn("Unclean shutdown detected on {0}, checking the index for errors. This may take a while.", indexDefinition.Name);

			var memoryStream = new MemoryStream();
			var stringWriter = new StreamWriter(memoryStream);
			var checkIndex = new CheckIndex(directory);

			if (startupLog.IsWarnEnabled)
				checkIndex.SetInfoStream(stringWriter);

			var sp = Stopwatch.StartNew();
			var status = checkIndex.CheckIndex_Renamed_Method();
			sp.Stop();
			if (startupLog.IsWarnEnabled)
			{
				startupLog.Warn("Checking index {0} took: {1}, clean: {2}", indexDefinition.Name, sp.Elapsed, status.clean);
				memoryStream.Position = 0;

				log.Warn(new StreamReader(memoryStream).ReadToEnd());
			}

			if (status.clean)
				return;

			startupLog.Warn("Attempting to fix index: {0}", indexDefinition.Name);
			sp.Restart();
			checkIndex.FixIndex(status);
			startupLog.Warn("Fixed index {0} in {1}", indexDefinition.Name, sp.Elapsed);
		}
Example #6
0
 public static void  Main(System.String[] args)
 {
     
     bool doFix = false;
     var onlySegments = new List<string>();
     System.String indexPath = null;
     int i = 0;
     while (i < args.Length)
     {
         if (args[i].Equals("-fix"))
         {
             doFix = true;
             i++;
         }
         else if (args[i].Equals("-segment"))
         {
             if (i == args.Length - 1)
             {
                 System.Console.Out.WriteLine("ERROR: missing name for -segment option");
                 System.Environment.Exit(1);
             }
             onlySegments.Add(args[i + 1]);
             i += 2;
         }
         else
         {
             if (indexPath != null)
             {
                 System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'");
                 System.Environment.Exit(1);
             }
             indexPath = args[i];
             i++;
         }
     }
     
     if (indexPath == null)
     {
         System.Console.Out.WriteLine("\nERROR: index path not specified");
         System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + "  -fix: actually write a new segments_N file, removing any problematic segments\n" + "  -segment X: only check the specified segments.  This can be specified multiple\n" + "              times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + "              You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index.  Always make\n" + "a backup copy of your index before running this!  Do not run this tool on an index\n" + "that is actively being written to.  You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified.  With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file.  This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n");
         System.Environment.Exit(1);
     }
     
     if (!AssertsOn())
         System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled");
     
     if (onlySegments.Count == 0)
         onlySegments = null;
     else if (doFix)
     {
         System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment");
         System.Environment.Exit(1);
     }
     
     System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n");
     Directory dir = null;
     try
     {
         dir = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath));
     }
     catch (Exception t)
     {
         Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting");
         Console.Out.WriteLine(t.StackTrace);
         Environment.Exit(1);
     }
     
     var checker = new CheckIndex(dir);
     var tempWriter = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding)
                          {AutoFlush = true};
     checker.SetInfoStream(tempWriter);
     
     Status result = checker.CheckIndex_Renamed_Method(onlySegments);
     if (result.missingSegments)
     {
         System.Environment.Exit(1);
     }
     
     if (!result.clean)
     {
         if (!doFix)
         {
             System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
         }
         else
         {
             Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
             Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
             for (var s = 0; s < 5; s++)
             {
                 System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
                 System.Console.Out.WriteLine("  " + (5 - s) + "...");
             }
             Console.Out.WriteLine("Writing...");
             checker.FixIndex(result);
             Console.Out.WriteLine("OK");
             Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\"");
         }
     }
     System.Console.Out.WriteLine("");
     
     int exitCode;
     if (result != null && result.clean == true)
         exitCode = 0;
     else
         exitCode = 1;
     System.Environment.Exit(exitCode);
 }
Example #7
0
        private void CheckIndexAndTryToFix(LuceneDirectory directory)
        {
            StartupLog.Warn(string.Format("Unclean shutdown detected on file system '{0}', checking the index for errors. This may take a while.", name));

            var memoryStream = new MemoryStream();
            var stringWriter = new StreamWriter(memoryStream);
            var checkIndex = new CheckIndex(directory);

            if (StartupLog.IsWarnEnabled)
                checkIndex.SetInfoStream(stringWriter);

            var sp = Stopwatch.StartNew();
            var status = checkIndex.CheckIndex_Renamed_Method();
            sp.Stop();

            if (StartupLog.IsWarnEnabled)
            {
                StartupLog.Warn("Checking index for file system '{0}' took: {1}, clean: {2}", name, sp.Elapsed, status.clean);
                memoryStream.Position = 0;

                Log.Warn(new StreamReader(memoryStream).ReadToEnd());
            }

            if (status.clean)
                return;

            StartupLog.Warn("Attempting to fix index of file system: '{0}'", name);
            sp.Restart();
            checkIndex.FixIndex(status);
            StartupLog.Warn("Fixed index of file system '{0}' in {1}", name, sp.Elapsed);
        }
Example #8
0
        private bool IsIndexStateValid(LuceneDirectory luceneDirectory)
        {
            // 1. If commitData is null it means that there were no commits, so just in case we are resetting to Etag.Empty
            // 2. If no 'LastEtag' in commitData then we consider it an invalid index
            // 3. If 'LastEtag' is present (and valid), then resetting to it (if it is lower than lastStoredEtag)

            var commitData = IndexReader.GetCommitUserData(luceneDirectory);            

            string value;
            Etag lastEtag = null;
            if (commitData != null && commitData.TryGetValue("LastEtag", out value))
                Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail

            var lastStoredEtag = GetLastEtagForIndex() ?? Etag.Empty;
            lastEtag = lastEtag ?? Etag.Empty;

            if (EtagUtil.IsGreaterThan(lastEtag, lastStoredEtag))
                return false;

            var checkIndex = new CheckIndex(luceneDirectory);
            var status = checkIndex.CheckIndex_Renamed_Method();
            return status.clean;
        }
        public virtual void TestUnsupportedOldIndexes()
        {
            for (int i = 0; i < unsupportedNames.Length; i++)
            {
                if (Verbose)
                {
                    Console.WriteLine("TEST: index " + unsupportedNames[i]);
                }
                DirectoryInfo oldIndexDir = CreateTempDir(unsupportedNames[i]);
                using (Stream dataFile = this.GetType().FindAndGetManifestResourceStream("unsupported." + unsupportedNames[i] + ".zip"))
                {
                    TestUtil.Unzip(dataFile, oldIndexDir);
                }
                BaseDirectoryWrapper dir = NewFSDirectory(oldIndexDir);
                // don't checkindex, these are intentionally not supported
                dir.CheckIndexOnDispose = false;

                IndexReader reader = null;
                IndexWriter writer = null;
                try
                {
                    reader = DirectoryReader.Open(dir);
                    Assert.Fail("DirectoryReader.open should not pass for " + unsupportedNames[i]);
                }
#pragma warning disable 168
                catch (IndexFormatTooOldException e)
#pragma warning restore 168
                {
                    // pass
                }
                finally
                {
                    if (reader != null)
                    {
                        reader.Dispose();
                    }
                    reader = null;
                }

                try
                {
                    writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
                    Assert.Fail("IndexWriter creation should not pass for " + unsupportedNames[i]);
                }
                catch (IndexFormatTooOldException e)
                {
                    // pass
                    if (Verbose)
                    {
                        Console.WriteLine("TEST: got expected exc:");
                        Console.WriteLine(e.StackTrace);
                    }
                    // Make sure exc message includes a path=
                    Assert.IsTrue(e.Message.IndexOf("path=\"", StringComparison.Ordinal) != -1, "got exc message: " + e.Message);
                }
                finally
                {
                    // we should fail to open IW, and so it should be null when we get here.
                    // However, if the test fails (i.e., IW did not fail on open), we need
                    // to close IW. However, if merges are run, IW may throw
                    // IndexFormatTooOldException, and we don't want to mask the Assert.Fail()
                    // above, so close without waiting for merges.
                    if (writer != null)
                    {
                        writer.Dispose(false);
                    }
                    writer = null;
                }

                StringBuilder bos     = new StringBuilder();
                CheckIndex    checker = new CheckIndex(dir);
                checker.InfoStream = new StringWriter(bos);
                CheckIndex.Status indexStatus = checker.DoCheckIndex();
                Assert.IsFalse(indexStatus.Clean);
                checker.InfoStream.Flush();
                Assert.IsTrue(bos.ToString().Contains(typeof(IndexFormatTooOldException).Name));

                dir.Dispose();
            }
        }
Example #10
0
        public virtual void TestDeletedDocs()
        {
            Directory   dir    = NewDirectory();
            IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2));

            for (int i = 0; i < 19; i++)
            {
                Document  doc        = new Document();
                FieldType customType = new FieldType(TextField.TYPE_STORED);
                customType.StoreTermVectors         = true;
                customType.StoreTermVectorPositions = true;
                customType.StoreTermVectorOffsets   = true;
                doc.Add(NewField("field", "aaa" + i, customType));
                writer.AddDocument(doc);
            }
            writer.ForceMerge(1);
            writer.Commit();
            writer.DeleteDocuments(new Term("field", "aaa5"));
            writer.Dispose();

            ByteArrayOutputStream bos     = new ByteArrayOutputStream(1024);
            CheckIndex            checker = new CheckIndex(dir);

            checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
            if (VERBOSE)
            {
                checker.InfoStream = Console.Out;
            }
            CheckIndex.Status indexStatus = checker.DoCheckIndex();
            if (indexStatus.Clean == false)
            {
                Console.WriteLine("CheckIndex failed");
                checker.FlushInfoStream();
                Console.WriteLine(bos.ToString());
                Assert.Fail();
            }

            CheckIndex.Status.SegmentInfoStatus seg = indexStatus.SegmentInfos[0];
            Assert.IsTrue(seg.OpenReaderPassed);

            Assert.IsNotNull(seg.Diagnostics);

            Assert.IsNotNull(seg.FieldNormStatus);
            Assert.IsNull(seg.FieldNormStatus.Error);
            Assert.AreEqual(1, seg.FieldNormStatus.TotFields);

            Assert.IsNotNull(seg.TermIndexStatus);
            Assert.IsNull(seg.TermIndexStatus.Error);
            Assert.AreEqual(18, seg.TermIndexStatus.TermCount);
            Assert.AreEqual(18, seg.TermIndexStatus.TotFreq);
            Assert.AreEqual(18, seg.TermIndexStatus.TotPos);

            Assert.IsNotNull(seg.StoredFieldStatus);
            Assert.IsNull(seg.StoredFieldStatus.Error);
            Assert.AreEqual(18, seg.StoredFieldStatus.DocCount);
            Assert.AreEqual(18, seg.StoredFieldStatus.TotFields);

            Assert.IsNotNull(seg.TermVectorStatus);
            Assert.IsNull(seg.TermVectorStatus.Error);
            Assert.AreEqual(18, seg.TermVectorStatus.DocCount);
            Assert.AreEqual(18, seg.TermVectorStatus.TotVectors);

            Assert.IsTrue(seg.Diagnostics.Count > 0);
            IList <string> onlySegments = new List <string>();

            onlySegments.Add("_0");

            Assert.IsTrue(checker.DoCheckIndex(onlySegments).Clean == true);
            dir.Dispose();
        }
Example #11
0
        public static void  Main(System.String[] args)
        {
            bool doFix = false;

            System.Collections.IList onlySegments = new System.Collections.ArrayList();
            System.String            indexPath    = null;
            int i = 0;

            while (i < args.Length)
            {
                if (args[i].Equals("-fix"))
                {
                    doFix = true;
                    i++;
                }
                else if (args[i].Equals("-segment"))
                {
                    if (i == args.Length - 1)
                    {
                        System.Console.Out.WriteLine("ERROR: missing name for -segment option");
                        System.Environment.Exit(1);
                    }
                    onlySegments.Add(args[i + 1]);
                    i += 2;
                }
                else
                {
                    if (indexPath != null)
                    {
                        System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'");
                        System.Environment.Exit(1);
                    }
                    indexPath = args[i];
                    i++;
                }
            }

            if (indexPath == null)
            {
                System.Console.Out.WriteLine("\nERROR: index path not specified");
                System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + "  -fix: actually write a new segments_N file, removing any problematic segments\n" + "  -segment X: only check the specified segments.  This can be specified multiple\n" + "              times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + "              You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index.  Always make\n" + "a backup copy of your index before running this!  Do not run this tool on an index\n" + "that is actively being written to.  You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified.  With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file.  This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n");
                System.Environment.Exit(1);
            }

            if (!AssertsOn())
            {
                System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled");
            }

            if (onlySegments.Count == 0)
            {
                onlySegments = null;
            }
            else if (doFix)
            {
                System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment");
                System.Environment.Exit(1);
            }

            System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n");
            Directory dir = null;

            try
            {
                dir = FSDirectory.Open(new System.IO.FileInfo(indexPath));
            }
            catch (System.Exception t)
            {
                System.Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting");
                System.Console.Out.WriteLine(t.StackTrace);
                System.Environment.Exit(1);
            }

            CheckIndex checker = new CheckIndex(dir);

            System.IO.StreamWriter temp_writer;
            temp_writer           = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
            temp_writer.AutoFlush = true;
            checker.SetInfoStream(temp_writer);

            Status result = checker.CheckIndex_Renamed_Method(onlySegments);

            if (result.missingSegments)
            {
                System.Environment.Exit(1);
            }

            if (!result.clean)
            {
                if (!doFix)
                {
                    System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
                }
                else
                {
                    System.Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
                    System.Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
                    for (int s = 0; s < 5; s++)
                    {
                        System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
                        System.Console.Out.WriteLine("  " + (5 - s) + "...");
                    }
                    System.Console.Out.WriteLine("Writing...");
                    checker.FixIndex(result);
                    System.Console.Out.WriteLine("OK");
                    System.Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\"");
                }
            }
            System.Console.Out.WriteLine("");

            int exitCode;

            if (result != null && result.clean == true)
            {
                exitCode = 0;
            }
            else
            {
                exitCode = 1;
            }
            System.Environment.Exit(exitCode);
        }
        public virtual void TestDeletesCheckIndexOutput()
        {
            Directory dir = NewDirectory();
            IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
            iwc.SetMaxBufferedDocs(2);
            IndexWriter w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
            Document doc = new Document();
            doc.Add(NewField("field", "0", StringField.TYPE_NOT_STORED));
            w.AddDocument(doc);

            doc = new Document();
            doc.Add(NewField("field", "1", StringField.TYPE_NOT_STORED));
            w.AddDocument(doc);
            w.Commit();
            Assert.AreEqual(1, w.SegmentCount);

            w.DeleteDocuments(new Term("field", "0"));
            w.Commit();
            Assert.AreEqual(1, w.SegmentCount);
            w.Dispose();

            ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
            //MemoryStream bos = new MemoryStream(1024);
            CheckIndex checker = new CheckIndex(dir);
            checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
            CheckIndex.Status indexStatus = checker.DoCheckIndex(null);
            Assert.IsTrue(indexStatus.Clean);
            checker.FlushInfoStream();
            string s = bos.ToString();

            // Segment should have deletions:
            Assert.IsTrue(s.Contains("has deletions"), "string was: " + s);
            w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
            w.ForceMerge(1);
            w.Dispose();

            bos = new ByteArrayOutputStream(1024);
            checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
            indexStatus = checker.DoCheckIndex(null);
            Assert.IsTrue(indexStatus.Clean);
            checker.FlushInfoStream();
            s = bos.ToString();
            Assert.IsFalse(s.Contains("has deletions"));
            dir.Dispose();
        }
        public virtual void TestDeletedDocs()
        {
            Directory dir = NewDirectory();
            IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
            for (int i = 0; i < 19; i++)
            {
                Document doc = new Document();
                FieldType customType = new FieldType(TextField.TYPE_STORED);
                customType.StoreTermVectors = true;
                customType.StoreTermVectorPositions = true;
                customType.StoreTermVectorOffsets = true;
                doc.Add(NewField("field", "aaa" + i, customType));
                writer.AddDocument(doc);
            }
            writer.ForceMerge(1);
            writer.Commit();
            writer.DeleteDocuments(new Term("field", "aaa5"));
            writer.Dispose();

            ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
            CheckIndex checker = new CheckIndex(dir);
            checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
            if (VERBOSE)
            {
                checker.InfoStream = Console.Out;
            }
            CheckIndex.Status indexStatus = checker.DoCheckIndex();
            if (indexStatus.Clean == false)
            {
                Console.WriteLine("CheckIndex failed");
                checker.FlushInfoStream();
                Console.WriteLine(bos.ToString());
                Assert.Fail();
            }

            CheckIndex.Status.SegmentInfoStatus seg = indexStatus.SegmentInfos[0];
            Assert.IsTrue(seg.OpenReaderPassed);

            Assert.IsNotNull(seg.Diagnostics);

            Assert.IsNotNull(seg.FieldNormStatus);
            Assert.IsNull(seg.FieldNormStatus.Error);
            Assert.AreEqual(1, seg.FieldNormStatus.TotFields);

            Assert.IsNotNull(seg.TermIndexStatus);
            Assert.IsNull(seg.TermIndexStatus.Error);
            Assert.AreEqual(18, seg.TermIndexStatus.TermCount);
            Assert.AreEqual(18, seg.TermIndexStatus.TotFreq);
            Assert.AreEqual(18, seg.TermIndexStatus.TotPos);

            Assert.IsNotNull(seg.StoredFieldStatus);
            Assert.IsNull(seg.StoredFieldStatus.Error);
            Assert.AreEqual(18, seg.StoredFieldStatus.DocCount);
            Assert.AreEqual(18, seg.StoredFieldStatus.TotFields);

            Assert.IsNotNull(seg.TermVectorStatus);
            Assert.IsNull(seg.TermVectorStatus.Error);
            Assert.AreEqual(18, seg.TermVectorStatus.DocCount);
            Assert.AreEqual(18, seg.TermVectorStatus.TotVectors);

            Assert.IsTrue(seg.Diagnostics.Count > 0);
            IList<string> onlySegments = new List<string>();
            onlySegments.Add("_0");

            Assert.IsTrue(checker.DoCheckIndex(onlySegments).Clean == true);
            dir.Dispose();
        }
Example #14
0
		public static bool Check(Directory dir, bool doFix, System.Collections.IList onlySegments)
		{
			CheckIndex checker = new CheckIndex(dir);
			Status status = checker.CheckIndex_Renamed_Method(onlySegments);
			if (doFix && !status.clean)
				checker.FixIndex(status);
			
			return status.clean;
		}
Example #15
0
        public virtual void TestUnsupportedOldIndexes()
        {
            for (int i = 0; i < UnsupportedNames.Length; i++)
            {
                if (VERBOSE)
                {
                    Console.WriteLine("TEST: index " + UnsupportedNames[i]);
                }
                DirectoryInfo oldIndexDir = CreateTempDir(UnsupportedNames[i]);
                TestUtil.Unzip(GetDataFile("unsupported." + UnsupportedNames[i] + ".zip"), oldIndexDir);
                BaseDirectoryWrapper dir = NewFSDirectory(oldIndexDir);
                // don't checkindex, these are intentionally not supported
                dir.CheckIndexOnClose = false;

                IndexReader reader = null;
                IndexWriter writer = null;
                try
                {
                    reader = DirectoryReader.Open(dir);
                    Assert.Fail("DirectoryReader.open should not pass for " + UnsupportedNames[i]);
                }
                catch (IndexFormatTooOldException e)
                {
                    // pass
                }
                finally
                {
                    if (reader != null)
                    {
                        reader.Dispose();
                    }
                    reader = null;
                }

                try
                {
                    writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
                    Assert.Fail("IndexWriter creation should not pass for " + UnsupportedNames[i]);
                }
                catch (IndexFormatTooOldException e)
                {
                    // pass
                    if (VERBOSE)
                    {
                        Console.WriteLine("TEST: got expected exc:");
                        Console.WriteLine(e.StackTrace);
                    }
                    // Make sure exc message includes a path=
                    Assert.IsTrue(e.Message.IndexOf("path=\"") != -1, "got exc message: " + e.Message);
                }
                finally
                {
                    // we should fail to open IW, and so it should be null when we get here.
                    // However, if the test fails (i.e., IW did not fail on open), we need
                    // to close IW. However, if merges are run, IW may throw
                    // IndexFormatTooOldException, and we don't want to mask the Assert.Fail()
                    // above, so close without waiting for merges.
                    if (writer != null)
                    {
                        writer.Dispose(false);
                    }
                    writer = null;
                }

                MemoryStream bos     = new MemoryStream(1024);
                CheckIndex   checker = new CheckIndex(dir);
                checker.InfoStream = new StreamWriter(bos.ToString(), false, IOUtils.CHARSET_UTF_8);
                CheckIndex.Status indexStatus = checker.DoCheckIndex();
                Assert.IsFalse(indexStatus.Clean);
                Assert.IsTrue(bos.ToString().Contains(typeof(IndexFormatTooOldException).Name));

                dir.Dispose();
            }
        }