private static void buildIndex(bool buildVerbose)
{
try
{
string[] si = MyIndex.GetAllIndexDirectory();
Stopwatch sw = new Stopwatch();
CleanIndexIfApplicable();
SearchAutoComplete searchIndex = null;
bool flagFirstTime = true;
if (si != null && si.Length > 0)
{
foreach (string directory in si)
{
if (directory != null)
{
searchIndex = new SearchAutoComplete(directory);
searchIndex.IsFirstTime = flagFirstTime;
System.IO.DirectoryInfo di = new System.IO.DirectoryInfo(directory);
if (di.Exists)
{
log.Info(string.Format("Index in directory {0} is building now...", di.Name));
FSDirectory fsd = FSDirectory.Open(di);
Lucene.Net.Index.CheckIndex ci = new CheckIndex(fsd);
Lucene.Net.Index.CheckIndex.Status st = ci.CheckIndex_Renamed_Method();
if (st.clean)
{
sw.Start();
searchIndex.BuildAutoCompleteIndex(fsd, MyIndex.GetAutoUpdateIndexDirectory(), buildVerbose);
sw.Stop();
TimeSpan ts = sw.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
log.Info(string.Format("Time taken {0} to build this index {1}", elapsedTime, di.Name));
flagFirstTime = false;
}
else
{
log.Info(string.Format("Following Index named {0} is corrupted, please rebuild that index using Sitecore Rebuild Index Wizard", di.Name));
}
}
}
}
}
}
catch (Exception ex)
{
log.Error("Error Occurred during buildIndex().", ex);
}
}
public virtual void TestUnsupportedOldIndexes()
{
for (int i = 0; i < UnsupportedNames.Length; i++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: index " + UnsupportedNames[i]);
}
DirectoryInfo oldIndxeDir = CreateTempDir(UnsupportedNames[i]);
using (Stream dataFile = this.GetType().Assembly.GetManifestResourceStream(CURRENT_RESOURCE_DIRECTORY + "unsupported." + UnsupportedNames[i] + ".zip"))
{
TestUtil.Unzip(dataFile, oldIndxeDir);
}
BaseDirectoryWrapper dir = NewFSDirectory(oldIndxeDir);
// don't checkindex, these are intentionally not supported
dir.CheckIndexOnClose = false;
IndexReader reader = null;
IndexWriter writer = null;
try
{
reader = DirectoryReader.Open(dir);
Assert.Fail("DirectoryReader.open should not pass for " + UnsupportedNames[i]);
}
catch (IndexFormatTooOldException e)
{
// pass
}
finally
{
if (reader != null)
{
reader.Dispose();
}
reader = null;
}
try
{
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Assert.Fail("IndexWriter creation should not pass for " + UnsupportedNames[i]);
}
catch (IndexFormatTooOldException e)
{
// pass
if (VERBOSE)
{
Console.WriteLine("TEST: got expected exc:");
Console.WriteLine(e.StackTrace);
}
// Make sure exc message includes a path=
Assert.IsTrue(e.Message.IndexOf("path=\"") != -1, "got exc message: " + e.Message);
}
finally
{
// we should fail to open IW, and so it should be null when we get here.
// However, if the test fails (i.e., IW did not fail on open), we need
// to close IW. However, if merges are run, IW may throw
// IndexFormatTooOldException, and we don't want to mask the Assert.Fail()
// above, so close without waiting for merges.
if (writer != null)
{
writer.Dispose(false);
}
writer = null;
}
StringBuilder sb = new StringBuilder(1024);
CheckIndex checker = new CheckIndex(dir);
CheckIndex.Status indexStatus;
using (var infoStream = new StringWriter(sb))
{
checker.InfoStream = infoStream;
indexStatus = checker.DoCheckIndex();
}
Assert.IsFalse(indexStatus.Clean);
Assert.IsTrue(sb.ToString().Contains(typeof(IndexFormatTooOldException).Name));
dir.Dispose();
TestUtil.Rm(oldIndxeDir);
}
}
public virtual void TestDeletedDocs()
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 19; i++)
{
writer.AddDocument(doc);
}
writer.Close();
IndexReader reader = IndexReader.Open(dir, false);
reader.DeleteDocument(5);
reader.Close();
System.IO.MemoryStream bos = new System.IO.MemoryStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.SetInfoStream(new System.IO.StreamWriter(bos));
//checker.setInfoStream(System.out);
CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method();
if (indexStatus.clean == false)
{
System.Console.Out.WriteLine("CheckIndex failed");
char[] tmpChar;
byte[] tmpByte;
tmpByte = bos.GetBuffer();
tmpChar = new char[bos.Length];
System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length);
System.Console.Out.WriteLine(new System.String(tmpChar));
Assert.Fail();
}
CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus) indexStatus.segmentInfos[0];
Assert.IsTrue(seg.openReaderPassed);
Assert.IsNotNull(seg.diagnostics);
Assert.IsNotNull(seg.fieldNormStatus);
Assert.IsNull(seg.fieldNormStatus.error);
Assert.AreEqual(1, seg.fieldNormStatus.totFields);
Assert.IsNotNull(seg.termIndexStatus);
Assert.IsNull(seg.termIndexStatus.error);
Assert.AreEqual(1, seg.termIndexStatus.termCount);
Assert.AreEqual(19, seg.termIndexStatus.totFreq);
Assert.AreEqual(18, seg.termIndexStatus.totPos);
Assert.IsNotNull(seg.storedFieldStatus);
Assert.IsNull(seg.storedFieldStatus.error);
Assert.AreEqual(18, seg.storedFieldStatus.docCount);
Assert.AreEqual(18, seg.storedFieldStatus.totFields);
Assert.IsNotNull(seg.termVectorStatus);
Assert.IsNull(seg.termVectorStatus.error);
Assert.AreEqual(18, seg.termVectorStatus.docCount);
Assert.AreEqual(18, seg.termVectorStatus.totVectors);
Assert.IsTrue(seg.diagnostics.Count > 0);
List<string> onlySegments = new List<string>();
onlySegments.Add("_0");
Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true);
}
public virtual void TestDeletedDocs()
{
MockRAMDirectory dir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(2);
Document doc = new Document();
doc.Add(new Field("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
for (int i = 0; i < 19; i++)
{
writer.AddDocument(doc);
}
writer.Close();
IndexReader reader = IndexReader.Open(dir);
reader.DeleteDocument(5);
reader.Close();
System.IO.MemoryStream bos = new System.IO.MemoryStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.SetInfoStream(new System.IO.StreamWriter(bos));
//checker.setInfoStream(System.out);
CheckIndex.Status indexStatus = checker.CheckIndex_Renamed_Method();
if (indexStatus.clean == false)
{
System.Console.Out.WriteLine("CheckIndex failed");
char[] tmpChar;
byte[] tmpByte;
tmpByte = bos.GetBuffer();
tmpChar = new char[bos.Length];
System.Array.Copy(tmpByte, 0, tmpChar, 0, tmpChar.Length);
System.Console.Out.WriteLine(new System.String(tmpChar));
Assert.Fail();
}
CheckIndex.Status.SegmentInfoStatus seg = (CheckIndex.Status.SegmentInfoStatus)indexStatus.segmentInfos[0];
Assert.IsTrue(seg.openReaderPassed);
Assert.IsNotNull(seg.diagnostics);
Assert.IsNotNull(seg.fieldNormStatus);
Assert.IsNull(seg.fieldNormStatus.error);
Assert.AreEqual(1, seg.fieldNormStatus.totFields);
Assert.IsNotNull(seg.termIndexStatus);
Assert.IsNull(seg.termIndexStatus.error);
Assert.AreEqual(1, seg.termIndexStatus.termCount);
Assert.AreEqual(19, seg.termIndexStatus.totFreq);
Assert.AreEqual(18, seg.termIndexStatus.totPos);
Assert.IsNotNull(seg.storedFieldStatus);
Assert.IsNull(seg.storedFieldStatus.error);
Assert.AreEqual(18, seg.storedFieldStatus.docCount);
Assert.AreEqual(18, seg.storedFieldStatus.totFields);
Assert.IsNotNull(seg.termVectorStatus);
Assert.IsNull(seg.termVectorStatus.error);
Assert.AreEqual(18, seg.termVectorStatus.docCount);
Assert.AreEqual(18, seg.termVectorStatus.totVectors);
Assert.IsTrue(seg.diagnostics.Count > 0);
System.Collections.IList onlySegments = new System.Collections.ArrayList();
onlySegments.Add("_0");
Assert.IsTrue(checker.CheckIndex_Renamed_Method(onlySegments).clean == true);
}
private static void CheckIndexAndTryToFix(Lucene.Net.Store.Directory directory, IndexDefinition indexDefinition)
{
startupLog.Warn("Unclean shutdown detected on {0}, checking the index for errors. This may take a while.", indexDefinition.Name);
var memoryStream = new MemoryStream();
var stringWriter = new StreamWriter(memoryStream);
var checkIndex = new CheckIndex(directory);
if (startupLog.IsWarnEnabled)
checkIndex.SetInfoStream(stringWriter);
var sp = Stopwatch.StartNew();
var status = checkIndex.CheckIndex_Renamed_Method();
sp.Stop();
if (startupLog.IsWarnEnabled)
{
startupLog.Warn("Checking index {0} took: {1}, clean: {2}", indexDefinition.Name, sp.Elapsed, status.clean);
memoryStream.Position = 0;
log.Warn(new StreamReader(memoryStream).ReadToEnd());
}
if (status.clean)
return;
startupLog.Warn("Attempting to fix index: {0}", indexDefinition.Name);
sp.Restart();
checkIndex.FixIndex(status);
startupLog.Warn("Fixed index {0} in {1}", indexDefinition.Name, sp.Elapsed);
}
public static void Main(System.String[] args)
{
bool doFix = false;
var onlySegments = new List<string>();
System.String indexPath = null;
int i = 0;
while (i < args.Length)
{
if (args[i].Equals("-fix"))
{
doFix = true;
i++;
}
else if (args[i].Equals("-segment"))
{
if (i == args.Length - 1)
{
System.Console.Out.WriteLine("ERROR: missing name for -segment option");
System.Environment.Exit(1);
}
onlySegments.Add(args[i + 1]);
i += 2;
}
else
{
if (indexPath != null)
{
System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'");
System.Environment.Exit(1);
}
indexPath = args[i];
i++;
}
}
if (indexPath == null)
{
System.Console.Out.WriteLine("\nERROR: index path not specified");
System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n");
System.Environment.Exit(1);
}
if (!AssertsOn())
System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled");
if (onlySegments.Count == 0)
onlySegments = null;
else if (doFix)
{
System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment");
System.Environment.Exit(1);
}
System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n");
Directory dir = null;
try
{
dir = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath));
}
catch (Exception t)
{
Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting");
Console.Out.WriteLine(t.StackTrace);
Environment.Exit(1);
}
var checker = new CheckIndex(dir);
var tempWriter = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding)
{AutoFlush = true};
checker.SetInfoStream(tempWriter);
Status result = checker.CheckIndex_Renamed_Method(onlySegments);
if (result.missingSegments)
{
System.Environment.Exit(1);
}
if (!result.clean)
{
if (!doFix)
{
System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
}
else
{
Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
for (var s = 0; s < 5; s++)
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
System.Console.Out.WriteLine(" " + (5 - s) + "...");
}
Console.Out.WriteLine("Writing...");
checker.FixIndex(result);
Console.Out.WriteLine("OK");
Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\"");
}
}
System.Console.Out.WriteLine("");
int exitCode;
if (result != null && result.clean == true)
exitCode = 0;
else
exitCode = 1;
System.Environment.Exit(exitCode);
}
private void CheckIndexAndTryToFix(LuceneDirectory directory)
{
StartupLog.Warn(string.Format("Unclean shutdown detected on file system '{0}', checking the index for errors. This may take a while.", name));
var memoryStream = new MemoryStream();
var stringWriter = new StreamWriter(memoryStream);
var checkIndex = new CheckIndex(directory);
if (StartupLog.IsWarnEnabled)
checkIndex.SetInfoStream(stringWriter);
var sp = Stopwatch.StartNew();
var status = checkIndex.CheckIndex_Renamed_Method();
sp.Stop();
if (StartupLog.IsWarnEnabled)
{
StartupLog.Warn("Checking index for file system '{0}' took: {1}, clean: {2}", name, sp.Elapsed, status.clean);
memoryStream.Position = 0;
Log.Warn(new StreamReader(memoryStream).ReadToEnd());
}
if (status.clean)
return;
StartupLog.Warn("Attempting to fix index of file system: '{0}'", name);
sp.Restart();
checkIndex.FixIndex(status);
StartupLog.Warn("Fixed index of file system '{0}' in {1}", name, sp.Elapsed);
}
private bool IsIndexStateValid(LuceneDirectory luceneDirectory)
{
// 1. If commitData is null it means that there were no commits, so just in case we are resetting to Etag.Empty
// 2. If no 'LastEtag' in commitData then we consider it an invalid index
// 3. If 'LastEtag' is present (and valid), then resetting to it (if it is lower than lastStoredEtag)
var commitData = IndexReader.GetCommitUserData(luceneDirectory);
string value;
Etag lastEtag = null;
if (commitData != null && commitData.TryGetValue("LastEtag", out value))
Etag.TryParse(value, out lastEtag); // etag will be null if parsing will fail
var lastStoredEtag = GetLastEtagForIndex() ?? Etag.Empty;
lastEtag = lastEtag ?? Etag.Empty;
if (EtagUtil.IsGreaterThan(lastEtag, lastStoredEtag))
return false;
var checkIndex = new CheckIndex(luceneDirectory);
var status = checkIndex.CheckIndex_Renamed_Method();
return status.clean;
}
public virtual void TestUnsupportedOldIndexes()
{
for (int i = 0; i < unsupportedNames.Length; i++)
{
if (Verbose)
{
Console.WriteLine("TEST: index " + unsupportedNames[i]);
}
DirectoryInfo oldIndexDir = CreateTempDir(unsupportedNames[i]);
using (Stream dataFile = this.GetType().FindAndGetManifestResourceStream("unsupported." + unsupportedNames[i] + ".zip"))
{
TestUtil.Unzip(dataFile, oldIndexDir);
}
BaseDirectoryWrapper dir = NewFSDirectory(oldIndexDir);
// don't checkindex, these are intentionally not supported
dir.CheckIndexOnDispose = false;
IndexReader reader = null;
IndexWriter writer = null;
try
{
reader = DirectoryReader.Open(dir);
Assert.Fail("DirectoryReader.open should not pass for " + unsupportedNames[i]);
}
#pragma warning disable 168
catch (IndexFormatTooOldException e)
#pragma warning restore 168
{
// pass
}
finally
{
if (reader != null)
{
reader.Dispose();
}
reader = null;
}
try
{
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)));
Assert.Fail("IndexWriter creation should not pass for " + unsupportedNames[i]);
}
catch (IndexFormatTooOldException e)
{
// pass
if (Verbose)
{
Console.WriteLine("TEST: got expected exc:");
Console.WriteLine(e.StackTrace);
}
// Make sure exc message includes a path=
Assert.IsTrue(e.Message.IndexOf("path=\"", StringComparison.Ordinal) != -1, "got exc message: " + e.Message);
}
finally
{
// we should fail to open IW, and so it should be null when we get here.
// However, if the test fails (i.e., IW did not fail on open), we need
// to close IW. However, if merges are run, IW may throw
// IndexFormatTooOldException, and we don't want to mask the Assert.Fail()
// above, so close without waiting for merges.
if (writer != null)
{
writer.Dispose(false);
}
writer = null;
}
StringBuilder bos = new StringBuilder();
CheckIndex checker = new CheckIndex(dir);
checker.InfoStream = new StringWriter(bos);
CheckIndex.Status indexStatus = checker.DoCheckIndex();
Assert.IsFalse(indexStatus.Clean);
checker.InfoStream.Flush();
Assert.IsTrue(bos.ToString().Contains(typeof(IndexFormatTooOldException).Name));
dir.Dispose();
}
}
public virtual void TestDeletedDocs()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2));
for (int i = 0; i < 19; i++)
{
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("field", "aaa" + i, customType));
writer.AddDocument(doc);
}
writer.ForceMerge(1);
writer.Commit();
writer.DeleteDocuments(new Term("field", "aaa5"));
writer.Dispose();
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
if (VERBOSE)
{
checker.InfoStream = Console.Out;
}
CheckIndex.Status indexStatus = checker.DoCheckIndex();
if (indexStatus.Clean == false)
{
Console.WriteLine("CheckIndex failed");
checker.FlushInfoStream();
Console.WriteLine(bos.ToString());
Assert.Fail();
}
CheckIndex.Status.SegmentInfoStatus seg = indexStatus.SegmentInfos[0];
Assert.IsTrue(seg.OpenReaderPassed);
Assert.IsNotNull(seg.Diagnostics);
Assert.IsNotNull(seg.FieldNormStatus);
Assert.IsNull(seg.FieldNormStatus.Error);
Assert.AreEqual(1, seg.FieldNormStatus.TotFields);
Assert.IsNotNull(seg.TermIndexStatus);
Assert.IsNull(seg.TermIndexStatus.Error);
Assert.AreEqual(18, seg.TermIndexStatus.TermCount);
Assert.AreEqual(18, seg.TermIndexStatus.TotFreq);
Assert.AreEqual(18, seg.TermIndexStatus.TotPos);
Assert.IsNotNull(seg.StoredFieldStatus);
Assert.IsNull(seg.StoredFieldStatus.Error);
Assert.AreEqual(18, seg.StoredFieldStatus.DocCount);
Assert.AreEqual(18, seg.StoredFieldStatus.TotFields);
Assert.IsNotNull(seg.TermVectorStatus);
Assert.IsNull(seg.TermVectorStatus.Error);
Assert.AreEqual(18, seg.TermVectorStatus.DocCount);
Assert.AreEqual(18, seg.TermVectorStatus.TotVectors);
Assert.IsTrue(seg.Diagnostics.Count > 0);
IList <string> onlySegments = new List <string>();
onlySegments.Add("_0");
Assert.IsTrue(checker.DoCheckIndex(onlySegments).Clean == true);
dir.Dispose();
}
public static void Main(System.String[] args)
{
bool doFix = false;
System.Collections.IList onlySegments = new System.Collections.ArrayList();
System.String indexPath = null;
int i = 0;
while (i < args.Length)
{
if (args[i].Equals("-fix"))
{
doFix = true;
i++;
}
else if (args[i].Equals("-segment"))
{
if (i == args.Length - 1)
{
System.Console.Out.WriteLine("ERROR: missing name for -segment option");
System.Environment.Exit(1);
}
onlySegments.Add(args[i + 1]);
i += 2;
}
else
{
if (indexPath != null)
{
System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'");
System.Environment.Exit(1);
}
indexPath = args[i];
i++;
}
}
if (indexPath == null)
{
System.Console.Out.WriteLine("\nERROR: index path not specified");
System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n");
System.Environment.Exit(1);
}
if (!AssertsOn())
{
System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled");
}
if (onlySegments.Count == 0)
{
onlySegments = null;
}
else if (doFix)
{
System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment");
System.Environment.Exit(1);
}
System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n");
Directory dir = null;
try
{
dir = FSDirectory.Open(new System.IO.FileInfo(indexPath));
}
catch (System.Exception t)
{
System.Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting");
System.Console.Out.WriteLine(t.StackTrace);
System.Environment.Exit(1);
}
CheckIndex checker = new CheckIndex(dir);
System.IO.StreamWriter temp_writer;
temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
temp_writer.AutoFlush = true;
checker.SetInfoStream(temp_writer);
Status result = checker.CheckIndex_Renamed_Method(onlySegments);
if (result.missingSegments)
{
System.Environment.Exit(1);
}
if (!result.clean)
{
if (!doFix)
{
System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
}
else
{
System.Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
System.Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
for (int s = 0; s < 5; s++)
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
System.Console.Out.WriteLine(" " + (5 - s) + "...");
}
System.Console.Out.WriteLine("Writing...");
checker.FixIndex(result);
System.Console.Out.WriteLine("OK");
System.Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\"");
}
}
System.Console.Out.WriteLine("");
int exitCode;
if (result != null && result.clean == true)
{
exitCode = 0;
}
else
{
exitCode = 1;
}
System.Environment.Exit(exitCode);
}
public virtual void TestDeletesCheckIndexOutput()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMaxBufferedDocs(2);
IndexWriter w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
Document doc = new Document();
doc.Add(NewField("field", "0", StringField.TYPE_NOT_STORED));
w.AddDocument(doc);
doc = new Document();
doc.Add(NewField("field", "1", StringField.TYPE_NOT_STORED));
w.AddDocument(doc);
w.Commit();
Assert.AreEqual(1, w.SegmentCount);
w.DeleteDocuments(new Term("field", "0"));
w.Commit();
Assert.AreEqual(1, w.SegmentCount);
w.Dispose();
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
//MemoryStream bos = new MemoryStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
CheckIndex.Status indexStatus = checker.DoCheckIndex(null);
Assert.IsTrue(indexStatus.Clean);
checker.FlushInfoStream();
string s = bos.ToString();
// Segment should have deletions:
Assert.IsTrue(s.Contains("has deletions"), "string was: " + s);
w = new IndexWriter(dir, (IndexWriterConfig)iwc.Clone());
w.ForceMerge(1);
w.Dispose();
bos = new ByteArrayOutputStream(1024);
checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
indexStatus = checker.DoCheckIndex(null);
Assert.IsTrue(indexStatus.Clean);
checker.FlushInfoStream();
s = bos.ToString();
Assert.IsFalse(s.Contains("has deletions"));
dir.Dispose();
}
public virtual void TestDeletedDocs()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
for (int i = 0; i < 19; i++)
{
Document doc = new Document();
FieldType customType = new FieldType(TextField.TYPE_STORED);
customType.StoreTermVectors = true;
customType.StoreTermVectorPositions = true;
customType.StoreTermVectorOffsets = true;
doc.Add(NewField("field", "aaa" + i, customType));
writer.AddDocument(doc);
}
writer.ForceMerge(1);
writer.Commit();
writer.DeleteDocuments(new Term("field", "aaa5"));
writer.Dispose();
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.InfoStream = new StreamWriter(bos, Encoding.UTF8);
if (VERBOSE)
{
checker.InfoStream = Console.Out;
}
CheckIndex.Status indexStatus = checker.DoCheckIndex();
if (indexStatus.Clean == false)
{
Console.WriteLine("CheckIndex failed");
checker.FlushInfoStream();
Console.WriteLine(bos.ToString());
Assert.Fail();
}
CheckIndex.Status.SegmentInfoStatus seg = indexStatus.SegmentInfos[0];
Assert.IsTrue(seg.OpenReaderPassed);
Assert.IsNotNull(seg.Diagnostics);
Assert.IsNotNull(seg.FieldNormStatus);
Assert.IsNull(seg.FieldNormStatus.Error);
Assert.AreEqual(1, seg.FieldNormStatus.TotFields);
Assert.IsNotNull(seg.TermIndexStatus);
Assert.IsNull(seg.TermIndexStatus.Error);
Assert.AreEqual(18, seg.TermIndexStatus.TermCount);
Assert.AreEqual(18, seg.TermIndexStatus.TotFreq);
Assert.AreEqual(18, seg.TermIndexStatus.TotPos);
Assert.IsNotNull(seg.StoredFieldStatus);
Assert.IsNull(seg.StoredFieldStatus.Error);
Assert.AreEqual(18, seg.StoredFieldStatus.DocCount);
Assert.AreEqual(18, seg.StoredFieldStatus.TotFields);
Assert.IsNotNull(seg.TermVectorStatus);
Assert.IsNull(seg.TermVectorStatus.Error);
Assert.AreEqual(18, seg.TermVectorStatus.DocCount);
Assert.AreEqual(18, seg.TermVectorStatus.TotVectors);
Assert.IsTrue(seg.Diagnostics.Count > 0);
IList<string> onlySegments = new List<string>();
onlySegments.Add("_0");
Assert.IsTrue(checker.DoCheckIndex(onlySegments).Clean == true);
dir.Dispose();
}
public static bool Check(Directory dir, bool doFix, System.Collections.IList onlySegments)
{
CheckIndex checker = new CheckIndex(dir);
Status status = checker.CheckIndex_Renamed_Method(onlySegments);
if (doFix && !status.clean)
checker.FixIndex(status);
return status.clean;
}
public virtual void TestUnsupportedOldIndexes()
{
for (int i = 0; i < UnsupportedNames.Length; i++)
{
if (VERBOSE)
{
Console.WriteLine("TEST: index " + UnsupportedNames[i]);
}
DirectoryInfo oldIndexDir = CreateTempDir(UnsupportedNames[i]);
TestUtil.Unzip(GetDataFile("unsupported." + UnsupportedNames[i] + ".zip"), oldIndexDir);
BaseDirectoryWrapper dir = NewFSDirectory(oldIndexDir);
// don't checkindex, these are intentionally not supported
dir.CheckIndexOnClose = false;
IndexReader reader = null;
IndexWriter writer = null;
try
{
reader = DirectoryReader.Open(dir);
Assert.Fail("DirectoryReader.open should not pass for " + UnsupportedNames[i]);
}
catch (IndexFormatTooOldException e)
{
// pass
}
finally
{
if (reader != null)
{
reader.Dispose();
}
reader = null;
}
try
{
writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
Assert.Fail("IndexWriter creation should not pass for " + UnsupportedNames[i]);
}
catch (IndexFormatTooOldException e)
{
// pass
if (VERBOSE)
{
Console.WriteLine("TEST: got expected exc:");
Console.WriteLine(e.StackTrace);
}
// Make sure exc message includes a path=
Assert.IsTrue(e.Message.IndexOf("path=\"") != -1, "got exc message: " + e.Message);
}
finally
{
// we should fail to open IW, and so it should be null when we get here.
// However, if the test fails (i.e., IW did not fail on open), we need
// to close IW. However, if merges are run, IW may throw
// IndexFormatTooOldException, and we don't want to mask the Assert.Fail()
// above, so close without waiting for merges.
if (writer != null)
{
writer.Dispose(false);
}
writer = null;
}
MemoryStream bos = new MemoryStream(1024);
CheckIndex checker = new CheckIndex(dir);
checker.InfoStream = new StreamWriter(bos.ToString(), false, IOUtils.CHARSET_UTF_8);
CheckIndex.Status indexStatus = checker.DoCheckIndex();
Assert.IsFalse(indexStatus.Clean);
Assert.IsTrue(bos.ToString().Contains(typeof(IndexFormatTooOldException).Name));
dir.Dispose();
}
}