/// <summary>
/// Recursively looks for indexes underneath <code>file</code>,
/// and runs checkindex on them. returns true if it found any indexes.
/// </summary>
public virtual bool CheckIndexes(DirectoryInfo file)
{
if (file.IsDirectory)
{
BaseDirectoryWrapper dir = NewFSDirectory(file);
dir.CheckIndexOnClose = false; // don't double-checkindex
if (DirectoryReader.IndexExists(dir))
{
if (VERBOSE)
{
Console.Error.WriteLine("Checking index: " + file);
}
// LUCENE-4738: if we crashed while writing first
// commit it's possible index will be corrupt (by
// design we don't try to be smart about this case
// since that too risky):
if (SegmentInfos.GetLastCommitGeneration(dir) > 1)
{
TestUtil.CheckIndex(dir);
}
dir.Dispose();
return(true);
}
dir.Dispose();
foreach (FileInfo f in file.ListAll())
{
if (CheckIndexes(f))
{
return(true);
}
}
}
return(false);
}
/// <summary>
/// Perform the upgrade. </summary>
public void Upgrade()
{
if (!DirectoryReader.IndexExists(dir))
{
throw new IndexNotFoundException(dir.ToString());
}
if (!deletePriorCommits)
{
ICollection <IndexCommit> commits = DirectoryReader.ListCommits(dir);
if (commits.Count > 1)
{
throw new System.ArgumentException("this tool was invoked to not delete prior commit points, but the following commits were found: " + commits);
}
}
IndexWriterConfig c = (IndexWriterConfig)iwc.Clone();
c.MergePolicy = new UpgradeIndexMergePolicy(c.MergePolicy);
c.IndexDeletionPolicy = new KeepOnlyLastCommitDeletionPolicy();
IndexWriter w = new IndexWriter(dir, c);
try
{
InfoStream infoStream = c.InfoStream;
if (infoStream.IsEnabled("IndexUpgrader"))
{
infoStream.Message("IndexUpgrader", "Upgrading all pre-" + Constants.LUCENE_MAIN_VERSION + " segments of index directory '" + dir + "' to version " + Constants.LUCENE_MAIN_VERSION + "...");
}
w.ForceMerge(1);
if (infoStream.IsEnabled("IndexUpgrader"))
{
infoStream.Message("IndexUpgrader", "All segments upgraded to version " + Constants.LUCENE_MAIN_VERSION);
}
}
finally
{
w.Dispose();
}
}
public virtual void Test()
{
MockDirectoryWrapper dir = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors"));
dir.PreventDoubleWrite = false;
double rate = Random.NextDouble() * 0.01;
//System.out.println("rate=" + rate);
dir.RandomIOExceptionRateOnOpen = rate;
int iters = AtLeast(20);
LineFileDocs docs = new LineFileDocs(Random, DefaultCodecSupportsDocValues);
IndexReader r = null;
DirectoryReader r2 = null;
bool any = false;
MockDirectoryWrapper dirCopy = null;
int lastNumDocs = 0;
for (int iter = 0; iter < iters; iter++)
{
IndexWriter w = null;
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter);
}
try
{
MockAnalyzer analyzer = new MockAnalyzer(Random);
analyzer.MaxTokenLength = TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
if (VERBOSE)
{
// Do this ourselves instead of relying on LTC so
// we see incrementing messageID:
iwc.SetInfoStream(new TextWriterInfoStream(Console.Out));
}
var ms = iwc.MergeScheduler;
if (ms is IConcurrentMergeScheduler)
{
((IConcurrentMergeScheduler)ms).SetSuppressExceptions();
}
w = new IndexWriter(dir, iwc);
if (r != null && Random.Next(5) == 3)
{
if (Random.NextBoolean())
{
if (VERBOSE)
{
Console.WriteLine("TEST: addIndexes IR[]");
}
w.AddIndexes(new IndexReader[] { r });
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: addIndexes Directory[]");
}
w.AddIndexes(new Directory[] { dirCopy });
}
}
else
{
if (VERBOSE)
{
Console.WriteLine("TEST: addDocument");
}
w.AddDocument(docs.NextDoc());
}
dir.RandomIOExceptionRateOnOpen = 0.0;
w.Dispose();
w = null;
// NOTE: this is O(N^2)! Only enable for temporary debugging:
//dir.setRandomIOExceptionRateOnOpen(0.0);
//TestUtil.CheckIndex(dir);
//dir.setRandomIOExceptionRateOnOpen(rate);
// Verify numDocs only increases, to catch IndexWriter
// accidentally deleting the index:
dir.RandomIOExceptionRateOnOpen = 0.0;
Assert.IsTrue(DirectoryReader.IndexExists(dir));
if (r2 == null)
{
r2 = DirectoryReader.Open(dir);
}
else
{
DirectoryReader r3 = DirectoryReader.OpenIfChanged(r2);
if (r3 != null)
{
r2.Dispose();
r2 = r3;
}
}
Assert.IsTrue(r2.NumDocs >= lastNumDocs, "before=" + lastNumDocs + " after=" + r2.NumDocs);
lastNumDocs = r2.NumDocs;
//System.out.println("numDocs=" + lastNumDocs);
dir.RandomIOExceptionRateOnOpen = rate;
any = true;
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + ": success");
}
}
catch (IOException ioe)
{
if (VERBOSE)
{
Console.WriteLine("TEST: iter=" + iter + ": exception");
Console.WriteLine(ioe.ToString());
Console.Write(ioe.StackTrace);
}
if (w != null)
{
// NOTE: leave random IO exceptions enabled here,
// to verify that rollback does not try to write
// anything:
w.Rollback();
}
}
if (any && r == null && Random.NextBoolean())
{
// Make a copy of a non-empty index so we can use
// it to addIndexes later:
dir.RandomIOExceptionRateOnOpen = 0.0;
r = DirectoryReader.Open(dir);
dirCopy = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors.copy"));
HashSet <string> files = new HashSet <string>();
foreach (string file in dir.ListAll())
{
dir.Copy(dirCopy, file, file, IOContext.DEFAULT);
files.Add(file);
}
dirCopy.Sync(files);
// Have IW kiss the dir so we remove any leftover
// files ... we can easily have leftover files at
// the time we take a copy because we are holding
// open a reader:
(new IndexWriter(dirCopy, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)))).Dispose();
dirCopy.RandomIOExceptionRate = rate;
dir.RandomIOExceptionRateOnOpen = rate;
}
}
if (r2 != null)
{
r2.Dispose();
}
if (r != null)
{
r.Dispose();
dirCopy.Dispose();
}
dir.Dispose();
}
public static void Main(string[] args)
{
if (args.Length < 5)
{
Console.Error.WriteLine("Usage: MultiPassIndexSplitter -out <outputDir> -num <numParts> [-seq] <inputIndex1> [<inputIndex2 ...]");
Console.Error.WriteLine("\tinputIndex\tpath to input index, multiple values are ok");
Console.Error.WriteLine("\t-out ouputDir\tpath to output directory to contain partial indexes");
Console.Error.WriteLine("\t-num numParts\tnumber of parts to produce");
Console.Error.WriteLine("\t-seq\tsequential docid-range split (default is round-robin)");
Environment.Exit(-1);
}
List <IndexReader> indexes = new List <IndexReader>();
string outDir = null;
int numParts = -1;
bool seq = false;
for (int i = 0; i < args.Length; i++)
{
if (args[i].Equals("-out"))
{
outDir = args[++i];
}
else if (args[i].Equals("-num"))
{
numParts = Convert.ToInt32(args[++i]);
}
else if (args[i].Equals("-seq"))
{
seq = true;
}
else
{
DirectoryInfo file = new DirectoryInfo(args[i]);
if (!file.Exists)
{
Console.Error.WriteLine("Invalid input path - skipping: " + file);
continue;
}
Store.Directory dir = FSDirectory.Open(new DirectoryInfo(args[i]));
try
{
if (!DirectoryReader.IndexExists(dir))
{
Console.Error.WriteLine("Invalid input index - skipping: " + file);
continue;
}
}
catch (Exception)
{
Console.Error.WriteLine("Invalid input index - skipping: " + file);
continue;
}
indexes.Add(DirectoryReader.Open(dir));
}
}
if (outDir == null)
{
throw new Exception("Required argument missing: -out outputDir");
}
if (numParts < 2)
{
throw new Exception("Invalid value of required argument: -num numParts");
}
if (indexes.Count == 0)
{
throw new Exception("No input indexes to process");
}
DirectoryInfo @out = new DirectoryInfo(outDir);
@out.Create();
if (!new DirectoryInfo(outDir).Exists)
{
throw new Exception("Can't create output directory: " + @out);
}
Store.Directory[] dirs = new Store.Directory[numParts];
for (int i = 0; i < numParts; i++)
{
dirs[i] = FSDirectory.Open(new DirectoryInfo(Path.Combine(@out.FullName, "part-" + i)));
}
MultiPassIndexSplitter splitter = new MultiPassIndexSplitter();
IndexReader input;
if (indexes.Count == 1)
{
input = indexes[0];
}
else
{
input = new MultiReader(indexes.ToArray());
}
#pragma warning disable 612, 618
splitter.Split(LuceneVersion.LUCENE_CURRENT, input, dirs, seq);
#pragma warning restore 612, 618
}