internal readonly Codec Codec; // for writing new segments
/// <summary>
/// Constructs a new IndexWriter per the settings given in <code>conf</code>.
/// If you want to make "live" changes to this writer instance, use
/// <seealso cref="#getConfig()"/>.
///
/// <p>
/// <b>NOTE:</b> after ths writer is created, the given configuration instance
/// cannot be passed to another writer. If you intend to do so, you should
/// <seealso cref="IndexWriterConfig#clone() clone"/> it beforehand.
/// </summary>
/// <param name="d">
/// the index directory. The index is either created or appended
/// according <code>conf.getOpenMode()</code>. </param>
/// <param name="conf">
/// the configuration settings according to which IndexWriter should
/// be initialized. </param>
/// <exception cref="IOException">
/// if the directory cannot be read/written to, or if it does not
/// exist and <code>conf.getOpenMode()</code> is
/// <code>OpenMode.APPEND</code> or if there is any other low-level
/// IO error </exception>
public IndexWriter(Directory d, IndexWriterConfig conf)
{
/*if (!InstanceFieldsInitialized)
{
InitializeInstanceFields();
InstanceFieldsInitialized = true;
}*/
readerPool = new ReaderPool(this);
conf.SetIndexWriter(this); // prevent reuse by other instances
Config_Renamed = new LiveIndexWriterConfig(conf);
directory = d;
analyzer = Config_Renamed.Analyzer;
infoStream = Config_Renamed.InfoStream;
mergePolicy = Config_Renamed.MergePolicy;
mergePolicy.IndexWriter = this;
mergeScheduler = Config_Renamed.MergeScheduler;
Codec = Config_Renamed.Codec;
BufferedUpdatesStream = new BufferedUpdatesStream(infoStream);
PoolReaders = Config_Renamed.ReaderPooling;
WriteLock = directory.MakeLock(WRITE_LOCK_NAME);
if (!WriteLock.Obtain(Config_Renamed.WriteLockTimeout)) // obtain write lock
{
throw new LockObtainFailedException("Index locked for write: " + WriteLock);
}
bool success = false;
try
{
OpenMode_e? mode = Config_Renamed.OpenMode;
bool create;
if (mode == OpenMode_e.CREATE)
{
create = true;
}
else if (mode == OpenMode_e.APPEND)
{
create = false;
}
else
{
// CREATE_OR_APPEND - create only if an index does not exist
create = !DirectoryReader.IndexExists(directory);
}
// If index is too old, reading the segments will throw
// IndexFormatTooOldException.
segmentInfos = new SegmentInfos();
bool initialIndexExists = true;
if (create)
{
// Try to read first. this is to allow create
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
try
{
segmentInfos.Read(directory);
segmentInfos.Clear();
}
catch (IOException)
{
// Likely this means it's a fresh directory
initialIndexExists = false;
}
// Record that we have a change (zero out all
// segments) pending:
Changed();
}
else
{
segmentInfos.Read(directory);
IndexCommit commit = Config_Renamed.IndexCommit;
if (commit != null)
{
// Swap out all segments, but, keep metadata in
// SegmentInfos, like version & generation, to
// preserve write-once. this is important if
// readers are open against the future commit
// points.
if (commit.Directory != directory)
{
throw new System.ArgumentException("IndexCommit's directory doesn't match my directory");
}
SegmentInfos oldInfos = new SegmentInfos();
oldInfos.Read(directory, commit.SegmentsFileName);
segmentInfos.Replace(oldInfos);
Changed();
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "init: loaded commit \"" + commit.SegmentsFileName + "\"");
}
}
}
RollbackSegments = segmentInfos.CreateBackupSegmentInfos();
// start with previous field numbers, but new FieldInfos
GlobalFieldNumberMap = FieldNumberMap;
Config_Renamed.FlushPolicy.Init(Config_Renamed);
DocWriter = new DocumentsWriter(this, Config_Renamed, directory);
eventQueue = DocWriter.EventQueue();
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
lock (this)
{
Deleter = new IndexFileDeleter(directory, Config_Renamed.DelPolicy, segmentInfos, infoStream, this, initialIndexExists);
}
if (Deleter.StartingCommitDeleted)
{
// Deletion policy deleted the "head" commit point.
// We have to mark ourself as changed so that if we
// are closed w/o any further changes we write a new
// segments_N file.
Changed();
}
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "init: create=" + create);
MessageState();
}
success = true;
}
finally
{
if (!success)
{
if (infoStream.IsEnabled("IW"))
{
infoStream.Message("IW", "init: hit exception on init; releasing write lock");
}
WriteLock.Release();
IOUtils.CloseWhileHandlingException(WriteLock);
WriteLock = null;
}
}
}
/// <summary>Returns true if index is clean, else false.</summary>
public static bool Check(Directory dir, bool doFix)
{
System.Globalization.NumberFormatInfo nf = System.Globalization.CultureInfo.CurrentCulture.NumberFormat;
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(dir);
}
catch (System.Exception t)
{
out_Renamed.WriteLine("ERROR: could not read any segments file in directory");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
return(false);
}
int numSegments = sis.Count;
System.String segmentsFileName = sis.GetCurrentSegmentFileName();
IndexInput input = null;
try
{
input = dir.OpenInput(segmentsFileName);
}
catch (System.Exception t)
{
out_Renamed.WriteLine("ERROR: could not open segments file in directory");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
return(false);
}
int format = 0;
try
{
format = input.ReadInt();
}
catch (System.Exception t)
{
out_Renamed.WriteLine("ERROR: could not read segment file version in directory");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
return(false);
}
finally
{
if (input != null)
{
input.Close();
}
}
System.String sFormat = "";
bool skip = false;
if (format == SegmentInfos.FORMAT)
{
sFormat = "FORMAT [Lucene Pre-2.1]";
}
if (format == SegmentInfos.FORMAT_LOCKLESS)
{
sFormat = "FORMAT_LOCKLESS [Lucene 2.1]";
}
else if (format == SegmentInfos.FORMAT_SINGLE_NORM_FILE)
{
sFormat = "FORMAT_SINGLE_NORM_FILE [Lucene 2.2]";
}
else if (format == SegmentInfos.FORMAT_SHARED_DOC_STORE)
{
sFormat = "FORMAT_SHARED_DOC_STORE [Lucene 2.3]";
}
else if (format < SegmentInfos.FORMAT_SHARED_DOC_STORE)
{
sFormat = "int=" + format + " [newer version of Lucene than this tool]";
skip = true;
}
else
{
sFormat = format + " [Lucene 1.3 or prior]";
}
out_Renamed.WriteLine("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat);
if (skip)
{
out_Renamed.WriteLine("\nERROR: this index appears to be created by a newer version of Lucene than this tool was compiled on; please re-compile this tool on the matching version of Lucene; exiting");
return(false);
}
SegmentInfos newSIS = (SegmentInfos)sis.Clone();
newSIS.Clear();
bool changed = false;
int totLoseDocCount = 0;
int numBadSegments = 0;
for (int i = 0; i < numSegments; i++)
{
SegmentInfo info = sis.Info(i);
out_Renamed.WriteLine(" " + (1 + i) + " of " + numSegments + ": name=" + info.name + " docCount=" + info.docCount);
int toLoseDocCount = info.docCount;
SegmentReader reader = null;
try
{
out_Renamed.WriteLine(" compound=" + info.GetUseCompoundFile());
out_Renamed.WriteLine(" numFiles=" + info.Files().Count);
out_Renamed.WriteLine(String.Format(nf, " size (MB)={0:f}", new Object[] { (info.SizeInBytes() / (1024.0 * 1024.0)) }));
int docStoreOffset = info.GetDocStoreOffset();
if (docStoreOffset != -1)
{
out_Renamed.WriteLine(" docStoreOffset=" + docStoreOffset);
out_Renamed.WriteLine(" docStoreSegment=" + info.GetDocStoreSegment());
out_Renamed.WriteLine(" docStoreIsCompoundFile=" + info.GetDocStoreIsCompoundFile());
}
System.String delFileName = info.GetDelFileName();
if (delFileName == null)
{
out_Renamed.WriteLine(" no deletions");
}
else
{
out_Renamed.WriteLine(" has deletions [delFileName=" + delFileName + "]");
}
out_Renamed.Write(" test: open reader.........");
reader = SegmentReader.Get(info);
int numDocs = reader.NumDocs();
toLoseDocCount = numDocs;
if (reader.HasDeletions())
{
out_Renamed.WriteLine("OK [" + (info.docCount - numDocs) + " deleted docs]");
}
else
{
out_Renamed.WriteLine("OK");
}
out_Renamed.Write(" test: fields, norms.......");
System.Collections.IDictionary fieldNames = (System.Collections.IDictionary)reader.GetFieldNames(IndexReader.FieldOption.ALL);
System.Collections.IEnumerator it = fieldNames.Keys.GetEnumerator();
while (it.MoveNext())
{
System.String fieldName = (System.String)it.Current;
byte[] b = reader.Norms(fieldName);
if (b.Length != info.docCount)
{
throw new System.SystemException("norms for field \"" + fieldName + "\" is length " + b.Length + " != maxDoc " + info.docCount);
}
}
out_Renamed.WriteLine("OK [" + fieldNames.Count + " fields]");
out_Renamed.Write(" test: terms, freq, prox...");
TermEnum termEnum = reader.Terms();
TermPositions termPositions = reader.TermPositions();
// Used only to count up # deleted docs for this
// term
MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader);
long termCount = 0;
long totFreq = 0;
long totPos = 0;
while (termEnum.Next())
{
termCount++;
Term term = termEnum.Term();
int docFreq = termEnum.DocFreq();
termPositions.Seek(term);
int lastDoc = -1;
int freq0 = 0;
totFreq += docFreq;
while (termPositions.Next())
{
freq0++;
int doc = termPositions.Doc();
int freq = termPositions.Freq();
if (doc <= lastDoc)
{
throw new System.SystemException("term " + term + ": doc " + doc + " < lastDoc " + lastDoc);
}
lastDoc = doc;
if (freq <= 0)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
}
int lastPos = -1;
totPos += freq;
for (int j = 0; j < freq; j++)
{
int pos = termPositions.NextPosition();
if (pos < 0)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
}
if (pos <= lastPos)
{
throw new System.SystemException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
}
}
}
// Now count how many deleted docs occurred in
// this term:
int delCount;
if (reader.HasDeletions())
{
myTermDocs.Seek(term);
while (myTermDocs.Next())
{
}
delCount = myTermDocs.delCount;
}
else
{
delCount = 0;
}
if (freq0 + delCount != docFreq)
{
throw new System.SystemException("term " + term + " docFreq=" + docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
}
}
out_Renamed.WriteLine("OK [" + termCount + " terms; " + totFreq + " terms/docs pairs; " + totPos + " tokens]");
out_Renamed.Write(" test: stored fields.......");
int docCount = 0;
long totFields = 0;
for (int j = 0; j < info.docCount; j++)
{
if (!reader.IsDeleted(j))
{
docCount++;
Document doc = reader.Document(j);
totFields += doc.GetFields().Count;
}
}
if (docCount != reader.NumDocs())
{
throw new System.SystemException("docCount=" + docCount + " but saw " + docCount + " undeleted docs");
}
out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total field count; avg {1:f} fields per doc]", new Object[] { totFields, (((float)totFields) / docCount) }));
out_Renamed.Write(" test: term vectors........");
int totVectors = 0;
for (int j = 0; j < info.docCount; j++)
{
if (!reader.IsDeleted(j))
{
TermFreqVector[] tfv = reader.GetTermFreqVectors(j);
if (tfv != null)
{
totVectors += tfv.Length;
}
}
}
out_Renamed.WriteLine(String.Format(nf, "OK [{0:d} total vector count; avg {1:f} term/freq vector fields per doc]", new Object[] { totVectors, (((float)totVectors) / docCount) }));
out_Renamed.WriteLine("");
}
catch (System.Exception t)
{
out_Renamed.WriteLine("FAILED");
System.String comment;
if (doFix)
{
comment = "will remove reference to this segment (-fix is specified)";
}
else
{
comment = "would remove reference to this segment (-fix was not specified)";
}
out_Renamed.WriteLine(" WARNING: " + comment + "; full exception:");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
out_Renamed.WriteLine("");
totLoseDocCount += toLoseDocCount;
numBadSegments++;
changed = true;
continue;
}
finally
{
if (reader != null)
{
reader.Close();
}
}
// Keeper
newSIS.Add(info.Clone());
}
if (!changed)
{
out_Renamed.WriteLine("No problems were detected with this index.\n");
return(true);
}
else
{
out_Renamed.WriteLine("WARNING: " + numBadSegments + " broken segments detected");
if (doFix)
{
out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents will be lost");
}
else
{
out_Renamed.WriteLine("WARNING: " + totLoseDocCount + " documents would be lost if -fix were specified");
}
out_Renamed.WriteLine();
}
if (doFix)
{
out_Renamed.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
for (int i = 0; i < 5; i++)
{
try
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
}
catch (System.Threading.ThreadInterruptedException)
{
SupportClass.ThreadClass.Current().Interrupt();
i--;
continue;
}
out_Renamed.WriteLine(" " + (5 - i) + "...");
}
out_Renamed.Write("Writing...");
try
{
newSIS.Write(dir);
}
catch (System.Exception t)
{
out_Renamed.WriteLine("FAILED; exiting");
out_Renamed.Write(t.StackTrace);
out_Renamed.Flush();
return(false);
}
out_Renamed.WriteLine("OK");
out_Renamed.WriteLine("Wrote new segments file \"" + newSIS.GetCurrentSegmentFileName() + "\"");
}
else
{
out_Renamed.WriteLine("NOTE: would write new segments file [-fix was not specified]");
}
out_Renamed.WriteLine("");
return(false);
}