internal void RollbackCommit(Directory dir)
{
if (pendingSegnOutput != null)
{
try
{
pendingSegnOutput.Close();
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
// in our caller
}
// Must carefully compute fileName from "generation"
// since lastGeneration isn't incremented:
try
{
System.String segmentFileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
dir.DeleteFile(segmentFileName);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
// in our caller
}
pendingSegnOutput = null;
}
}
/// <summary>Call this to start a commit. This writes the new
/// segments file, but writes an invalid checksum at the
/// end, so that it is not visible to readers. Once this
/// is called you must call <see cref="FinishCommit" /> to complete
/// the commit or <see cref="RollbackCommit" /> to abort it.
/// </summary>
internal void PrepareCommit(Directory dir)
{
if (pendingSegnOutput != null)
{
throw new System.SystemException("prepareCommit was already called");
}
Write(dir);
}
/// <summary> Get the generation (N) of the current segments_N file
/// in the directory.
///
/// </summary>
/// <param name="directory">-- directory to search for the latest segments_N file
/// </param>
public static long GetCurrentSegmentGeneration(Directory directory)
{
try
{
return(GetCurrentSegmentGeneration(directory.ListAll()));
}
catch (NoSuchDirectoryException)
{
return(-1);
}
}
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, System.String segmentName, System.String docStoreSegmentName, int numDocs, int numDocsInStore, int termIndexInterval)
{
this.docWriter = docWriter;
this.directory = directory;
this.segmentName = segmentName;
this.docStoreSegmentName = docStoreSegmentName;
this.numDocs = numDocs;
this.numDocsInStore = numDocsInStore;
this.termIndexInterval = termIndexInterval;
flushedFiles = new System.Collections.Generic.HashSet<string>();
}
public TermVectorsWriter(Directory directory, System.String segment, FieldInfos fieldInfos)
{
// Open files for TermVector storage
tvx = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_INDEX_EXTENSION);
tvx.WriteInt(TermVectorsReader.FORMAT_CURRENT);
tvd = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_DOCUMENTS_EXTENSION);
tvd.WriteInt(TermVectorsReader.FORMAT_CURRENT);
tvf = directory.CreateOutput(segment + "." + IndexFileNames.VECTORS_FIELDS_EXTENSION);
tvf.WriteInt(TermVectorsReader.FORMAT_CURRENT);
this.fieldInfos = fieldInfos;
}
internal CompoundFileWriter(Directory dir, System.String name, SegmentMerger.CheckAbort checkAbort)
{
if (dir == null)
throw new ArgumentNullException("dir");
if (name == null)
throw new ArgumentNullException("name");
this.checkAbort = checkAbort;
directory = dir;
fileName = name;
ids = new HashSet<string>();
entries = new LinkedList<FileEntry>();
}
public CompoundFileReader(Directory dir, System.String name, int readBufferSize)
{
directory = dir;
fileName = name;
this.readBufferSize = readBufferSize;
bool success = false;
try
{
stream = dir.OpenInput(name, readBufferSize);
// read the directory and init files
int count = stream.ReadVInt();
FileEntry entry = null;
for (int i = 0; i < count; i++)
{
long offset = stream.ReadLong();
System.String id = stream.ReadString();
if (entry != null)
{
// set length of the previous entry
entry.length = offset - entry.offset;
}
entry = new FileEntry {offset = offset};
entries[id] = entry;
}
// set the length of the final entry
if (entry != null)
{
entry.length = stream.Length() - entry.offset;
}
success = true;
}
finally
{
if (!success && (stream != null))
{
try
{
stream.Close();
}
catch (System.IO.IOException)
{
}
}
}
}
// Used only for testing
public bool HasExternalSegments(Directory dir)
{
int numSegments = Count;
for (int i = 0; i < numSegments; i++)
{
if (Info(i).dir != dir)
{
return(true);
}
}
return(false);
}
/// <summary> Construct a FieldInfos object using the directory and the name of the file
/// IndexInput
/// </summary>
/// <param name="d">The directory to open the IndexInput from
/// </param>
/// <param name="name">The name of the file to open the IndexInput from in the Directory
/// </param>
/// <throws> IOException </throws>
public /*internal*/ FieldInfos(Directory d, String name)
{
IndexInput input = d.OpenInput(name);
try
{
try
{
Read(input, name);
}
catch (System.IO.IOException)
{
if (format == FORMAT_PRE)
{
// LUCENE-1623: FORMAT_PRE (before there was a
// format) may be 2.3.2 (pre-utf8) or 2.4.x (utf8)
// encoding; retry with input set to pre-utf8
input.Seek(0);
input.SetModifiedUTF8StringsMode();
byNumber.Clear();
byName.Clear();
bool rethrow = false;
try
{
Read(input, name);
}
catch (Exception)
{
// Ignore any new exception & set to throw original IOE
rethrow = true;
}
if(rethrow)
{
// Preserve stack trace
throw;
}
}
else
{
// The IOException cannot be caused by
// LUCENE-1623, so re-throw it
throw;
}
}
}
finally
{
input.Close();
}
}
/// <summary> Current version number from segments file.</summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public static long ReadCurrentVersion(Directory directory)
{
// Fully read the segments file: this ensures that it's
// completely written so that if
// IndexWriter.prepareCommit has been called (but not
// yet commit), then the reader will still see itself as
// current:
var sis = new SegmentInfos();
sis.Read(directory);
return(sis.version);
//return (long) ((System.Int64) new AnonymousClassFindSegmentsFile1(directory).Run());
//DIGY: AnonymousClassFindSegmentsFile1 can safely be deleted
}
/// <summary>Returns all file names referenced by SegmentInfo
/// instances matching the provided Directory (ie files
/// associated with any "external" segments are skipped).
/// The returned collection is recomputed on each
/// invocation.
/// </summary>
public System.Collections.Generic.ICollection <string> Files(Directory dir, bool includeSegmentsFile)
{
System.Collections.Generic.HashSet <string> files = new System.Collections.Generic.HashSet <string>();
if (includeSegmentsFile)
{
files.Add(GetCurrentSegmentFileName());
}
int size = Count;
for (int i = 0; i < size; i++)
{
SegmentInfo info = Info(i);
if (info.dir == dir)
{
files.UnionWith(Info(i).Files());
}
}
return(files);
}
public FormatPostingsFieldsWriter(SegmentWriteState state, FieldInfos fieldInfos):base()
{
dir = state.directory;
segment = state.segmentName;
totalNumDocs = state.numDocs;
this.fieldInfos = fieldInfos;
termsOut = new TermInfosWriter(dir, segment, fieldInfos, state.termIndexInterval);
// TODO: this is a nasty abstraction violation (that we
// peek down to find freqOut/proxOut) -- we need a
// better abstraction here whereby these child consumers
// can provide skip data or not
skipListWriter = new DefaultSkipListWriter(termsOut.skipInterval, termsOut.maxSkipLevels, totalNumDocs, null, null);
state.flushedFiles.Add(state.SegmentFileName(IndexFileNames.TERMS_EXTENSION));
state.flushedFiles.Add(state.SegmentFileName(IndexFileNames.TERMS_INDEX_EXTENSION));
termsWriter = new FormatPostingsTermsWriter(state, this);
}
public System.String SegString(Directory directory)
{
lock (this)
{
var buffer = new System.Text.StringBuilder();
int count = Count;
for (int i = 0; i < count; i++)
{
if (i > 0)
{
buffer.Append(' ');
}
SegmentInfo info = Info(i);
buffer.Append(info.SegString(directory));
if (info.dir != directory)
{
buffer.Append("**");
}
}
return(buffer.ToString());
}
}
private void Write(Directory directory)
{
System.String segmentFileName = GetNextSegmentFileName();
// Always advance the generation on write:
if (generation == -1)
{
generation = 1;
}
else
{
generation++;
}
var segnOutput = new ChecksumIndexOutput(directory.CreateOutput(segmentFileName));
bool success = false;
try
{
segnOutput.WriteInt(CURRENT_FORMAT); // write FORMAT
segnOutput.WriteLong(++version); // every write changes
// the index
segnOutput.WriteInt(counter); // write counter
segnOutput.WriteInt(Count); // write infos
for (int i = 0; i < Count; i++)
{
Info(i).Write(segnOutput);
}
segnOutput.WriteStringStringMap(userData);
segnOutput.PrepareCommit();
success = true;
pendingSegnOutput = segnOutput;
}
finally
{
if (!success)
{
// We hit an exception above; try to close the file
// but suppress any exception:
try
{
segnOutput.Close();
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
try
{
// Try not to leave a truncated segments_N file in
// the index:
directory.DeleteFile(segmentFileName);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
}
/// <summary>Create a new CheckIndex on the directory. </summary>
public CheckIndex(Directory dir)
{
this.dir = dir;
infoStream = null;
}
internal virtual void CheckAborted(Directory dir)
{
lock (this)
{
if (aborted)
throw new MergeAbortedException("merge is aborted: " + SegString(dir));
}
}
public virtual String SegString(Directory dir)
{
var b = new System.Text.StringBuilder();
b.Append("MergeSpec:\n");
int count = merges.Count;
for (int i = 0; i < count; i++)
b.Append(" ").Append(1 + i).Append(": ").Append(merges[i].SegString(dir));
return b.ToString();
}
/// <summary> Copy contents of a directory src to a directory dest.
/// If a file in src already exists in dest then the
/// one in dest will be blindly overwritten.
///
/// <p/><b>NOTE:</b> the source directory cannot change
/// while this method is running. Otherwise the results
/// are undefined and you could easily hit a
/// FileNotFoundException.
///
/// <p/><b>NOTE:</b> this method only copies files that look
/// like index files (ie, have extensions matching the
/// known extensions of index files).
///
/// </summary>
/// <param name="src">source directory
/// </param>
/// <param name="dest">destination directory
/// </param>
/// <param name="closeDirSrc">if <c>true</c>, call <see cref="Close()" /> method on source directory
/// </param>
/// <throws> IOException </throws>
public static void Copy(Directory src, Directory dest, bool closeDirSrc)
{
System.String[] files = src.ListAll();
IndexFileNameFilter filter = IndexFileNameFilter.Filter;
byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE];
for (int i = 0; i < files.Length; i++)
{
if (!filter.Accept(null, files[i]))
continue;
IndexOutput os = null;
IndexInput is_Renamed = null;
try
{
// create file in dest directory
os = dest.CreateOutput(files[i]);
// read current file
is_Renamed = src.OpenInput(files[i]);
// and copy to dest directory
long len = is_Renamed.Length();
long readCount = 0;
while (readCount < len)
{
int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len?(int) (len - readCount):BufferedIndexOutput.BUFFER_SIZE;
is_Renamed.ReadBytes(buf, 0, toRead);
os.WriteBytes(buf, toRead);
readCount += toRead;
}
}
finally
{
// graceful cleanup
try
{
if (os != null)
os.Close();
}
finally
{
if (is_Renamed != null)
is_Renamed.Close();
}
}
}
if (closeDirSrc)
src.Close();
}
public static void Main(System.String[] args)
{
bool doFix = false;
var onlySegments = new List<string>();
System.String indexPath = null;
int i = 0;
while (i < args.Length)
{
if (args[i].Equals("-fix"))
{
doFix = true;
i++;
}
else if (args[i].Equals("-segment"))
{
if (i == args.Length - 1)
{
System.Console.Out.WriteLine("ERROR: missing name for -segment option");
System.Environment.Exit(1);
}
onlySegments.Add(args[i + 1]);
i += 2;
}
else
{
if (indexPath != null)
{
System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'");
System.Environment.Exit(1);
}
indexPath = args[i];
i++;
}
}
if (indexPath == null)
{
System.Console.Out.WriteLine("\nERROR: index path not specified");
System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n");
System.Environment.Exit(1);
}
if (!AssertsOn())
System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled");
if (onlySegments.Count == 0)
onlySegments = null;
else if (doFix)
{
System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment");
System.Environment.Exit(1);
}
System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n");
Directory dir = null;
try
{
dir = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath));
}
catch (Exception t)
{
Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting");
Console.Out.WriteLine(t.StackTrace);
Environment.Exit(1);
}
var checker = new CheckIndex(dir);
var tempWriter = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding)
{AutoFlush = true};
checker.SetInfoStream(tempWriter);
Status result = checker.CheckIndex_Renamed_Method(onlySegments);
if (result.missingSegments)
{
System.Environment.Exit(1);
}
if (!result.clean)
{
if (!doFix)
{
System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
}
else
{
Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
for (var s = 0; s < 5; s++)
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
System.Console.Out.WriteLine(" " + (5 - s) + "...");
}
Console.Out.WriteLine("Writing...");
checker.FixIndex(result);
Console.Out.WriteLine("OK");
Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\"");
}
}
System.Console.Out.WriteLine("");
int exitCode;
if (result != null && result.clean == true)
exitCode = 0;
else
exitCode = 1;
System.Environment.Exit(exitCode);
}
public MergeException(System.Exception exc, Directory dir):base(null, exc)
{
this.dir = dir;
}
public CompoundFileReader(Directory dir, System.String name):this(dir, name, BufferedIndexInput.BUFFER_SIZE)
{
}
/// <summary> Get the filename of the current segments_N file
/// in the directory.
///
/// </summary>
/// <param name="directory">-- directory to search for the latest segments_N file
/// </param>
public static System.String GetCurrentSegmentFileName(Directory directory)
{
return(IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", GetCurrentSegmentGeneration(directory)));
}
public static void Main(System.String[] args)
{
bool doFix = false;
var onlySegments = new List <string>();
System.String indexPath = null;
int i = 0;
while (i < args.Length)
{
if (args[i].Equals("-fix"))
{
doFix = true;
i++;
}
else if (args[i].Equals("-segment"))
{
if (i == args.Length - 1)
{
System.Console.Out.WriteLine("ERROR: missing name for -segment option");
System.Environment.Exit(1);
}
onlySegments.Add(args[i + 1]);
i += 2;
}
else
{
if (indexPath != null)
{
System.Console.Out.WriteLine("ERROR: unexpected extra argument '" + args[i] + "'");
System.Environment.Exit(1);
}
indexPath = args[i];
i++;
}
}
if (indexPath == null)
{
System.Console.Out.WriteLine("\nERROR: index path not specified");
System.Console.Out.WriteLine("\nUsage: java Lucene.Net.Index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + " You can't use this with the -fix option\n" + "\n" + "**WARNING**: -fix should only be used on an emergency basis as it will cause\n" + "documents (perhaps many) to be permanently removed from the index. Always make\n" + "a backup copy of your index before running this! Do not run this tool on an index\n" + "that is actively being written to. You have been warned!\n" + "\n" + "Run without -fix, this tool will open the index, report version information\n" + "and report any exceptions it hits and what action it would take if -fix were\n" + "specified. With -fix, this tool will remove any segments that have issues and\n" + "write a new segments_N file. This means all documents contained in the affected\n" + "segments will be removed.\n" + "\n" + "This tool exits with exit code 1 if the index cannot be opened or has any\n" + "corruption, else 0.\n");
System.Environment.Exit(1);
}
if (!AssertsOn())
{
System.Console.Out.WriteLine("\nNOTE: testing will be more thorough if you run java with '-ea:Lucene.Net...', so assertions are enabled");
}
if (onlySegments.Count == 0)
{
onlySegments = null;
}
else if (doFix)
{
System.Console.Out.WriteLine("ERROR: cannot specify both -fix and -segment");
System.Environment.Exit(1);
}
System.Console.Out.WriteLine("\nOpening index @ " + indexPath + "\n");
Directory dir = null;
try
{
dir = FSDirectory.Open(new System.IO.DirectoryInfo(indexPath));
}
catch (Exception t)
{
Console.Out.WriteLine("ERROR: could not open directory \"" + indexPath + "\"; exiting");
Console.Out.WriteLine(t.StackTrace);
Environment.Exit(1);
}
var checker = new CheckIndex(dir);
var tempWriter = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding)
{
AutoFlush = true
};
checker.SetInfoStream(tempWriter);
Status result = checker.CheckIndex_Renamed_Method(onlySegments);
if (result.missingSegments)
{
System.Environment.Exit(1);
}
if (!result.clean)
{
if (!doFix)
{
System.Console.Out.WriteLine("WARNING: would write new segments file, and " + result.totLoseDocCount + " documents would be lost, if -fix were specified\n");
}
else
{
Console.Out.WriteLine("WARNING: " + result.totLoseDocCount + " documents will be lost\n");
Console.Out.WriteLine("NOTE: will write new segments file in 5 seconds; this will remove " + result.totLoseDocCount + " docs from the index. THIS IS YOUR LAST CHANCE TO CTRL+C!");
for (var s = 0; s < 5; s++)
{
System.Threading.Thread.Sleep(new System.TimeSpan((System.Int64) 10000 * 1000));
System.Console.Out.WriteLine(" " + (5 - s) + "...");
}
Console.Out.WriteLine("Writing...");
checker.FixIndex(result);
Console.Out.WriteLine("OK");
Console.Out.WriteLine("Wrote new segments file \"" + result.newSegments.GetCurrentSegmentFileName() + "\"");
}
}
System.Console.Out.WriteLine("");
int exitCode;
if (result != null && result.clean == true)
{
exitCode = 0;
}
else
{
exitCode = 1;
}
System.Environment.Exit(exitCode);
}
protected FindSegmentsFile(Directory directory)
{
this.directory = directory;
}
/// <summary> Returns userData from latest segments file</summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public static System.Collections.Generic.IDictionary <string, string> ReadCurrentUserData(Directory directory)
{
var sis = new SegmentInfos();
sis.Read(directory);
return(sis.UserData);
}
/// <summary> Read a particular segmentFileName. Note that this may
/// throw an IOException if a commit is in process.
///
/// </summary>
/// <param name="directory">-- directory containing the segments file
/// </param>
/// <param name="segmentFileName">-- segment file to load
/// </param>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public void Read(Directory directory, System.String segmentFileName)
{
bool success = false;
// Clear any previous segments:
Clear();
var input = new ChecksumIndexInput(directory.OpenInput(segmentFileName));
generation = GenerationFromSegmentsFileName(segmentFileName);
lastGeneration = generation;
try
{
int format = input.ReadInt();
if (format < 0)
{
// file contains explicit format info
// check that it is a format we can understand
if (format < CURRENT_FORMAT)
{
throw new CorruptIndexException("Unknown format version: " + format);
}
version = input.ReadLong(); // read version
counter = input.ReadInt(); // read counter
}
else
{
// file is in old format without explicit format info
counter = format;
}
for (int i = input.ReadInt(); i > 0; i--)
{
// read segmentInfos
Add(new SegmentInfo(directory, format, input));
}
if (format >= 0)
{
// in old format the version number may be at the end of the file
if (input.FilePointer >= input.Length())
{
version = (DateTime.UtcNow.Ticks / TimeSpan.TicksPerMillisecond);
}
// old file format without version number
else
{
version = input.ReadLong(); // read version
}
}
if (format <= FORMAT_USER_DATA)
{
if (format <= FORMAT_DIAGNOSTICS)
{
userData = input.ReadStringStringMap();
}
else if (0 != input.ReadByte())
{
// TODO: Should be read-only map
userData = new HashMap <string, string> {
{ "userData", input.ReadString() }
};
}
else
{
// TODO: Should be empty read-only map
userData = new HashMap <string, string>();
}
}
else
{
// TODO: Should be empty read-only map
userData = new HashMap <string, string>();
}
if (format <= FORMAT_CHECKSUM)
{
long checksumNow = input.Checksum;
long checksumThen = input.ReadLong();
if (checksumNow != checksumThen)
{
throw new CorruptIndexException("checksum mismatch in segments file");
}
}
success = true;
}
finally
{
input.Close();
if (!success)
{
// Clear any segment infos we had loaded so we
// have a clean slate on retry:
Clear();
}
}
}
/// <summary> Creates a new <c>RAMDirectory</c> instance from a different
/// <c>Directory</c> implementation. This can be used to load
/// a disk-based index into memory.
/// <p/>
/// This should be used only with indices that can fit into memory.
/// <p/>
/// Note that the resulting <c>RAMDirectory</c> instance is fully
/// independent from the original <c>Directory</c> (it is a
/// complete copy). Any subsequent changes to the
/// original <c>Directory</c> will not be visible in the
/// <c>RAMDirectory</c> instance.
///
/// </summary>
/// <param name="dir">a <c>Directory</c> value
/// </param>
/// <exception cref="System.IO.IOException">if an error occurs
/// </exception>
public RAMDirectory(Directory dir):this(dir, false)
{
}
/// <summary> Expert: constructs an IndexWriter with a custom <see cref="IndexDeletionPolicy" />
///, for the index in <c>d</c>.
/// Text will be analyzed with <c>a</c>. If
/// <c>create</c> is true, then a new, empty index
/// will be created in <c>d</c>, replacing the index
/// already there, if any.
///
/// </summary>
/// <param name="d">the index directory
/// </param>
/// <param name="a">the analyzer to use
/// </param>
/// <param name="create"><c>true</c> to create the index or overwrite
/// the existing one; <c>false</c> to append to the existing
/// index
/// </param>
/// <param name="deletionPolicy">see <a href="#deletionPolicy">above</a>
/// </param>
/// <param name="mfl"><see cref="Lucene.Net.Index.IndexWriter.MaxFieldLength" />, whether or not to limit field lengths. Value is in number of terms/tokens
/// </param>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> LockObtainFailedException if another writer </throws>
/// <summary> has this index open (<c>write.lock</c> could not
/// be obtained)
/// </summary>
/// <throws> IOException if the directory cannot be read/written to, or </throws>
/// <summary> if it does not exist and <c>create</c> is
/// <c>false</c> or if there is any other low-level
/// IO error
/// </summary>
public IndexWriter(Directory d, Analyzer a, bool create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl)
{
InitBlock();
Init(d, a, create, deletionPolicy, mfl.Limit, null, null);
}
private RAMDirectory(Directory dir, bool closeDir):this()
{
Directory.Copy(dir, this, closeDir);
}
/// <summary> Expert: constructs an IndexWriter on specific commit
/// point, with a custom <see cref="IndexDeletionPolicy" />, for
/// the index in <c>d</c>. Text will be analyzed
/// with <c>a</c>.
///
/// <p/> This is only meaningful if you've used a <see cref="IndexDeletionPolicy" />
/// in that past that keeps more than
/// just the last commit.
///
/// <p/>This operation is similar to <see cref="Rollback()" />,
/// except that method can only rollback what's been done
/// with the current instance of IndexWriter since its last
/// commit, whereas this method can rollback to an
/// arbitrary commit point from the past, assuming the
/// <see cref="IndexDeletionPolicy" /> has preserved past
/// commits.
///
/// </summary>
/// <param name="d">the index directory
/// </param>
/// <param name="a">the analyzer to use
/// </param>
/// <param name="deletionPolicy">see <a href="#deletionPolicy">above</a>
/// </param>
/// <param name="mfl">whether or not to limit field lengths, value is in number of terms/tokens. See <see cref="Lucene.Net.Index.IndexWriter.MaxFieldLength" />.
/// </param>
/// <param name="commit">which commit to open
/// </param>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> LockObtainFailedException if another writer </throws>
/// <summary> has this index open (<c>write.lock</c> could not
/// be obtained)
/// </summary>
/// <throws> IOException if the directory cannot be read/written to, or </throws>
/// <summary> if it does not exist and <c>create</c> is
/// <c>false</c> or if there is any other low-level
/// IO error
/// </summary>
public IndexWriter(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexCommit commit)
{
InitBlock();
Init(d, a, false, deletionPolicy, mfl.Limit, null, commit);
}
internal FieldsReader(Directory d, System.String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size)
{
bool success = false;
isOriginal = true;
try
{
fieldInfos = fn;
cloneableFieldsStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_EXTENSION, readBufferSize);
cloneableIndexStream = d.OpenInput(segment + "." + IndexFileNames.FIELDS_INDEX_EXTENSION, readBufferSize);
// First version of fdx did not include a format
// header, but, the first int will always be 0 in that
// case
int firstInt = cloneableIndexStream.ReadInt();
format = firstInt == 0 ? 0 : firstInt;
if (format > FieldsWriter.FORMAT_CURRENT)
{
throw new CorruptIndexException("Incompatible format version: " + format + " expected " + FieldsWriter.FORMAT_CURRENT + " or lower");
}
formatSize = format > FieldsWriter.FORMAT ? 4 : 0;
if (format < FieldsWriter.FORMAT_VERSION_UTF8_LENGTH_IN_BYTES)
{
cloneableFieldsStream.SetModifiedUTF8StringsMode();
}
fieldsStream = (IndexInput)cloneableFieldsStream.Clone();
long indexSize = cloneableIndexStream.Length() - formatSize;
if (docStoreOffset != -1)
{
// We read only a slice out of this shared fields file
this.docStoreOffset = docStoreOffset;
this.size = size;
// Verify the file is long enough to hold all of our
// docs
System.Diagnostics.Debug.Assert(((int)(indexSize / 8)) >= size + this.docStoreOffset, "indexSize=" + indexSize + " size=" + size + " docStoreOffset=" + docStoreOffset);
}
else
{
this.docStoreOffset = 0;
this.size = (int)(indexSize >> 3);
}
indexStream = (IndexInput)cloneableIndexStream.Clone();
numTotalDocs = (int)(indexSize >> 3);
success = true;
}
finally
{
// With lock-less commits, it's entirely possible (and
// fine) to hit a FileNotFound exception above. In
// this case, we want to explicitly close any subset
// of things that were opened so that we don't have to
// wait for a GC to do so.
if (!success)
{
Dispose();
}
}
}
private void Init(Directory d, Analyzer a, bool create, IndexDeletionPolicy deletionPolicy, int maxFieldLength, DocumentsWriter.IndexingChain indexingChain, IndexCommit commit)
{
directory = d;
analyzer = a;
SetMessageID(defaultInfoStream);
this.maxFieldLength = maxFieldLength;
if (indexingChain == null)
indexingChain = DocumentsWriter.DefaultIndexingChain;
if (create)
{
// Clear the write lock in case it's leftover:
directory.ClearLock(WRITE_LOCK_NAME);
}
Lock writeLock = directory.MakeLock(WRITE_LOCK_NAME);
if (!writeLock.Obtain(writeLockTimeout))
// obtain write lock
{
throw new LockObtainFailedException("Index locked for write: " + writeLock);
}
this.writeLock = writeLock; // save it
bool success = false;
try
{
if (create)
{
// Try to read first. This is to allow create
// against an index that's currently open for
// searching. In this case we write the next
// segments_N file with no segments:
bool doCommit;
try
{
segmentInfos.Read(directory);
segmentInfos.Clear();
doCommit = false;
}
catch (System.IO.IOException)
{
// Likely this means it's a fresh directory
doCommit = true;
}
if (doCommit)
{
// Only commit if there is no segments file
// in this dir already.
segmentInfos.Commit(directory);
synced.UnionWith(segmentInfos.Files(directory, true));
}
else
{
// Record that we have a change (zero out all
// segments) pending:
changeCount++;
}
}
else
{
segmentInfos.Read(directory);
if (commit != null)
{
// Swap out all segments, but, keep metadata in
// SegmentInfos, like version & generation, to
// preserve write-once. This is important if
// readers are open against the future commit
// points.
if (commit.Directory != directory)
throw new System.ArgumentException("IndexCommit's directory doesn't match my directory");
SegmentInfos oldInfos = new SegmentInfos();
oldInfos.Read(directory, commit.SegmentsFileName);
segmentInfos.Replace(oldInfos);
changeCount++;
if (infoStream != null)
Message("init: loaded commit \"" + commit.SegmentsFileName + "\"");
}
// We assume that this segments_N was previously
// properly sync'd:
synced.UnionWith(segmentInfos.Files(directory, true));
}
SetRollbackSegmentInfos(segmentInfos);
docWriter = new DocumentsWriter(directory, this, indexingChain);
docWriter.SetInfoStream(infoStream);
docWriter.SetMaxFieldLength(maxFieldLength);
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
deleter = new IndexFileDeleter(directory, deletionPolicy == null?new KeepOnlyLastCommitDeletionPolicy():deletionPolicy, segmentInfos, infoStream, docWriter, synced);
if (deleter.startingCommitDeleted)
// Deletion policy deleted the "head" commit point.
// We have to mark ourself as changed so that if we
// are closed w/o any further changes we write a new
// segments_N file.
changeCount++;
PushMaxBufferedDocs();
if (infoStream != null)
{
Message("init: create=" + create);
MessageState();
}
success = true;
}
finally
{
if (!success)
{
if (infoStream != null)
{
Message("init: hit exception on init; releasing write lock");
}
try
{
writeLock.Release();
}
catch (Exception)
{
// don't mask the original exception
}
writeLock = null;
}
}
}
/// <summary>Create a new CheckIndex on the directory. </summary>
public CheckIndex(Directory dir)
{
this.dir = dir;
infoStream = null;
}
/// <summary> Returns <c>true</c> iff the index in the named directory is
/// currently locked.
/// </summary>
/// <param name="directory">the directory to check for a lock
/// </param>
/// <throws> IOException if there is a low-level IO error </throws>
public static bool IsLocked(Directory directory)
{
return directory.MakeLock(WRITE_LOCK_NAME).IsLocked();
}
internal DocumentsWriter(Directory directory, IndexWriter writer, IndexingChain indexingChain)
{
InitBlock();
this.directory = directory;
this.writer = writer;
this.similarity = writer.Similarity;
flushedDocCount = writer.MaxDoc();
consumer = indexingChain.GetChain(this);
if (consumer is DocFieldProcessor)
{
docFieldProcessor = (DocFieldProcessor) consumer;
}
}
internal FieldsReader(Directory d, System.String segment, FieldInfos fn, int readBufferSize) : this(d, segment, fn, readBufferSize, -1, 0)
{
}
internal virtual String SegString(Directory dir)
{
var b = new System.Text.StringBuilder();
int numSegments = segments.Count;
for (int i = 0; i < numSegments; i++)
{
if (i > 0)
b.Append(' ');
b.Append(segments.Info(i).SegString(dir));
}
if (info != null)
b.Append(" into ").Append(info.name);
if (optimize)
b.Append(" [optimize]");
if (mergeDocStores)
{
b.Append(" [mergeDocStores]");
}
return b.ToString();
}
internal ReadOnlyDirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, System.Collections.Generic.IDictionary<string, byte[]> oldNormsCache, bool doClone, int termInfosIndexDivisor)
: base(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor)
{
}
public MergeException(System.String message, Directory dir):base(message)
{
this.dir = dir;
}
/// <summary> This version of read uses the retry logic (for lock-less
/// commits) to find the right segments file to load.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public void Read(Directory directory)
{
generation = lastGeneration = -1;
new AnonymousClassFindSegmentsFile(this, directory).Run();
}
/// <summary> Constructs an IndexWriter for the index in
/// <c>d</c>, first creating it if it does not
/// already exist.
///
/// </summary>
/// <param name="d">the index directory
/// </param>
/// <param name="a">the analyzer to use
/// </param>
/// <param name="mfl">Maximum field length in number of terms/tokens: LIMITED, UNLIMITED, or user-specified
/// via the MaxFieldLength constructor.
/// </param>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> LockObtainFailedException if another writer </throws>
/// <summary> has this index open (<c>write.lock</c> could not
/// be obtained)
/// </summary>
/// <throws> IOException if the directory cannot be </throws>
/// <summary> read/written to or if there is any other low-level
/// IO error
/// </summary>
public IndexWriter(Directory d, Analyzer a, MaxFieldLength mfl)
{
InitBlock();
Init(d, a, null, mfl.Limit, null, null);
}
internal void FinishCommit(Directory dir)
{
if (pendingSegnOutput == null)
{
throw new System.SystemException("prepareCommit was not called");
}
bool success = false;
try
{
pendingSegnOutput.FinishCommit();
pendingSegnOutput.Close();
pendingSegnOutput = null;
success = true;
}
finally
{
if (!success)
{
RollbackCommit(dir);
}
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
System.String fileName = IndexFileNames.FileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try
{
dir.Sync(fileName);
success = true;
}
finally
{
if (!success)
{
try
{
dir.DeleteFile(fileName);
}
catch (System.Exception)
{
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try
{
IndexOutput genOutput = dir.CreateOutput(IndexFileNames.SEGMENTS_GEN);
try
{
genOutput.WriteInt(FORMAT_LOCKLESS);
genOutput.WriteLong(generation);
genOutput.WriteLong(generation);
}
finally
{
genOutput.Close();
}
}
catch (System.Exception)
{
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
/// <summary> Expert: constructs an IndexWriter with a custom <see cref="IndexDeletionPolicy" />
/// and <see cref="DocumentsWriter.IndexingChain" />,
/// for the index in <c>d</c>.
/// Text will be analyzed with <c>a</c>. If
/// <c>create</c> is true, then a new, empty index
/// will be created in <c>d</c>, replacing the index
/// already there, if any.
///
/// </summary>
/// <param name="d">the index directory
/// </param>
/// <param name="a">the analyzer to use
/// </param>
/// <param name="create"><c>true</c> to create the index or overwrite
/// the existing one; <c>false</c> to append to the existing
/// index
/// </param>
/// <param name="deletionPolicy">see <a href="#deletionPolicy">above</a>
/// </param>
/// <param name="mfl">whether or not to limit field lengths, value is in number of terms/tokens. See <see cref="Lucene.Net.Index.IndexWriter.MaxFieldLength" />.
/// </param>
/// <param name="indexingChain">the <see cref="DocConsumer" /> chain to be used to
/// process documents
/// </param>
/// <param name="commit">which commit to open
/// </param>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> LockObtainFailedException if another writer </throws>
/// <summary> has this index open (<c>write.lock</c> could not
/// be obtained)
/// </summary>
/// <throws> IOException if the directory cannot be read/written to, or </throws>
/// <summary> if it does not exist and <c>create</c> is
/// <c>false</c> or if there is any other low-level
/// IO error
/// </summary>
internal IndexWriter(Directory d, Analyzer a, bool create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, DocumentsWriter.IndexingChain indexingChain, IndexCommit commit)
{
InitBlock();
Init(d, a, create, deletionPolicy, mfl.Limit, indexingChain, commit);
}
/// <summary>Writes & syncs to the Directory dir, taking care to
/// remove the segments file on exception
/// </summary>
public /*internal*/ void Commit(Directory dir)
{
PrepareCommit(dir);
FinishCommit(dir);
}
private void Init(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, int maxFieldLength, DocumentsWriter.IndexingChain indexingChain, IndexCommit commit)
{
if (IndexReader.IndexExists(d))
{
Init(d, a, false, deletionPolicy, maxFieldLength, indexingChain, commit);
}
else
{
Init(d, a, true, deletionPolicy, maxFieldLength, indexingChain, commit);
}
}
public CheckAbort(MergePolicy.OneMerge merge, Directory dir)
{
this.merge = merge;
this.dir = dir;
}
private void NoDupDirs(Directory[] dirs)
{
HashSet<Directory> dups = new HashSet<Directory>();
for (int i = 0; i < dirs.Length; i++)
{
if (dups.Contains(dirs[i]))
{
throw new System.ArgumentException("Directory " + dirs[i] + " appears more than once");
}
if (dirs[i] == directory)
throw new System.ArgumentException("Cannot add directory to itself");
dups.Add(dirs[i]);
}
}
internal ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor):base(directory, sis, deletionPolicy, true, termInfosIndexDivisor)
{
}
/// <summary> Forcibly unlocks the index in the named directory.
/// <p/>
/// Caution: this should only be used by failure recovery code,
/// when it is known that no other process nor thread is in fact
/// currently accessing this index.
/// </summary>
public static void Unlock(Directory directory)
{
directory.MakeLock(IndexWriter.WRITE_LOCK_NAME).Release();
}
public /*internal*/ FieldsReader(Directory d, String segment, FieldInfos fn) : this(d, segment, fn, BufferedIndexInput.BUFFER_SIZE, -1, 0)
{
}
