internal virtual System.Collections.ArrayList Files()
{
System.Collections.ArrayList files = System.Collections.ArrayList.Synchronized(new System.Collections.ArrayList(16));
if (si.GetUseCompoundFile())
{
System.String name = segment + ".cfs";
if (Directory().FileExists(name))
{
files.Add(name);
}
}
else
{
for (int i = 0; i < IndexFileNames.INDEX_EXTENSIONS.Length; i++)
{
System.String name = segment + "." + IndexFileNames.INDEX_EXTENSIONS[i];
if (Directory().FileExists(name))
{
files.Add(name);
}
}
}
if (si.HasDeletions())
{
files.Add(si.GetDelFileName());
}
bool addedNrm = false;
for (int i = 0; i < fieldInfos.Size(); i++)
{
System.String name = si.GetNormFileName(i);
if (name != null && Directory().FileExists(name))
{
if (name.EndsWith("." + IndexFileNames.NORMS_EXTENSION))
{
if (addedNrm)
{
continue; // add .nrm just once
}
addedNrm = true;
}
files.Add(name);
}
}
return(files);
}
internal void ReWrite(SegmentInfo si)
{
// NOTE: norms are re-written in regular directory, not cfs
System.String oldFileName = si.GetNormFileName(this.number);
if (oldFileName != null && !oldFileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION))
{
// Mark this file for deletion. Note that we don't
// actually try to delete it until the new segments files is
// successfully written:
Enclosing_Instance.deleter.AddPendingFile(oldFileName);
}
si.AdvanceNormGen(this.number);
IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number));
try
{
out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc());
}
finally
{
out_Renamed.Close();
}
this.dirty = false;
}
private void OpenNorms(Directory cfsDir, int readBufferSize)
{
long nextNormSeek = SegmentMerger.NORMS_HEADER.Length; //skip header (header unused for now)
int maxDoc = MaxDoc();
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (norms.Contains(fi.name))
{
// in case this SegmentReader is being re-opened, we might be able to
// reuse some norm instances and skip loading them here
continue;
}
if (fi.isIndexed && !fi.omitNorms)
{
Directory d = Directory();
System.String fileName = si.GetNormFileName(fi.number);
if (!si.HasSeparateNorms(fi.number))
{
d = cfsDir;
}
// singleNormFile means multiple norms share this file
bool singleNormFile = fileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION);
IndexInput normInput = null;
long normSeek;
if (singleNormFile)
{
normSeek = nextNormSeek;
if (singleNormStream == null)
{
singleNormStream = d.OpenInput(fileName, readBufferSize);
}
// All norms in the .nrm file can share a single IndexInput since
// they are only used in a synchronized context.
// If this were to change in the future, a clone could be done here.
normInput = singleNormStream;
}
else
{
normSeek = 0;
normInput = d.OpenInput(fileName);
}
norms[fi.name] = new Norm(this, normInput, singleNormFile, fi.number, normSeek);
nextNormSeek += maxDoc; // increment also if some norms are separate
}
}
}
internal void ReWrite(SegmentInfo si)
{
// NOTE: norms are re-written in regular directory, not cfs
System.String oldFileName = si.GetNormFileName(this.number);
if (oldFileName != null && !oldFileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION))
{
// Mark this file for deletion. Note that we don't
// actually try to delete it until the new segments files is
// successfully written:
Enclosing_Instance.deleter.AddPendingFile(oldFileName);
}
si.AdvanceNormGen(this.number);
IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number));
try
{
out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc());
}
finally
{
out_Renamed.Close();
}
this.dirty = false;
}
internal virtual SegmentReader ReopenSegment(SegmentInfo si, bool doClone, bool openReadOnly)
{
lock (this)
{
bool deletionsUpToDate = (this.si.HasDeletions() == si.HasDeletions()) && (!si.HasDeletions() || this.si.GetDelFileName().Equals(si.GetDelFileName()));
bool normsUpToDate = true;
bool[] fieldNormsChanged = new bool[core.fieldInfos.Size()];
int fieldCount = core.fieldInfos.Size();
for (int i = 0; i < fieldCount; i++)
{
if (!this.si.GetNormFileName(i).Equals(si.GetNormFileName(i)))
{
normsUpToDate = false;
fieldNormsChanged[i] = true;
}
}
// if we're cloning we need to run through the reopenSegment logic
// also if both old and new readers aren't readonly, we clone to avoid sharing modifications
if (normsUpToDate && deletionsUpToDate && !doClone && openReadOnly && readOnly)
{
return this;
}
// When cloning, the incoming SegmentInfos should not
// have any changes in it:
System.Diagnostics.Debug.Assert(!doClone ||(normsUpToDate && deletionsUpToDate));
// clone reader
SegmentReader clone;
try
{
if (openReadOnly)
clone = (SegmentReader) System.Activator.CreateInstance(READONLY_IMPL);
else
clone = (SegmentReader) System.Activator.CreateInstance(IMPL);
}
catch (System.Exception e)
{
throw new System.SystemException("cannot load SegmentReader class: " + e, e);
}
bool success = false;
try
{
core.IncRef();
clone.core = core;
clone.readOnly = openReadOnly;
clone.si = si;
clone.readBufferSize = readBufferSize;
if (!openReadOnly && hasChanges)
{
// My pending changes transfer to the new reader
clone.pendingDeleteCount = pendingDeleteCount;
clone.deletedDocsDirty = deletedDocsDirty;
clone.normsDirty = normsDirty;
clone.hasChanges = hasChanges;
hasChanges = false;
}
if (doClone)
{
if (deletedDocs != null)
{
deletedDocsRef.IncRef();
clone.deletedDocs = deletedDocs;
clone.deletedDocsRef = deletedDocsRef;
}
}
else
{
if (!deletionsUpToDate)
{
// load deleted docs
System.Diagnostics.Debug.Assert(clone.deletedDocs == null);
clone.LoadDeletedDocs();
}
else if (deletedDocs != null)
{
deletedDocsRef.IncRef();
clone.deletedDocs = deletedDocs;
clone.deletedDocsRef = deletedDocsRef;
}
}
clone.SetDisableFakeNorms(GetDisableFakeNorms());
clone.norms = new System.Collections.Hashtable();
// Clone norms
for (int i = 0; i < fieldNormsChanged.Length; i++)
{
// Clone unchanged norms to the cloned reader
if (doClone || !fieldNormsChanged[i])
{
System.String curField = core.fieldInfos.FieldInfo(i).name;
Norm norm = (Norm) this.norms[curField];
if (norm != null)
clone.norms[curField] = norm.Clone();
}
}
// If we are not cloning, then this will open anew
// any norms that have changed:
clone.OpenNorms(si.GetUseCompoundFile()?core.GetCFSReader():Directory(), readBufferSize);
success = true;
}
finally
{
if (!success)
{
// An exception occured during reopen, we have to decRef the norms
// that we incRef'ed already and close singleNormsStream and FieldsReader
clone.DecRef();
}
}
return clone;
}
}
// Flush all pending changes to the next generation
// separate norms file.
public void ReWrite(SegmentInfo si)
{
System.Diagnostics.Debug.Assert(refCount > 0 && (origNorm == null || origNorm.refCount > 0), "refCount=" + refCount + " origNorm=" + origNorm);
// NOTE: norms are re-written in regular directory, not cfs
si.AdvanceNormGen(this.number);
IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number));
try
{
out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc());
}
finally
{
out_Renamed.Close();
}
this.dirty = false;
}
internal virtual SegmentReader ReopenSegment(SegmentInfo si)
{
lock (this)
{
bool deletionsUpToDate = (this.si.HasDeletions() == si.HasDeletions()) && (!si.HasDeletions() || this.si.GetDelFileName().Equals(si.GetDelFileName()));
bool normsUpToDate = true;
bool[] fieldNormsChanged = new bool[fieldInfos.Size()];
if (normsUpToDate)
{
for (int i = 0; i < fieldInfos.Size(); i++)
{
if (!this.si.GetNormFileName(i).Equals(si.GetNormFileName(i)))
{
normsUpToDate = false;
fieldNormsChanged[i] = true;
}
}
}
if (normsUpToDate && deletionsUpToDate)
{
return(this);
}
// clone reader
SegmentReader clone;
if (readOnly)
{
clone = new ReadOnlySegmentReader();
}
else
{
clone = new SegmentReader();
}
bool success = false;
try
{
clone.readOnly = readOnly;
clone.directory = directory;
clone.si = si;
clone.segment = segment;
clone.readBufferSize = readBufferSize;
clone.cfsReader = cfsReader;
clone.storeCFSReader = storeCFSReader;
clone.fieldInfos = fieldInfos;
clone.tis = tis;
clone.freqStream = freqStream;
clone.proxStream = proxStream;
clone.termVectorsReaderOrig = termVectorsReaderOrig;
// we have to open a new FieldsReader, because it is not thread-safe
// and can thus not be shared among multiple SegmentReaders
// TODO: Change this in case FieldsReader becomes thread-safe in the future
System.String fieldsSegment;
Directory storeDir = Directory();
if (si.GetDocStoreOffset() != -1)
{
fieldsSegment = si.GetDocStoreSegment();
if (storeCFSReader != null)
{
storeDir = storeCFSReader;
}
}
else
{
fieldsSegment = segment;
if (cfsReader != null)
{
storeDir = cfsReader;
}
}
if (fieldsReader != null)
{
clone.fieldsReader = new FieldsReader(storeDir, fieldsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
}
if (!deletionsUpToDate)
{
// load deleted docs
clone.deletedDocs = null;
clone.LoadDeletedDocs();
}
else
{
clone.deletedDocs = this.deletedDocs;
}
clone.norms = new System.Collections.Hashtable();
if (!normsUpToDate)
{
// load norms
for (int i = 0; i < fieldNormsChanged.Length; i++)
{
// copy unchanged norms to the cloned reader and incRef those norms
if (!fieldNormsChanged[i])
{
System.String curField = fieldInfos.FieldInfo(i).name;
Norm norm = (Norm)this.norms[curField];
norm.IncRef();
clone.norms[curField] = norm;
}
}
clone.OpenNorms(si.GetUseCompoundFile() ? cfsReader : Directory(), readBufferSize);
}
else
{
System.Collections.IEnumerator it = norms.Keys.GetEnumerator();
while (it.MoveNext())
{
System.String field = (System.String)it.Current;
Norm norm = (Norm)norms[field];
norm.IncRef();
clone.norms[field] = norm;
}
}
if (clone.singleNormStream == null)
{
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.isIndexed && !fi.omitNorms)
{
Directory d = si.GetUseCompoundFile() ? cfsReader : Directory();
System.String fileName = si.GetNormFileName(fi.number);
if (si.HasSeparateNorms(fi.number))
{
continue;
}
if (fileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION))
{
clone.singleNormStream = d.OpenInput(fileName, readBufferSize);
break;
}
}
}
}
success = true;
}
finally
{
if (this.referencedSegmentReader != null)
{
// this reader shares resources with another SegmentReader,
// so we increment the other readers refCount. We don't
// increment the refCount of the norms because we did
// that already for the shared norms
clone.referencedSegmentReader = this.referencedSegmentReader;
referencedSegmentReader.IncRefReaderNotNorms();
}
else
{
// this reader wasn't reopened, so we increment this
// readers refCount
clone.referencedSegmentReader = this;
IncRefReaderNotNorms();
}
if (!success)
{
// An exception occured during reopen, we have to decRef the norms
// that we incRef'ed already and close singleNormsStream and FieldsReader
clone.DecRef();
}
}
return(clone);
}
}
internal void ReWrite(SegmentInfo si)
{
// NOTE: norms are re-written in regular directory, not cfs
si.AdvanceNormGen(this.number);
IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number));
try
{
out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc());
}
finally
{
out_Renamed.Close();
}
this.dirty = false;
}
// Flush all pending changes to the next generation
// separate norms file.
public void ReWrite(SegmentInfo si)
{
System.Diagnostics.Debug.Assert(refCount > 0 && (origNorm == null || origNorm.refCount > 0), "refCount=" + refCount + " origNorm=" + origNorm);
// NOTE: norms are re-written in regular directory, not cfs
si.AdvanceNormGen(this.number);
string normFileName = si.GetNormFileName(this.number);
IndexOutput @out = enclosingInstance.Directory().CreateOutput(normFileName);
bool success = false;
try
{
try {
@out.WriteBytes(bytes, enclosingInstance.MaxDoc());
} finally {
@out.Close();
}
success = true;
}
finally
{
if (!success)
{
try
{
enclosingInstance.Directory().DeleteFile(normFileName);
}
catch (Exception t)
{
// suppress this so we keep throwing the
// original exception
}
}
}
this.dirty = false;
}
internal virtual SegmentReader ReopenSegment(SegmentInfo si)
{
lock (this)
{
bool deletionsUpToDate = (this.si.HasDeletions() == si.HasDeletions()) && (!si.HasDeletions() || this.si.GetDelFileName().Equals(si.GetDelFileName()));
bool normsUpToDate = true;
bool[] fieldNormsChanged = new bool[fieldInfos.Size()];
if (normsUpToDate)
{
for (int i = 0; i < fieldInfos.Size(); i++)
{
if (!this.si.GetNormFileName(i).Equals(si.GetNormFileName(i)))
{
normsUpToDate = false;
fieldNormsChanged[i] = true;
}
}
}
if (normsUpToDate && deletionsUpToDate)
{
return this;
}
// clone reader
SegmentReader clone;
if (readOnly)
clone = new ReadOnlySegmentReader();
else
clone = new SegmentReader();
bool success = false;
try
{
clone.readOnly = readOnly;
clone.directory = directory;
clone.si = si;
clone.segment = segment;
clone.readBufferSize = readBufferSize;
clone.cfsReader = cfsReader;
clone.storeCFSReader = storeCFSReader;
clone.fieldInfos = fieldInfos;
clone.tis = tis;
clone.freqStream = freqStream;
clone.proxStream = proxStream;
clone.termVectorsReaderOrig = termVectorsReaderOrig;
// we have to open a new FieldsReader, because it is not thread-safe
// and can thus not be shared among multiple SegmentReaders
// TODO: Change this in case FieldsReader becomes thread-safe in the future
System.String fieldsSegment;
Directory storeDir = Directory();
if (si.GetDocStoreOffset() != - 1)
{
fieldsSegment = si.GetDocStoreSegment();
if (storeCFSReader != null)
{
storeDir = storeCFSReader;
}
}
else
{
fieldsSegment = segment;
if (cfsReader != null)
{
storeDir = cfsReader;
}
}
if (fieldsReader != null)
{
clone.fieldsReader = new FieldsReader(storeDir, fieldsSegment, fieldInfos, readBufferSize, si.GetDocStoreOffset(), si.docCount);
}
if (!deletionsUpToDate)
{
// load deleted docs
clone.deletedDocs = null;
clone.LoadDeletedDocs();
}
else
{
clone.deletedDocs = this.deletedDocs;
}
clone.norms = new System.Collections.Hashtable();
if (!normsUpToDate)
{
// load norms
for (int i = 0; i < fieldNormsChanged.Length; i++)
{
// copy unchanged norms to the cloned reader and incRef those norms
if (!fieldNormsChanged[i])
{
System.String curField = fieldInfos.FieldInfo(i).name;
Norm norm = (Norm) this.norms[curField];
norm.IncRef();
clone.norms[curField] = norm;
}
}
clone.OpenNorms(si.GetUseCompoundFile() ? cfsReader : Directory(), readBufferSize);
}
else
{
System.Collections.IEnumerator it = norms.Keys.GetEnumerator();
while (it.MoveNext())
{
System.String field = (System.String) it.Current;
Norm norm = (Norm) norms[field];
norm.IncRef();
clone.norms[field] = norm;
}
}
if (clone.singleNormStream == null)
{
for (int i = 0; i < fieldInfos.Size(); i++)
{
FieldInfo fi = fieldInfos.FieldInfo(i);
if (fi.isIndexed && !fi.omitNorms)
{
Directory d = si.GetUseCompoundFile() ? cfsReader : Directory();
System.String fileName = si.GetNormFileName(fi.number);
if (si.HasSeparateNorms(fi.number))
{
continue;
}
if (fileName.EndsWith("." + IndexFileNames.NORMS_EXTENSION))
{
clone.singleNormStream = d.OpenInput(fileName, readBufferSize);
break;
}
}
}
}
success = true;
}
finally
{
if (this.referencedSegmentReader != null)
{
// this reader shares resources with another SegmentReader,
// so we increment the other readers refCount. We don't
// increment the refCount of the norms because we did
// that already for the shared norms
clone.referencedSegmentReader = this.referencedSegmentReader;
referencedSegmentReader.IncRefReaderNotNorms();
}
else
{
// this reader wasn't reopened, so we increment this
// readers refCount
clone.referencedSegmentReader = this;
IncRefReaderNotNorms();
}
if (!success)
{
// An exception occured during reopen, we have to decRef the norms
// that we incRef'ed already and close singleNormsStream and FieldsReader
clone.DecRef();
}
}
return clone;
}
}
internal void ReWrite(SegmentInfo si)
{
// NOTE: norms are re-written in regular directory, not cfs
si.AdvanceNormGen(this.number);
IndexOutput out_Renamed = Enclosing_Instance.Directory().CreateOutput(si.GetNormFileName(this.number));
try
{
out_Renamed.WriteBytes(bytes, Enclosing_Instance.MaxDoc());
}
finally
{
out_Renamed.Close();
}
this.dirty = false;
}
/// <summary>Determine index files that are no longer referenced
/// and therefore should be deleted. This is called once
/// (by the writer), and then subsequently we add onto
/// deletable any files that are no longer needed at the
/// point that we create the unused file (eg when merging
/// segments), and we only remove from deletable when a
/// file is successfully deleted.
/// </summary>
public void FindDeletableFiles()
{
// Gather all "current" segments:
System.Collections.Hashtable current = new System.Collections.Hashtable();
for (int j = 0; j < segmentInfos.Count; j++)
{
SegmentInfo segmentInfo = (SegmentInfo)segmentInfos[j];
current[segmentInfo.name] = segmentInfo;
}
// Then go through all files in the Directory that are
// Lucene index files, and add to deletable if they are
// not referenced by the current segments info:
System.String segmentsInfosFileName = segmentInfos.GetCurrentSegmentFileName();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
System.String[] files = directory.List();
for (int i = 0; i < files.Length; i++)
{
if (filter.Accept(null, files[i]) && !files[i].Equals(segmentsInfosFileName) && !files[i].Equals(IndexFileNames.SEGMENTS_GEN))
{
System.String segmentName;
System.String extension;
// First remove any extension:
int loc = files[i].IndexOf((System.Char) '.');
if (loc != -1)
{
extension = files[i].Substring(1 + loc);
segmentName = files[i].Substring(0, (loc) - (0));
}
else
{
extension = null;
segmentName = files[i];
}
// Then, remove any generation count:
loc = segmentName.IndexOf((System.Char) '_', 1);
if (loc != -1)
{
segmentName = segmentName.Substring(0, (loc) - (0));
}
// Delete this file if it's not a "current" segment,
// or, it is a single index file but there is now a
// corresponding compound file:
bool doDelete = false;
if (!current.ContainsKey(segmentName))
{
// Delete if segment is not referenced:
doDelete = true;
}
else
{
// OK, segment is referenced, but file may still
// be orphan'd:
SegmentInfo info = (SegmentInfo)current[segmentName];
if (filter.IsCFSFile(files[i]) && info.GetUseCompoundFile())
{
// This file is in fact stored in a CFS file for
// this segment:
doDelete = true;
}
else
{
Pattern p = new System.Text.RegularExpressions.Regex("s\\d+");
if ("del".Equals(extension))
{
// This is a _segmentName_N.del file:
if (!files[i].Equals(info.GetDelFileName()))
{
// If this is a seperate .del file, but it
// doesn't match the current del filename for
// this segment, then delete it:
doDelete = true;
}
}
else if (extension != null && extension.StartsWith("s") && p.Match(extension).Success)
{
int field = System.Int32.Parse(extension.Substring(1));
// This is a _segmentName_N.sX file:
if (!files[i].Equals(info.GetNormFileName(field)))
{
// This is an orphan'd separate norms file:
doDelete = true;
}
}
else if ("cfs".Equals(extension) && !info.GetUseCompoundFile())
{
// This is a partially written
// _segmentName.cfs:
doDelete = true;
}
}
}
if (doDelete)
{
AddDeletableFile(files[i]);
if (infoStream != null)
{
infoStream.WriteLine("IndexFileDeleter: file \"" + files[i] + "\" is unreferenced in index and will be deleted on next commit");
}
}
}
}
}
