/// <summary> Writer calls this when it has hit an error and had to
/// roll back, to tell us that there may now be
/// unreferenced files in the filesystem. So we re-list
/// the filesystem and delete such files. If segmentName
/// is non-null, we will only delete files corresponding to
/// that segment.
/// </summary>
public void Refresh(System.String segmentName)
{
System.String[] files = directory.ListAll();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
System.String segmentPrefix1;
System.String segmentPrefix2;
if (segmentName != null)
{
segmentPrefix1 = segmentName + ".";
segmentPrefix2 = segmentName + "_";
}
else
{
segmentPrefix1 = null;
segmentPrefix2 = null;
}
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && (segmentName == null || fileName.StartsWith(segmentPrefix1) || fileName.StartsWith(segmentPrefix2)) && !refCounts.ContainsKey(fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Unreferenced file, so remove it
if (infoStream != null)
{
Message("refresh [prefix=" + segmentName + "]: removing newly created unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
}
/*
* Return all files referenced by this SegmentInfo. The
* returns List is a locally cached List so you should not
* modify it.
*/
public System.Collections.Generic.IList <string> Files()
{
if (files != null)
{
// Already cached:
return(files);
}
System.Collections.Generic.List <string> fileList = new System.Collections.Generic.List <string>();
bool useCompoundFile = GetUseCompoundFile();
if (useCompoundFile)
{
fileList.Add(name + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
}
else
{
System.String[] exts = IndexFileNames.NON_STORE_INDEX_EXTENSIONS;
for (int i = 0; i < exts.Length; i++)
{
AddIfExists(fileList, name + "." + exts[i]);
}
}
if (docStoreOffset != -1)
{
// We are sharing doc stores (stored fields, term
// vectors) with other segments
System.Diagnostics.Debug.Assert(docStoreSegment != null);
if (docStoreIsCompoundFile)
{
fileList.Add(docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION);
}
else
{
System.String[] exts = IndexFileNames.STORE_INDEX_EXTENSIONS;
for (int i = 0; i < exts.Length; i++)
{
AddIfExists(fileList, docStoreSegment + "." + exts[i]);
}
}
}
else if (!useCompoundFile)
{
// We are not sharing, and, these files were not
// included in the compound file
System.String[] exts = IndexFileNames.STORE_INDEX_EXTENSIONS;
for (int i = 0; i < exts.Length; i++)
{
AddIfExists(fileList, name + "." + exts[i]);
}
}
System.String delFileName = IndexFileNames.FileNameFromGeneration(name, "." + IndexFileNames.DELETES_EXTENSION, delGen);
if (delFileName != null && (delGen >= YES || dir.FileExists(delFileName)))
{
fileList.Add(delFileName);
}
// Careful logic for norms files
if (normGen != null)
{
for (int i = 0; i < normGen.Length; i++)
{
long gen = normGen[i];
if (gen >= YES)
{
// Definitely a separate norm file, with generation:
fileList.Add(IndexFileNames.FileNameFromGeneration(name, "." + IndexFileNames.SEPARATE_NORMS_EXTENSION + i, gen));
}
else if (NO == gen)
{
// No separate norms but maybe plain norms
// in the non compound file case:
if (!hasSingleNormFile && !useCompoundFile)
{
System.String fileName = name + "." + IndexFileNames.PLAIN_NORMS_EXTENSION + i;
if (dir.FileExists(fileName))
{
fileList.Add(fileName);
}
}
}
else if (CHECK_DIR == gen)
{
// Pre-2.1: we have to check file existence
System.String fileName = null;
if (useCompoundFile)
{
fileName = name + "." + IndexFileNames.SEPARATE_NORMS_EXTENSION + i;
}
else if (!hasSingleNormFile)
{
fileName = name + "." + IndexFileNames.PLAIN_NORMS_EXTENSION + i;
}
if (fileName != null && dir.FileExists(fileName))
{
fileList.Add(fileName);
}
}
}
}
else if (preLockless || (!hasSingleNormFile && !useCompoundFile))
{
// Pre-2.1: we have to scan the dir to find all
// matching _X.sN/_X.fN files for our segment:
System.String prefix;
if (useCompoundFile)
{
prefix = name + "." + IndexFileNames.SEPARATE_NORMS_EXTENSION;
}
else
{
prefix = name + "." + IndexFileNames.PLAIN_NORMS_EXTENSION;
}
int prefixLength = prefix.Length;
System.String[] allFiles = dir.ListAll();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
for (int i = 0; i < allFiles.Length; i++)
{
System.String fileName = allFiles[i];
if (filter.Accept(null, fileName) && fileName.Length > prefixLength && System.Char.IsDigit(fileName[prefixLength]) && fileName.StartsWith(prefix))
{
fileList.Add(fileName);
}
}
}
//System.Diagnostics.Debug.Assert();
files = fileList;
return(files);
}
/// <summary> Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. This will remove
/// any files not referenced by any of the commits.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, System.Collections.Generic.Dictionary <string, string> synced)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
this.synced = synced;
if (infoStream != null)
{
Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.GetGeneration();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
System.String[] files = directory.ListAll();
CommitPoint currentCommitPoint = null;
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (infoStream != null)
{
Message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (System.IO.FileNotFoundException e)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != null)
{
Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
catch (System.IO.IOException e)
{
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
{
throw e;
}
else
{
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
sis = null;
}
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
if (sis.GetGeneration() == segmentInfos.GetGeneration())
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
if (lastSegmentInfos == null || sis.GetGeneration() > lastSegmentInfos.GetGeneration())
{
lastSegmentInfos = sis;
}
}
}
}
}
if (currentCommitPoint == null)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
}
catch (System.IO.IOException e)
{
throw new CorruptIndexException("failed to locate current segments_N file");
}
if (infoStream != null)
{
Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
}
currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
commits.Sort();
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
System.Collections.Generic.IEnumerator <System.Collections.Generic.KeyValuePair <System.String, RefCount> > it = refCounts.GetEnumerator();
while (it.MoveNext())
{
System.String fileName = (System.String)it.Current.Key;
RefCount rc = (RefCount)refCounts[fileName];
if (0 == rc.count)
{
if (infoStream != null)
{
Message("init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
policy.OnInit(commits);
// Always protect the incoming segmentInfos since
// sometime it may not be the most recent commit
Checkpoint(segmentInfos, false);
startingCommitDeleted = currentCommitPoint.IsDeleted();
DeleteCommits();
}
