/// <summary>
/// Adds the specified commit point to the master file
/// </summary>
/// <param name="storeLocation">The path to the store directory</param>
/// <param name="commitPoint">The commit point to add</param>
/// <param name="overwrite">Specifies if the master file should be overwritten with just this commit point. Defaults to false.</param>
/// <remarks>The <paramref name="overwrite"/> parameter is provided to enable store consolidation operations.</remarks>
public void AppendCommitPoint(string storeLocation, CommitPoint commitPoint, bool overwrite = false)
{
Logging.LogDebug("AbstractStoreManager.AppendCommitPoint {0}, overwrite={1}", storeLocation, overwrite);
var masterFileLocation = Path.Combine(storeLocation, MasterFileName);
if (overwrite)
{
// Use a file delete operation to try to avoid file locking issues.
Logging.LogDebug("AbstractStoreManager: Overwrite enabled. Deleting existing master file");
_persistenceManager.DeleteFile(masterFileLocation);
}
if (!_persistenceManager.FileExists(masterFileLocation))
{
Logging.LogDebug("AbstractStoreManager: Master file not found at {0}. Creating new master file.", masterFileLocation);
_persistenceManager.CreateFile(masterFileLocation);
}
using (var fs = _persistenceManager.GetOutputStream(masterFileLocation, FileMode.Append))
{
var binaryWriter = new BinaryWriter(fs);
ulong val = 0;
if (_persistenceManager.GetFileLength(masterFileLocation) == 0)
{
// new master file so add header
for (int i = 0; i < MasterfileHeaderLongCount; i++)
{
binaryWriter.Write(val);
}
}
commitPoint.Save(fs);
}
Logging.LogDebug("AbstractStoreManager.UpdateMsterFile {0} completed.", storeLocation);
}
public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint)
{
if (commitPoint.ExcludedSerialNos.Count != 0)
{
throw new Exception($"{commitPoint.ExcludedSerialNos.Count} writes unexpectedly excluded from checkpoint");
}
}
internal ulong GetLatestStorePositionFromMasterFile(string masterFilePath)
{
try
{
Logging.LogDebug("Retrieving latest store position from masterfile : {0}", masterFilePath);
using (var fs = _persistenceManager.GetInputStream(masterFilePath))
{
Logging.LogDebug("Masterfile stream length is {0}", fs.Length);
Logging.LogDebug("Attempting to seek to {0} bytes from end of stream.", CommitPoint.RecordSize);
fs.Seek(-CommitPoint.RecordSize, SeekOrigin.End);
Logging.LogDebug("Seek completed ok. Attempting to load commit point");
var commitPoint = CommitPoint.Load(fs);
Logging.LogDebug("Commit point load completed OK. Returning commit point offset as {0}", commitPoint.LocationOffset);
return(commitPoint.LocationOffset);
}
}
catch (InvalidCommitPointException icp)
{
Logging.LogInfo("Caught InvalidCommitPointException: {0}", icp);
// start reading from the start of the file until we get to the dud one.
// truncate the file at this point. log it and try again.
var count = 0;
CommitPoint validCommitPoint = null;
const int headerSize = MasterfileHeaderLongCount * 8;
while (true)
{
using (var fs = _persistenceManager.GetInputStream(masterFilePath))
{
try
{
Logging.LogInfo("Reading commit point at " + CommitPoint.RecordSize * count);
fs.Seek((CommitPoint.RecordSize * count) + headerSize, SeekOrigin.Begin);
var commitPoint = CommitPoint.Load(fs);
validCommitPoint = commitPoint;
count++;
}
catch (BrightstarInternalException)
{
var startOfBadCommit = (CommitPoint.RecordSize * count) + headerSize;
Logging.LogInfo("Truncating file at " + startOfBadCommit);
// truncate file.
using (var stream = _persistenceManager.GetOutputStream(masterFilePath, FileMode.Truncate))
{
stream.SetLength(startOfBadCommit);
}
// return last good commit
return(validCommitPoint.LocationOffset);
}
}
}
}
catch (Exception ex)
{
throw new BrightstarInternalException("Error while trying to recover to last valid commit point.", ex);
}
}
public void RevertToCommitPoint(string storeId, ulong commitPointLocation)
{
// TODO: Validate that this is a proper commit point location
//var commitPoint = _storeManager.GetCommitPoint(_baseLocation + "\\" + storeId, commitPointLocation);
var lastCommit = _storeManager.GetMasterFile(Path.Combine(_baseLocation, storeId)).GetLatestCommitPoint();
var commitPoint = new CommitPoint(commitPointLocation, lastCommit.NextCommitNumber, DateTime.UtcNow, Guid.Empty);
var storeWorker = GetStoreWorker(storeId);
storeWorker.WriteStore.RevertToCommitPoint(commitPoint);
storeWorker.InvalidateReadStore();
}
/// <summary>
/// Makes the provided commit point the most recent one.
/// </summary>
/// <param name="commitPoint">The commitpoint to make the most recent.</param>
public void RevertToCommitPoint(CommitPoint commitPoint)
{
var storeManager = StoreManagerFactory.GetStoreManager();
var masterFile = storeManager.GetMasterFile(DirectoryPath);
if (masterFile.PersistenceType == PersistenceType.Rewrite)
{
throw new BrightstarClientException("Revert is not supported by a store using the binary page persistence type.");
}
masterFile.AppendCommitPoint(commitPoint);
}
public CommitPoint GetCommitPoint(string storeLocation, ulong offset)
{
var masterFileLocation = Path.Combine(storeLocation, MasterFileName);
using (var fs = _persistenceManager.GetInputStream(masterFileLocation))
{
fs.Seek((long)offset, SeekOrigin.Begin);
var commitPoint = CommitPoint.Load(fs);
return(commitPoint);
}
}
public override void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint)
{
switch (checkpointCallbackExpectation)
{
case 0:
Assert.IsTrue(false, "Unexpected checkpoint callback");
break;
default:
Interlocked.Decrement(ref checkpointCallbackExpectation);
break;
}
}
/// <summary> Remove the CommitPoints in the commitsToDelete List by
/// DecRef'ing all files from each SegmentInfos.
/// </summary>
private void DeleteCommits()
{
int size = commitsToDelete.Count;
if (size > 0)
{
// First decref all files that had been referred to by
// the now-deleted commits:
for (int i = 0; i < size; i++)
{
CommitPoint commit = (CommitPoint)commitsToDelete[i];
if (infoStream != null)
{
Message("deleteCommits: now remove commit \"" + commit.GetSegmentsFileName() + "\"");
}
int size2 = commit.files.Count;
for (int j = 0; j < size2; j++)
{
DecRef((System.String)commit.files[j]);
}
}
commitsToDelete.Clear();
// Now compact commits to remove deleted ones (preserving the sort):
size = commits.Count;
int readFrom = 0;
int writeTo = 0;
while (readFrom < size)
{
CommitPoint commit = (CommitPoint)commits[readFrom];
if (!commit.deleted)
{
if (writeTo != readFrom)
{
commits[writeTo] = commits[readFrom];
}
writeTo++;
}
readFrom++;
}
while (size > writeTo)
{
commits.RemoveAt(size - 1);
size--;
}
}
}
/// <summary> Remove the CommitPoints in the commitsToDelete List by
/// DecRef'ing all files from each SegmentInfos.
/// </summary>
private void DeleteCommits()
{
int size = commitsToDelete.Count;
if (size > 0)
{
// First decref all files that had been referred to by
// the now-deleted commits:
for (int i = 0; i < size; i++)
{
CommitPoint commit = (CommitPoint)commitsToDelete[i];
if (infoStream != null)
{
Message("deleteCommits: now decRef commit \"" + commit.GetSegmentsFileName() + "\"");
}
System.Collections.Generic.IEnumerator <string> it = commit.files.GetEnumerator();
while (it.MoveNext())
{
DecRef(it.Current);
}
}
commitsToDelete.Clear();
// Now compact commits to remove deleted ones (preserving the sort):
size = commits.Count;
int readFrom = 0;
int writeTo = 0;
while (readFrom < size)
{
CommitPoint commit = (CommitPoint)commits[readFrom];
if (!commit.deleted)
{
if (writeTo != readFrom)
{
commits[writeTo] = commits[readFrom];
}
writeTo++;
}
readFrom++;
}
while (size > writeTo)
{
commits.RemoveAt(size - 1);
size--;
}
}
}
public IEnumerable <CommitPoint> GetCommitPoints(string storeLocation)
{
var masterFileLocation = Path.Combine(storeLocation, MasterFileName);
var pos = 1;
using (var fs = _persistenceManager.GetInputStream(masterFileLocation))
{
while ((pos * CommitPoint.RecordSize) + MasterfileHeaderSize <= fs.Length)
{
fs.Seek(-(pos * CommitPoint.RecordSize), SeekOrigin.End);
var commitPoint = CommitPoint.Load(fs);
pos++;
yield return(commitPoint);
}
}
}
/// <summary>
/// Remove the CommitPoints in the commitsToDelete List by
/// DecRef'ing all files from each SegmentInfos.
/// </summary>
private void DeleteCommits()
{
int size = commitsToDelete.Count;
if (size > 0)
{
// First decref all files that had been referred to by
// the now-deleted commits:
for (int i = 0; i < size; i++)
{
CommitPoint commit = commitsToDelete[i];
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "deleteCommits: now decRef commit \"" + commit.SegmentsFileName + "\"");
}
foreach (string file in commit.files)
{
DecRef(file);
}
}
commitsToDelete.Clear();
// Now compact commits to remove deleted ones (preserving the sort):
size = commits.Count;
int readFrom = 0;
int writeTo = 0;
while (readFrom < size)
{
CommitPoint commit = commits[readFrom];
if (!commit.deleted)
{
if (writeTo != readFrom)
{
commits[writeTo] = commits[readFrom];
}
writeTo++;
}
readFrom++;
}
while (size > writeTo)
{
commits.RemoveAt(size - 1);
size--;
}
}
}
public int CompareTo(System.Object obj)
{
CommitPoint commit = (CommitPoint)obj;
if (gen < commit.gen)
{
return(-1);
}
else if (gen > commit.gen)
{
return(1);
}
else
{
return(0);
}
}
/// <summary> Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. The incoming
/// segmentInfos must have been loaded from a commit point
/// and not yet modified. This will remove any files not
/// referenced by any of the commits.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.TextWriter infoStream, DocumentsWriter docWriter)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
if (infoStream != null)
{
Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.GetGeneration();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
System.String[] files = directory.List();
if (files == null)
{
throw new System.IO.IOException("cannot read directory " + directory + ": list() returned null");
}
CommitPoint currentCommitPoint = null;
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
{
if (infoStream != null)
{
Message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (System.IO.FileNotFoundException e)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != null)
{
Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(this, sis);
if (sis.GetGeneration() == segmentInfos.GetGeneration())
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
}
}
}
}
}
if (currentCommitPoint == null)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
}
catch (System.IO.IOException e)
{
throw new CorruptIndexException("failed to locate current segments_N file");
}
if (infoStream != null)
{
Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
}
currentCommitPoint = new CommitPoint(this, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
commits.Sort();
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
System.Collections.IEnumerator it = refCounts.Keys.GetEnumerator();
while (it.MoveNext())
{
System.String fileName = (System.String)it.Current;
RefCount rc = (RefCount)refCounts[fileName];
if (0 == rc.count)
{
if (infoStream != null)
{
Message("init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
policy.OnInit(commits);
// It's OK for the onInit to remove the current commit
// point; we just have to checkpoint our in-memory
// SegmentInfos to protect those files that it uses:
if (currentCommitPoint.deleted)
{
Checkpoint(segmentInfos, false);
}
DeleteCommits();
}
/// <summary>
/// Initialize the deleter: find all previous commits in
/// the <see cref="Directory"/>, incref the files they reference, call
/// the policy to let it delete commits. this will remove
/// any files not referenced by any of the commits. </summary>
/// <exception cref="IOException"> if there is a low-level IO error </exception>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, InfoStream infoStream, IndexWriter writer, bool initialIndexExists)
{
this.infoStream = infoStream;
this.writer = writer;
string currentSegmentsFile = segmentInfos.GetSegmentsFileName();
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: current segments file is \"" + currentSegmentsFile + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.Generation;
CommitPoint currentCommitPoint = null;
string[] files = null;
try
{
files = directory.ListAll();
}
#pragma warning disable 168
catch (DirectoryNotFoundException e)
#pragma warning restore 168
{
// it means the directory is empty, so ignore it.
files = new string[0];
}
if (currentSegmentsFile != null)
{
Regex r = IndexFileNames.CODEC_FILE_PATTERN;
foreach (string fileName in files)
{
if (!fileName.EndsWith("write.lock", StringComparison.Ordinal) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN, StringComparison.Ordinal) &&
(r.IsMatch(fileName) || fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal)))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS, StringComparison.Ordinal))
{
// this is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
#pragma warning disable 168
catch (FileNotFoundException e)
#pragma warning restore 168
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
// LUCENENET specific - .NET (thankfully) only has one FileNotFoundException, so we don't need this
//catch (NoSuchFileException)
//{
// // LUCENE-948: on NFS (and maybe others), if
// // you have writers switching back and forth
// // between machines, it's very likely that the
// // dir listing will be stale and will claim a
// // file segments_X exists when in fact it
// // doesn't. So, we catch this and handle it
// // as if the file does not exist
// if (infoStream.IsEnabled("IFD"))
// {
// infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
// }
// sis = null;
//}
// LUCENENET specific - since NoSuchDirectoryException subclasses FileNotFoundException
// in Lucene, we need to catch it here to be on the safe side.
catch (System.IO.DirectoryNotFoundException)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
catch (IOException /*e*/)
{
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen && directory.FileLength(fileName) > 0)
{
throw; // LUCENENET: CA2200: Rethrow to preserve stack details (https://docs.microsoft.com/en-us/visualstudio/code-quality/ca2200-rethrow-to-preserve-stack-details)
}
else
{
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
sis = null;
}
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis);
if (sis.Generation == segmentInfos.Generation)
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation)
{
lastSegmentInfos = sis;
}
}
}
}
}
}
if (currentCommitPoint == null && currentSegmentsFile != null && initialIndexExists)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. this can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, currentSegmentsFile);
}
catch (IOException e)
{
throw new CorruptIndexException("failed to locate current segments_N file \"" + currentSegmentsFile + "\"" + e.ToString(), e);
}
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "forced open of current segments file " + segmentInfos.GetSegmentsFileName());
}
currentCommitPoint = new CommitPoint(commitsToDelete, directory, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
CollectionUtil.TimSort(commits);
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
foreach (KeyValuePair <string, RefCount> entry in refCounts)
{
RefCount rc = entry.Value;
string fileName = entry.Key;
if (0 == rc.count)
{
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
this.policy.OnInit(commits);
// Always protect the incoming segmentInfos since
// sometime it may not be the most recent commit
Checkpoint(segmentInfos, false);
startingCommitDeleted = currentCommitPoint == null ? false : currentCommitPoint.IsDeleted;
DeleteCommits();
}
/// <summary> Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. This will remove
/// any files not referenced by any of the commits.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, System.Collections.Generic.Dictionary<string, string> synced)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
this.synced = synced;
if (infoStream != null)
{
Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.GetGeneration();
IndexFileNameFilter filter = IndexFileNameFilter.GetFilter();
System.String[] files = directory.ListAll();
CommitPoint currentCommitPoint = null;
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (infoStream != null)
{
Message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (System.IO.FileNotFoundException e)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != null)
{
Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
catch (System.IO.IOException e)
{
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
{
throw e;
}
else
{
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
sis = null;
}
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(this,commitsToDelete, directory, sis);
if (sis.GetGeneration() == segmentInfos.GetGeneration())
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
if (lastSegmentInfos == null || sis.GetGeneration() > lastSegmentInfos.GetGeneration())
{
lastSegmentInfos = sis;
}
}
}
}
}
if (currentCommitPoint == null)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
}
catch (System.IO.IOException e)
{
throw new CorruptIndexException("failed to locate current segments_N file");
}
if (infoStream != null)
Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
commits.Sort();
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
System.Collections.Generic.IEnumerator<System.Collections.Generic.KeyValuePair<System.String, RefCount>> it = refCounts.GetEnumerator();
while (it.MoveNext())
{
System.String fileName = (System.String) it.Current.Key;
RefCount rc = (RefCount) refCounts[fileName];
if (0 == rc.count)
{
if (infoStream != null)
{
Message("init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
policy.OnInit(commits);
// Always protect the incoming segmentInfos since
// sometime it may not be the most recent commit
Checkpoint(segmentInfos, false);
startingCommitDeleted = currentCommitPoint.IsDeleted();
DeleteCommits();
}
/// <summary>
/// Makes the provided commit point the most recent one.
/// </summary>
/// <param name="commitPoint">The commitpoint to make the most recent.</param>
public void RevertToCommitPoint(CommitPoint commitPoint)
{
var storeManager = StoreManagerFactory.GetStoreManager();
var masterFile = storeManager.GetMasterFile(DirectoryPath);
if (masterFile.PersistenceType == PersistenceType.Rewrite)
{
throw new BrightstarClientException("Revert is not supported by a store using the binary page persistence type.");
}
masterFile.AppendCommitPoint(commitPoint);
}
/// <summary>
/// Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. this will remove
/// any files not referenced by any of the commits. </summary>
/// <exception cref="IOException"> if there is a low-level IO error </exception>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, InfoStream infoStream, IndexWriter writer, bool initialIndexExists)
{
this.InfoStream = infoStream;
this.Writer = writer;
string currentSegmentsFile = segmentInfos.SegmentsFileName;
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: current segments file is \"" + currentSegmentsFile + "\"; deletionPolicy=" + policy);
}
this.Policy = policy;
this.Directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.Generation;
CommitPoint currentCommitPoint = null;
string[] files = null;
try
{
files = directory.ListAll();
}
catch (NoSuchDirectoryException e)
{
// it means the directory is empty, so ignore it.
files = new string[0];
}
if (currentSegmentsFile != null)
{
Regex r = IndexFileNames.CODEC_FILE_PATTERN;
foreach (string fileName in files)
{
if (!fileName.EndsWith("write.lock") && !fileName.Equals(IndexFileNames.SEGMENTS_GEN)
&& (r.IsMatch(fileName) || fileName.StartsWith(IndexFileNames.SEGMENTS)))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// this is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (FileNotFoundException e)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
catch (IOException e)
{
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen && directory.FileLength(fileName) > 0)
{
throw e;
}
else
{
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
sis = null;
}
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(CommitsToDelete, directory, sis);
if (sis.Generation == segmentInfos.Generation)
{
currentCommitPoint = commitPoint;
}
Commits.Add(commitPoint);
IncRef(sis, true);
if (LastSegmentInfos_Renamed == null || sis.Generation > LastSegmentInfos_Renamed.Generation)
{
LastSegmentInfos_Renamed = sis;
}
}
}
}
}
}
if (currentCommitPoint == null && currentSegmentsFile != null && initialIndexExists)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. this can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, currentSegmentsFile);
}
catch (IOException e)
{
throw new CorruptIndexException("failed to locate current segments_N file \"" + currentSegmentsFile + "\"");
}
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "forced open of current segments file " + segmentInfos.SegmentsFileName);
}
currentCommitPoint = new CommitPoint(CommitsToDelete, directory, sis);
Commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
CollectionUtil.TimSort(Commits);
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
foreach (KeyValuePair<string, RefCount> entry in RefCounts)
{
RefCount rc = entry.Value;
string fileName = entry.Key;
if (0 == rc.Count)
{
if (infoStream.IsEnabled("IFD"))
{
infoStream.Message("IFD", "init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
Policy.OnInit(Commits);
// Always protect the incoming segmentInfos since
// sometime it may not be the most recent commit
Checkpoint(segmentInfos, false);
StartingCommitDeleted = currentCommitPoint == null ? false : currentCommitPoint.Deleted;
DeleteCommits();
}
public void CheckpointCompletionCallback(int sessionID, string sessionName, CommitPoint commitPoint)
=> Debug.WriteLine($"Session {sessionID} ({(sessionName ?? "null")}) reports persistence until {commitPoint.UntilSerialNo}");
public override void CheckpointCompletionCallback(int sessionID, string sessionName, CommitPoint commitPoint)
{
switch (checkpointCallbackExpectation)
{
case 0:
Assert.Fail("Unexpected checkpoint callback");
break;
default:
Interlocked.Decrement(ref checkpointCallbackExpectation);
break;
}
}
public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint)
{
throw new NotImplementedException();
}
public void CheckpointCompletionCallback(int sessionID, string sessionName, CommitPoint commitPoint)
{
}
/// <summary> Initialize the deleter: find all previous commits in
/// the Directory, incref the files they reference, call
/// the policy to let it delete commits. This will remove
/// any files not referenced by any of the commits.
/// </summary>
/// <throws> CorruptIndexException if the index is corrupt </throws>
/// <throws> IOException if there is a low-level IO error </throws>
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, HashSet <string> synced)
{
this.docWriter = docWriter;
this.infoStream = infoStream;
this.synced = synced;
if (infoStream != null)
{
Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
}
this.policy = policy;
this.directory = directory;
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.Generation;
IndexFileNameFilter filter = IndexFileNameFilter.Filter;
System.String[] files = directory.ListAll();
CommitPoint currentCommitPoint = null;
for (int i = 0; i < files.Length; i++)
{
System.String fileName = files[i];
if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
{
// Add this file to refCounts with initial count 0:
GetRefCount(fileName);
if (fileName.StartsWith(IndexFileNames.SEGMENTS))
{
// This is a commit (segments or segments_N), and
// it's valid (<= the max gen). Load it, then
// incref all files it refers to:
if (infoStream != null)
{
Message("init: load commit \"" + fileName + "\"");
}
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, fileName);
}
catch (System.IO.FileNotFoundException)
{
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
if (infoStream != null)
{
Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
}
sis = null;
}
catch (System.IO.IOException)
{
if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
{
throw;
}
else
{
// Most likely we are opening an index that
// has an aborted "future" commit, so suppress
// exc in this case
sis = null;
}
}
if (sis != null)
{
CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
if (sis.Generation == segmentInfos.Generation)
{
currentCommitPoint = commitPoint;
}
commits.Add(commitPoint);
IncRef(sis, true);
if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation)
{
lastSegmentInfos = sis;
}
}
}
}
}
if (currentCommitPoint == null)
{
// We did not in fact see the segments_N file
// corresponding to the segmentInfos that was passed
// in. Yet, it must exist, because our caller holds
// the write lock. This can happen when the directory
// listing was stale (eg when index accessed via NFS
// client with stale directory listing cache). So we
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try
{
sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
}
catch (System.IO.IOException)
{
throw new CorruptIndexException("failed to locate current segments_N file");
}
if (infoStream != null)
{
Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
}
currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
commits.Add(currentCommitPoint);
IncRef(sis, true);
}
// We keep commits list in sorted order (oldest to newest):
commits.Sort();
// Now delete anything with ref count at 0. These are
// presumably abandoned files eg due to crash of
// IndexWriter.
foreach (KeyValuePair <string, RefCount> entry in refCounts)
{
string fileName = entry.Key;
RefCount rc = refCounts[fileName];
if (0 == rc.count)
{
if (infoStream != null)
{
Message("init: removing unreferenced file \"" + fileName + "\"");
}
DeleteFile(fileName);
}
}
// Finally, give policy a chance to remove things on
// startup:
policy.OnInit(commits);
// Always protect the incoming segmentInfos since
// sometime it may not be the most recent commit
Checkpoint(segmentInfos, false);
startingCommitDeleted = currentCommitPoint.IsDeleted;
DeleteCommits();
}
public void CheckpointCompletionCallback(int sessionID, string sessionName, CommitPoint commitPoint) => functions.CheckpointCompletionCallback(sessionID, sessionName, commitPoint);
public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint)
{
Console.WriteLine("Session {0} reports persistence until {1}", sessionId, commitPoint.UntilSerialNo);
}
public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) => functions.CheckpointCompletionCallback(sessionId, commitPoint);
public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint)
{
Logger.Info(nameof(CheckpointCompletionCallback), $"SessionId: {sessionId}", $"CommitPoint: {JsonSerializer.Serialize(commitPoint)}");
}
public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint)
{
}
/// <inheritdoc />
public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint)
{
_logger.LogTrace($"Checkpoint created, session-id '{sessionId}'");
}