public NoPreviousBookmark()
{
var bookmarkProvider = Substitute.For <IBookmarkProvider>();
bookmarkProvider
.When(x => x.UpdateBookmark(Arg.Any <FileSetPosition>()))
.Do(x => _sut = x.ArgAt <FileSetPosition>(0));
bookmarkProvider.GetCurrentBookmarkPosition().Returns(null as FileSetPosition);
IFileSystemAdapter fsAdapter = CreateFileSystemAdapter();
var provider = new FileBufferDataProvider(
BaseBufferFileName,
fsAdapter,
bookmarkProvider,
Utf8Encoder,
BatchLimit,
EventSizeLimit,
null);
provider.MoveBookmarkForward();
}
public RetentionLimitLessThenNumberOfBufferFilesAndBookMarkOnSecondFile()
{
var bookmarkProvider = Substitute.For <IBookmarkProvider>();
bookmarkProvider
.When(x => x.UpdateBookmark(Arg.Any <FileSetPosition>()))
.Do(x => _sut = x.ArgAt <FileSetPosition>(0));
IFileSystemAdapter fsAdapter = CreateFileSystemAdapter();
var provider = new FileBufferDataProviderThatAllowscurrentBookmarkToBeSet(
BaseBufferFileName,
fsAdapter,
bookmarkProvider,
Utf8Encoder,
BatchLimit,
EventSizeLimit,
Limit);
//force the current Bookmark to be second file:
provider.DefineCurrentBookmark(123, @"c:\a\file002.json");
provider.MoveBookmarkForward();
}
bool IsValidBookmark(FileSetPosition bookmark)
{
return(bookmark?.File != null &&
_fileSystemAdapter.Exists(bookmark.File));
}
public void MoveBookmarkForward()
{
//curren Batch is empty, so we should clear it out so that the enxt read cycle will refresh it correctly
_currentBatchOfEventsToProcess = null;
// Only advance the bookmark if no other process has the
// current file locked, and its length is as we found it.
// NOTE: we will typically enter this method after any buffer file is finished
// (no events read from previous file). This is the oportunity to clear out files
// especially the prevously read file
var fileSet = GetEventBufferFileSet();
try
{
//if we only have two files, move to the next one imediately, unless a locking situation
// impeads us from doing so
if (fileSet.Length == 2 &&
fileSet.First() == _currentBookmark.File &&
IsUnlockedAtLength(_currentBookmark.File, _currentBookmark.NextLineStart))
{
//move to next file
_currentBookmark = new FileSetPosition(0, fileSet[1]);
_bookmarkProvider.UpdateBookmark(_currentBookmark);
//we can also delete the previously read file since we no longer need it
_fileSystemAdapter.DeleteFile(fileSet[0]);
}
if (fileSet.Length > 2)
{
//when we have more files, we want to delete older ones, but this depends on the
// limit retention policy. If no limit retention policy is in place, the intent is to
// send all messages, no matter how old. In this case, we should only delete the current
// file (since we are finished with it) and start at the next one. If we do have some
// retention policy in place, then delete anything older then the limit and the next
// message read should be at the start of the policy limit
if (_retainedFileCountLimit.HasValue)
{
//move to first file within retention limite
_currentBookmark = _retainedFileCountLimit.Value >= fileSet.Length
? new FileSetPosition(0, fileSet[1])
: new FileSetPosition(0, fileSet[fileSet.Length - _retainedFileCountLimit.Value]);
_bookmarkProvider.UpdateBookmark(_currentBookmark);
//delete all the old files
foreach (var oldFile in fileSet.Take(fileSet.Length - _retainedFileCountLimit.Value))
{
_fileSystemAdapter.DeleteFile(oldFile);
}
}
else
{
//move to the next file and delete the current one
_currentBookmark = new FileSetPosition(0, fileSet[1]);
_bookmarkProvider.UpdateBookmark(_currentBookmark); //set the file bookmark to avoid
_fileSystemAdapter.DeleteFile(fileSet[0]);
}
}
}
catch (Exception ex)
{
SelfLog.WriteLine("An error occured while deleteing the files...{0}", ex.Message);
}
finally
{
//even if reading / deleteing files fails, we can / should update the bookmark file
//it is important that the file have the reset position, otherwise we risk failing to
// move forward in the next read cycle
//it's possible that no bookmark exists, especially if no valid messages have forced a
// durable log file to be created. In this case, the bookmark file will be empty and
// on disk
if (_currentBookmark != null)
{
_bookmarkProvider.UpdateBookmark(_currentBookmark);
}
}
}
public void CanCreateBookmarkInstance()
{
var marker = new FileSetPosition(0, @"C:\test");
Assert.NotNull(marker);
}
public void MoveBookmarkForward()
{
//curren Batch is empty, so we should clear it out so that the enxt read cycle will refresh it correctly
_currentBatchOfEventsToProcess = null;
// Only advance the bookmark if no other process has the
// current file locked, and its length is as we found it.
// NOTE: we will typically enter this method after any buffer file is finished
// (no events read from previous file). This is the oportunity to clear out files
// especially the prevously read file
var fileSet = GetEventBufferFileSet();
try
{
//if we only have two files, move to the next one imediately, unless a locking situation
// impeads us from doing so
if (fileSet.Length == 2 &&
fileSet.First() == CurrentBookmark.File &&
IsUnlockedAtLength(CurrentBookmark.File, CurrentBookmark.NextLineStart))
{
//move to next file
CurrentBookmark = new FileSetPosition(0, fileSet[1]);
//we can also delete the previously read file since we no longer need it
_fileSystemAdapter.DeleteFile(fileSet[0]);
}
if (fileSet.Length > 2)
{
//determine where in the fileset on disk is the current bookmark (if any)
//there is no garantee the the current one is the first, so we need to make
//sure we start reading the next one based on the current one
var currentBookMarkedFileInFileSet = CurrentBookmark != null
? Array.FindIndex(fileSet, f => f == CurrentBookmark.File)
: -1;
//when we have more files, we want to delete older ones, but this depends on the
// limit retention policy. If no limit retention policy is in place, the intent is to
// send all messages, no matter how old. In this case, we should only delete the current
// file (since we are finished with it) and start the bookmark at the next one. If we do have some
// retention policy in place, then delete anything older then the limit and the next
// message read should be at the start of the policy limit
if (_retainedFileCountLimit.HasValue)
{
if (fileSet.Length <= _retainedFileCountLimit.Value)
{
//start
var fileIndexToStartAt = Math.Max(0, currentBookMarkedFileInFileSet + 1);
//if index of current is not the last , use next; otherwise preserve current
CurrentBookmark = fileIndexToStartAt <= fileSet.Length - 1
? new FileSetPosition(0, fileSet[fileIndexToStartAt])
: CurrentBookmark;
//delete all the old files
DeleteFilesInFileSetUpToIndex(fileSet, fileIndexToStartAt);
}
else
{
//start at next or first in retention count
var fileIndexToStartAt = Math.Max(fileSet.Length - _retainedFileCountLimit.Value, currentBookMarkedFileInFileSet + 1);
CurrentBookmark = new FileSetPosition(0, fileSet[fileIndexToStartAt]);
//delete all the old files
DeleteFilesInFileSetUpToIndex(fileSet, fileIndexToStartAt);
}
}
else
{
// not sure this can occur, but if for some reason the file is no longer in the list
// we should start from the beginning, maybe; being a bit defensive here due to
// https://github.com/serilog/serilog-sinks-loggly/issues/25
if (currentBookMarkedFileInFileSet == -1)
{
//if not in file set, use first in set (or none)
CurrentBookmark = fileSet.Length > 0
? new FileSetPosition(0, fileSet[0])
: null;
}
else
{
//if index of current is not the last , use next; otherwise preserve current
CurrentBookmark = currentBookMarkedFileInFileSet <= fileSet.Length - 2
? new FileSetPosition(0, fileSet[currentBookMarkedFileInFileSet + 1])
: CurrentBookmark;
}
//also clear all the previous files in the set to avoid problems (and because
//they should no longer be considered). If no previous exists (index is -1)
//keep existing; also do not reomve last file as it may be written to / locked
DeleteFilesInFileSetUpToIndex(fileSet, currentBookMarkedFileInFileSet + 1);
}
}
}
catch (Exception ex)
{
SelfLog.WriteLine("An error occured while deleteing the files...{0}", ex.Message);
}
finally
{
//even if reading / deleteing files fails, we can / should update the bookmark file
//it is important that the file have the reset position, otherwise we risk failing to
// move forward in the next read cycle
//it's possible that no bookmark exists, especially if no valid messages have forced a
// durable log file to be created. In this case, the bookmark file will be empty and
// on disk
if (CurrentBookmark != null)
{
_bookmarkProvider.UpdateBookmark(CurrentBookmark);
}
}
}
/// <summary>
/// This helps set the inital bookmark in tests
/// </summary>
/// <param name="bookmarkToUse"></param>
internal void DefineCurrentBookmark(long nextLineStartToUse, string file)
{
CurrentBookmark = new FileSetPosition(nextLineStartToUse, file);
}
