public NoPreviousBookmark() { var bookmarkProvider = Substitute.For <IBookmarkProvider>(); bookmarkProvider .When(x => x.UpdateBookmark(Arg.Any <FileSetPosition>())) .Do(x => _sut = x.ArgAt <FileSetPosition>(0)); bookmarkProvider.GetCurrentBookmarkPosition().Returns(null as FileSetPosition); IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); var provider = new FileBufferDataProvider( BaseBufferFileName, fsAdapter, bookmarkProvider, Utf8Encoder, BatchLimit, EventSizeLimit, null); provider.MoveBookmarkForward(); }
public RetentionLimitLessThenNumberOfBufferFilesAndBookMarkOnSecondFile() { var bookmarkProvider = Substitute.For <IBookmarkProvider>(); bookmarkProvider .When(x => x.UpdateBookmark(Arg.Any <FileSetPosition>())) .Do(x => _sut = x.ArgAt <FileSetPosition>(0)); IFileSystemAdapter fsAdapter = CreateFileSystemAdapter(); var provider = new FileBufferDataProviderThatAllowscurrentBookmarkToBeSet( BaseBufferFileName, fsAdapter, bookmarkProvider, Utf8Encoder, BatchLimit, EventSizeLimit, Limit); //force the current Bookmark to be second file: provider.DefineCurrentBookmark(123, @"c:\a\file002.json"); provider.MoveBookmarkForward(); }
bool IsValidBookmark(FileSetPosition bookmark) { return(bookmark?.File != null && _fileSystemAdapter.Exists(bookmark.File)); }
public void MoveBookmarkForward() { //curren Batch is empty, so we should clear it out so that the enxt read cycle will refresh it correctly _currentBatchOfEventsToProcess = null; // Only advance the bookmark if no other process has the // current file locked, and its length is as we found it. // NOTE: we will typically enter this method after any buffer file is finished // (no events read from previous file). This is the oportunity to clear out files // especially the prevously read file var fileSet = GetEventBufferFileSet(); try { //if we only have two files, move to the next one imediately, unless a locking situation // impeads us from doing so if (fileSet.Length == 2 && fileSet.First() == _currentBookmark.File && IsUnlockedAtLength(_currentBookmark.File, _currentBookmark.NextLineStart)) { //move to next file _currentBookmark = new FileSetPosition(0, fileSet[1]); _bookmarkProvider.UpdateBookmark(_currentBookmark); //we can also delete the previously read file since we no longer need it _fileSystemAdapter.DeleteFile(fileSet[0]); } if (fileSet.Length > 2) { //when we have more files, we want to delete older ones, but this depends on the // limit retention policy. If no limit retention policy is in place, the intent is to // send all messages, no matter how old. In this case, we should only delete the current // file (since we are finished with it) and start at the next one. If we do have some // retention policy in place, then delete anything older then the limit and the next // message read should be at the start of the policy limit if (_retainedFileCountLimit.HasValue) { //move to first file within retention limite _currentBookmark = _retainedFileCountLimit.Value >= fileSet.Length ? new FileSetPosition(0, fileSet[1]) : new FileSetPosition(0, fileSet[fileSet.Length - _retainedFileCountLimit.Value]); _bookmarkProvider.UpdateBookmark(_currentBookmark); //delete all the old files foreach (var oldFile in fileSet.Take(fileSet.Length - _retainedFileCountLimit.Value)) { _fileSystemAdapter.DeleteFile(oldFile); } } else { //move to the next file and delete the current one _currentBookmark = new FileSetPosition(0, fileSet[1]); _bookmarkProvider.UpdateBookmark(_currentBookmark); //set the file bookmark to avoid _fileSystemAdapter.DeleteFile(fileSet[0]); } } } catch (Exception ex) { SelfLog.WriteLine("An error occured while deleteing the files...{0}", ex.Message); } finally { //even if reading / deleteing files fails, we can / should update the bookmark file //it is important that the file have the reset position, otherwise we risk failing to // move forward in the next read cycle //it's possible that no bookmark exists, especially if no valid messages have forced a // durable log file to be created. In this case, the bookmark file will be empty and // on disk if (_currentBookmark != null) { _bookmarkProvider.UpdateBookmark(_currentBookmark); } } }
public void CanCreateBookmarkInstance() { var marker = new FileSetPosition(0, @"C:\test"); Assert.NotNull(marker); }
public void MoveBookmarkForward() { //curren Batch is empty, so we should clear it out so that the enxt read cycle will refresh it correctly _currentBatchOfEventsToProcess = null; // Only advance the bookmark if no other process has the // current file locked, and its length is as we found it. // NOTE: we will typically enter this method after any buffer file is finished // (no events read from previous file). This is the oportunity to clear out files // especially the prevously read file var fileSet = GetEventBufferFileSet(); try { //if we only have two files, move to the next one imediately, unless a locking situation // impeads us from doing so if (fileSet.Length == 2 && fileSet.First() == CurrentBookmark.File && IsUnlockedAtLength(CurrentBookmark.File, CurrentBookmark.NextLineStart)) { //move to next file CurrentBookmark = new FileSetPosition(0, fileSet[1]); //we can also delete the previously read file since we no longer need it _fileSystemAdapter.DeleteFile(fileSet[0]); } if (fileSet.Length > 2) { //determine where in the fileset on disk is the current bookmark (if any) //there is no garantee the the current one is the first, so we need to make //sure we start reading the next one based on the current one var currentBookMarkedFileInFileSet = CurrentBookmark != null ? Array.FindIndex(fileSet, f => f == CurrentBookmark.File) : -1; //when we have more files, we want to delete older ones, but this depends on the // limit retention policy. If no limit retention policy is in place, the intent is to // send all messages, no matter how old. In this case, we should only delete the current // file (since we are finished with it) and start the bookmark at the next one. If we do have some // retention policy in place, then delete anything older then the limit and the next // message read should be at the start of the policy limit if (_retainedFileCountLimit.HasValue) { if (fileSet.Length <= _retainedFileCountLimit.Value) { //start var fileIndexToStartAt = Math.Max(0, currentBookMarkedFileInFileSet + 1); //if index of current is not the last , use next; otherwise preserve current CurrentBookmark = fileIndexToStartAt <= fileSet.Length - 1 ? new FileSetPosition(0, fileSet[fileIndexToStartAt]) : CurrentBookmark; //delete all the old files DeleteFilesInFileSetUpToIndex(fileSet, fileIndexToStartAt); } else { //start at next or first in retention count var fileIndexToStartAt = Math.Max(fileSet.Length - _retainedFileCountLimit.Value, currentBookMarkedFileInFileSet + 1); CurrentBookmark = new FileSetPosition(0, fileSet[fileIndexToStartAt]); //delete all the old files DeleteFilesInFileSetUpToIndex(fileSet, fileIndexToStartAt); } } else { // not sure this can occur, but if for some reason the file is no longer in the list // we should start from the beginning, maybe; being a bit defensive here due to // https://github.com/serilog/serilog-sinks-loggly/issues/25 if (currentBookMarkedFileInFileSet == -1) { //if not in file set, use first in set (or none) CurrentBookmark = fileSet.Length > 0 ? new FileSetPosition(0, fileSet[0]) : null; } else { //if index of current is not the last , use next; otherwise preserve current CurrentBookmark = currentBookMarkedFileInFileSet <= fileSet.Length - 2 ? new FileSetPosition(0, fileSet[currentBookMarkedFileInFileSet + 1]) : CurrentBookmark; } //also clear all the previous files in the set to avoid problems (and because //they should no longer be considered). If no previous exists (index is -1) //keep existing; also do not reomve last file as it may be written to / locked DeleteFilesInFileSetUpToIndex(fileSet, currentBookMarkedFileInFileSet + 1); } } } catch (Exception ex) { SelfLog.WriteLine("An error occured while deleteing the files...{0}", ex.Message); } finally { //even if reading / deleteing files fails, we can / should update the bookmark file //it is important that the file have the reset position, otherwise we risk failing to // move forward in the next read cycle //it's possible that no bookmark exists, especially if no valid messages have forced a // durable log file to be created. In this case, the bookmark file will be empty and // on disk if (CurrentBookmark != null) { _bookmarkProvider.UpdateBookmark(CurrentBookmark); } } }
/// <summary> /// This helps set the inital bookmark in tests /// </summary> /// <param name="bookmarkToUse"></param> internal void DefineCurrentBookmark(long nextLineStartToUse, string file) { CurrentBookmark = new FileSetPosition(nextLineStartToUse, file); }