private async Task UploadOrTouchFileAsync(OperationContext context, AbsolutePath file, ConcurrentDictionary <string, string> newCheckpointInfo, string incrementalCheckpointsPrefix)
{
var relativePath = file.Path.Substring(_checkpointStagingDirectory.Path.Length + 1);
var incrementalCheckpointFile = _incrementalCheckpointDirectory / relativePath;
if (_incrementalCheckpointInfo.TryGetValue(relativePath, out var storageId) && _database.IsImmutable(file) &&
_fileSystem.FileExists(incrementalCheckpointFile))
{
// File was present in last checkpoint. Just add it to the new incremental checkpoint info
await _storage.TouchBlobAsync(context, file, storageId, isUploader : true).ThrowIfFailure();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploadSkipped].Increment();
}
else
{
// File is new or mutable. Need to add to storage and update local incremental checkpoint
await HardlinkWithFallBackAsync(context, file, incrementalCheckpointFile);
storageId = await _storage.UploadFileAsync(context, file, incrementalCheckpointsPrefix + file.FileName).ThrowIfFailureAsync();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploaded].Increment();
}
}
private async Task <bool> CreateCheckpointIncrementalAsync(OperationContext context, EventSequencePoint sequencePoint, bool successfullyUpdatedIncrementalState)
{
InitializeIncrementalCheckpointIfNeeded();
BoolResult result = BoolResult.Success;
var incrementalCheckpointsPrefix = $"incrementalCheckpoints/{sequencePoint.SequenceNumber}.{Guid.NewGuid()}.";
var newCheckpointInfo = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
// Get files in checkpoint and apply changes to the incremental checkpoint directory (locally and in blob storage)
var files = _fileSystem.EnumerateFiles(_checkpointStagingDirectory, EnumerateOptions.Recurse).Select(s => s.FullPath).ToList();
foreach (var file in files)
{
var relativePath = file.Path.Substring(_checkpointStagingDirectory.Path.Length + 1);
var incrementalCheckpointFile = _incrementalCheckpointDirectory / relativePath;
if (_incrementalCheckpointInfo.TryGetValue(relativePath, out var storageId) && _database.IsImmutable(file) && _fileSystem.FileExists(incrementalCheckpointFile))
{
// File was present in last checkpoint. Just add it to the new incremental checkpoint info
await _storage.TouchBlobAsync(context, file, storageId, isUploader : true).ThrowIfFailure();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploadSkipped].Increment();
}
else
{
// File is new or mutable. Need to add to storage and update local incremental checkpoint
await HardlinkWithFallBackAsync(context, file, incrementalCheckpointFile);
storageId = await _storage.UploadFileAsync(context, file, incrementalCheckpointsPrefix + file.FileName).ThrowIfFailureAsync();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploaded].Increment();
}
}
// Finalize by writing the checkpoint info into the incremental checkpoint directory and updating checkpoint registry and storage
WriteCheckpointInfo(_incrementalCheckpointInfoFile, newCheckpointInfo);
var checkpointId = await _storage.UploadFileAsync(context, _incrementalCheckpointInfoFile, incrementalCheckpointsPrefix + _incrementalCheckpointInfoFile.FileName, garbageCollect : true).ThrowIfFailureAsync();
// Add incremental suffix so consumer knows that the checkpoint is an incremental checkpoint
checkpointId += IncrementalCheckpointIdSuffix;
await _checkpointRegistry.RegisterCheckpointAsync(context, checkpointId, sequencePoint).ThrowIfFailure();
UpdateIncrementalCheckpointInfo(newCheckpointInfo);
successfullyUpdatedIncrementalState = true;
return(successfullyUpdatedIncrementalState);
}
