private async Task UploadOrTouchFileAsync(OperationContext context, AbsolutePath file, ConcurrentDictionary <string, string> newCheckpointInfo, string incrementalCheckpointsPrefix)
{
var relativePath = file.Path.Substring(_checkpointStagingDirectory.Path.Length + 1);
var incrementalCheckpointFile = _incrementalCheckpointDirectory / relativePath;
if (_incrementalCheckpointInfo.TryGetValue(relativePath, out var storageId) && _database.IsImmutable(file) &&
_fileSystem.FileExists(incrementalCheckpointFile))
{
// File was present in last checkpoint. Just add it to the new incremental checkpoint info
await _storage.TouchBlobAsync(context, file, storageId, isUploader : true).ThrowIfFailure();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploadSkipped].Increment();
}
else
{
// File is new or mutable. Need to add to storage and update local incremental checkpoint
await HardlinkWithFallBackAsync(context, file, incrementalCheckpointFile);
storageId = await _storage.UploadFileAsync(context, file, incrementalCheckpointsPrefix + file.FileName).ThrowIfFailureAsync();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploaded].Increment();
}
}
private async Task <bool> CreateCheckpointIncrementalAsync(OperationContext context, EventSequencePoint sequencePoint, bool successfullyUpdatedIncrementalState)
{
InitializeIncrementalCheckpointIfNeeded();
BoolResult result = BoolResult.Success;
var incrementalCheckpointsPrefix = $"incrementalCheckpoints/{sequencePoint.SequenceNumber}.{Guid.NewGuid()}.";
var newCheckpointInfo = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
// Get files in checkpoint and apply changes to the incremental checkpoint directory (locally and in blob storage)
var files = _fileSystem.EnumerateFiles(_checkpointStagingDirectory, EnumerateOptions.Recurse).Select(s => s.FullPath).ToList();
foreach (var file in files)
{
var relativePath = file.Path.Substring(_checkpointStagingDirectory.Path.Length + 1);
var incrementalCheckpointFile = _incrementalCheckpointDirectory / relativePath;
if (_incrementalCheckpointInfo.TryGetValue(relativePath, out var storageId) && _database.IsImmutable(file) && _fileSystem.FileExists(incrementalCheckpointFile))
{
// File was present in last checkpoint. Just add it to the new incremental checkpoint info
await _storage.TouchBlobAsync(context, file, storageId, isUploader : true).ThrowIfFailure();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploadSkipped].Increment();
}
else
{
// File is new or mutable. Need to add to storage and update local incremental checkpoint
await HardlinkWithFallBackAsync(context, file, incrementalCheckpointFile);
storageId = await _storage.UploadFileAsync(context, file, incrementalCheckpointsPrefix + file.FileName).ThrowIfFailureAsync();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploaded].Increment();
}
}
// Finalize by writing the checkpoint info into the incremental checkpoint directory and updating checkpoint registry and storage
WriteCheckpointInfo(_incrementalCheckpointInfoFile, newCheckpointInfo);
var checkpointId = await _storage.UploadFileAsync(context, _incrementalCheckpointInfoFile, incrementalCheckpointsPrefix + _incrementalCheckpointInfoFile.FileName, garbageCollect : true).ThrowIfFailureAsync();
// Add incremental suffix so consumer knows that the checkpoint is an incremental checkpoint
checkpointId += IncrementalCheckpointIdSuffix;
await _checkpointRegistry.RegisterCheckpointAsync(context, checkpointId, sequencePoint).ThrowIfFailure();
UpdateIncrementalCheckpointInfo(newCheckpointInfo);
successfullyUpdatedIncrementalState = true;
return(successfullyUpdatedIncrementalState);
}
protected override async Task <BoolResult> TouchBlobCoreAsync(OperationContext context, AbsolutePath file, string storageId, bool isUploader, bool isImmutable)
{
var(hash, fallbackStorageId) = ParseCompositeStorageId(storageId);
// Need to touch in fallback storage as well so it knows the content is still in use
var touchTask = _fallbackStorage.TouchBlobAsync(context, file, fallbackStorageId, isUploader, isImmutable).ThrowIfFailure();
// Ensure content is present in private CAS and registered
var registerTask = PutAndRegisterFileAsync(context, file, hash, isImmutable);
await Task.WhenAll(touchTask, registerTask);
return(BoolResult.Success);
}