public async Task <BoolResult> OnChangeCheckpointAsync(OperationContext context, CheckpointState initialState, CheckpointManifest manifest)
{
_activeCheckpointInfo = _activeCheckpointInfo with
{
Manifest = manifest
};
if (initialState.FileName == null)
{
return(BoolResult.Success);
}
return(await context.PerformOperationAsync(
Tracer,
async() =>
{
var(state, index) = await _storage.ReadModifyWriteAsync <CheckpointState, int>(context, initialState.FileName.Value, state =>
{
var index = state.Consumers.Count;
var updated = state.Consumers.TryAdd(_primaryMachineLocation);
if (!updated)
{
index = state.Consumers.IndexOf(_primaryMachineLocation);
}
return (state, index, updated);
},
defaultValue: () => initialState).ThrowIfFailureAsync();
var locations = GetCandidateLocations(state, index);
_activeCheckpointInfo = new CheckpointInfoSnapshot(manifest, locations);
return Result.Success(index);
},
public Task <BoolResult> ClearCheckpointsAsync(OperationContext context)
{
return(context.PerformOperationAsync(
Tracer,
async() => await GarbageCollectAsync(context, retentionLimit: 0),
traceOperationStarted: false));
}
private Task WriteBatchAsync(string[] logEventInfos)
{
return(_context.PerformOperationAsync(Tracer, async() =>
{
// TODO: retry policy for writing to file?
var blobName = GenerateBlobName();
var stagingLogFilePath = _configuration.StagingFolderPath / blobName;
var logFile = (await WriteLogsToFileAsync(_context, stagingLogFilePath, logEventInfos).ThrowIfFailure()).Value;
var uploadLogFilePath = _configuration.UploadFolderPath / blobName;
_fileSystem.MoveFile(stagingLogFilePath, uploadLogFilePath, replaceExisting: true);
logFile.Path = uploadLogFilePath;
_uploadQueue.Enqueue(logFile);
return BoolResult.Success;
},
counter: Counters[AzureBlobStorageLogCounters.ProcessBatchCalls]));
}
private Task <Result <LogFile> > WriteLogsToFileAsync(OperationContext context, AbsolutePath logFilePath, string[] logs)
{
return(context.PerformOperationAsync(Tracer, async() =>
{
long compressedSizeBytes = 0;
long uncompressedSizeBytes = 0;
using (Stream fileStream = await _fileSystem.OpenSafeAsync(
logFilePath,
FileAccess.Write,
FileMode.CreateNew,
FileShare.None,
FileOptions.SequentialScan | FileOptions.Asynchronous))
{
// We need to make sure we close the compression stream before we take the fileStream's
// position, because the compression stream won't write everything until it's been closed,
// which leads to bad recorded values in compressedSizeBytes.
using (var gzipStream = new GZipStream(fileStream, CompressionLevel.Fastest, leaveOpen: true))
{
using var recordingStream = new CountingStream(gzipStream);
using var streamWriter = new StreamWriter(recordingStream, Encoding.UTF8, bufferSize: 32 * 1024, leaveOpen: true);
if (OnFileOpen != null)
{
await OnFileOpen(streamWriter);
}
foreach (var log in logs)
{
await streamWriter.WriteLineAsync(log);
}
if (OnFileClose != null)
{
await OnFileClose(streamWriter);
}
// Needed to ensure the recording stream receives everything it needs to receive
await streamWriter.FlushAsync();
uncompressedSizeBytes = recordingStream.BytesWritten;
}
compressedSizeBytes = fileStream.Position;
}
Tracer.TrackMetric(context, $"LogLinesWritten", logs.Length);
Tracer.TrackMetric(context, $"CompressedBytesWritten", compressedSizeBytes);
Tracer.TrackMetric(context, $"UncompressedBytesWritten", uncompressedSizeBytes);
return new Result <LogFile>(new LogFile()
{
Path = logFilePath,
UncompressedSizeBytes = uncompressedSizeBytes,
CompressedSizeBytes = compressedSizeBytes,
});
},
internal void TriggerGarbageCollection(OperationContext context)
{
context.PerformOperationAsync <BoolResult>(Tracer, () =>
{
return(_gcGate.DeduplicatedOperationAsync(
(timeWaiting, currentCount) => GarbageCollectAsync(context, retentionLimit: _configuration.CheckpointLimit),
(timeWaiting, currentCount) => BoolResult.SuccessTask,
token: context.Token));
},
traceOperationStarted: false).FireAndForget(context);
}
private Task <Result <LogFile> > WriteLogsToFileAsync(OperationContext context, AbsolutePath logFilePath, string[] logs)
{
return(context.PerformOperationAsync(Tracer, async() =>
{
using var fileStream = await _fileSystem.OpenSafeAsync(
logFilePath,
FileAccess.Write,
FileMode.CreateNew,
FileShare.None,
FileOptions.SequentialScan | FileOptions.Asynchronous);
using var gzipStream = new GZipStream(fileStream, CompressionLevel.Fastest, leaveOpen: true);
using var recordingStream = new CountingStream(gzipStream);
using var streamWriter = new StreamWriter(recordingStream, Encoding.UTF8, bufferSize: 32 * 1024, leaveOpen: true);
if (OnFileOpen != null)
{
await OnFileOpen(streamWriter);
}
foreach (var log in logs)
{
await streamWriter.WriteLineAsync(log);
}
if (OnFileClose != null)
{
await OnFileClose(streamWriter);
}
await streamWriter.FlushAsync();
Tracer.TrackMetric(context, $"LogLinesWritten", logs.Length);
var compressedSizeBytes = fileStream.Position;
Tracer.TrackMetric(context, $"CompressedBytesWritten", compressedSizeBytes);
var uncompressedSizeBytes = recordingStream.BytesWritten;
Tracer.TrackMetric(context, $"UncompressedBytesWritten", uncompressedSizeBytes);
return new Result <LogFile>(new LogFile()
{
Path = logFilePath,
UncompressedSizeBytes = uncompressedSizeBytes,
CompressedSizeBytes = compressedSizeBytes,
});
},
public Task <Result <MachineState> > UpdateClusterStateAsync(
OperationContext context,
MachineState machineState = MachineState.Unknown,
ClusterState?clusterState = null,
Role?currentRole = null)
{
clusterState ??= ClusterState;
// Due to initialization issues the instance level ClusterState still can be null.
if (clusterState is null)
{
return(Task.FromResult(Result.FromErrorMessage <MachineState>("Failed to update cluster state because the existing cluster state is null.")));
}
var startMaxMachineId = clusterState.MaxMachineId;
int postDbMaxMachineId = startMaxMachineId;
int postGlobalMaxMachineId = startMaxMachineId;
return(context.PerformOperationAsync(
Tracer,
async() =>
{
var updateResult = await UpdateClusterStateCoreAsync(context, clusterState, machineState);
postGlobalMaxMachineId = clusterState.MaxMachineId;
if (currentRole == Role.Master && _configuration.UseBinManager)
{
Tracer.Info(context, $"Initializing bin manager");
clusterState.InitializeBinManagerIfNeeded(locationsPerBin: _configuration.ProactiveCopyLocationsThreshold, _clock, expiryTime: _configuration.PreferredLocationsExpiryTime);
}
return updateResult;
},
extraEndMessage: result => $"[MaxMachineId=({startMaxMachineId} -> (Db={postDbMaxMachineId}, Global={postGlobalMaxMachineId}))]"));
}