public async Task <BoolResult> OnChangeCheckpointAsync(OperationContext context, CheckpointState initialState, CheckpointManifest manifest) { _activeCheckpointInfo = _activeCheckpointInfo with { Manifest = manifest }; if (initialState.FileName == null) { return(BoolResult.Success); } return(await context.PerformOperationAsync( Tracer, async() => { var(state, index) = await _storage.ReadModifyWriteAsync <CheckpointState, int>(context, initialState.FileName.Value, state => { var index = state.Consumers.Count; var updated = state.Consumers.TryAdd(_primaryMachineLocation); if (!updated) { index = state.Consumers.IndexOf(_primaryMachineLocation); } return (state, index, updated); }, defaultValue: () => initialState).ThrowIfFailureAsync(); var locations = GetCandidateLocations(state, index); _activeCheckpointInfo = new CheckpointInfoSnapshot(manifest, locations); return Result.Success(index); },
public Task <BoolResult> ClearCheckpointsAsync(OperationContext context) { return(context.PerformOperationAsync( Tracer, async() => await GarbageCollectAsync(context, retentionLimit: 0), traceOperationStarted: false)); }
private Task WriteBatchAsync(string[] logEventInfos) { return(_context.PerformOperationAsync(Tracer, async() => { // TODO: retry policy for writing to file? var blobName = GenerateBlobName(); var stagingLogFilePath = _configuration.StagingFolderPath / blobName; var logFile = (await WriteLogsToFileAsync(_context, stagingLogFilePath, logEventInfos).ThrowIfFailure()).Value; var uploadLogFilePath = _configuration.UploadFolderPath / blobName; _fileSystem.MoveFile(stagingLogFilePath, uploadLogFilePath, replaceExisting: true); logFile.Path = uploadLogFilePath; _uploadQueue.Enqueue(logFile); return BoolResult.Success; }, counter: Counters[AzureBlobStorageLogCounters.ProcessBatchCalls])); }
private Task <Result <LogFile> > WriteLogsToFileAsync(OperationContext context, AbsolutePath logFilePath, string[] logs) { return(context.PerformOperationAsync(Tracer, async() => { long compressedSizeBytes = 0; long uncompressedSizeBytes = 0; using (Stream fileStream = await _fileSystem.OpenSafeAsync( logFilePath, FileAccess.Write, FileMode.CreateNew, FileShare.None, FileOptions.SequentialScan | FileOptions.Asynchronous)) { // We need to make sure we close the compression stream before we take the fileStream's // position, because the compression stream won't write everything until it's been closed, // which leads to bad recorded values in compressedSizeBytes. using (var gzipStream = new GZipStream(fileStream, CompressionLevel.Fastest, leaveOpen: true)) { using var recordingStream = new CountingStream(gzipStream); using var streamWriter = new StreamWriter(recordingStream, Encoding.UTF8, bufferSize: 32 * 1024, leaveOpen: true); if (OnFileOpen != null) { await OnFileOpen(streamWriter); } foreach (var log in logs) { await streamWriter.WriteLineAsync(log); } if (OnFileClose != null) { await OnFileClose(streamWriter); } // Needed to ensure the recording stream receives everything it needs to receive await streamWriter.FlushAsync(); uncompressedSizeBytes = recordingStream.BytesWritten; } compressedSizeBytes = fileStream.Position; } Tracer.TrackMetric(context, $"LogLinesWritten", logs.Length); Tracer.TrackMetric(context, $"CompressedBytesWritten", compressedSizeBytes); Tracer.TrackMetric(context, $"UncompressedBytesWritten", uncompressedSizeBytes); return new Result <LogFile>(new LogFile() { Path = logFilePath, UncompressedSizeBytes = uncompressedSizeBytes, CompressedSizeBytes = compressedSizeBytes, }); },
internal void TriggerGarbageCollection(OperationContext context) { context.PerformOperationAsync <BoolResult>(Tracer, () => { return(_gcGate.DeduplicatedOperationAsync( (timeWaiting, currentCount) => GarbageCollectAsync(context, retentionLimit: _configuration.CheckpointLimit), (timeWaiting, currentCount) => BoolResult.SuccessTask, token: context.Token)); }, traceOperationStarted: false).FireAndForget(context); }
private Task <Result <LogFile> > WriteLogsToFileAsync(OperationContext context, AbsolutePath logFilePath, string[] logs) { return(context.PerformOperationAsync(Tracer, async() => { using var fileStream = await _fileSystem.OpenSafeAsync( logFilePath, FileAccess.Write, FileMode.CreateNew, FileShare.None, FileOptions.SequentialScan | FileOptions.Asynchronous); using var gzipStream = new GZipStream(fileStream, CompressionLevel.Fastest, leaveOpen: true); using var recordingStream = new CountingStream(gzipStream); using var streamWriter = new StreamWriter(recordingStream, Encoding.UTF8, bufferSize: 32 * 1024, leaveOpen: true); if (OnFileOpen != null) { await OnFileOpen(streamWriter); } foreach (var log in logs) { await streamWriter.WriteLineAsync(log); } if (OnFileClose != null) { await OnFileClose(streamWriter); } await streamWriter.FlushAsync(); Tracer.TrackMetric(context, $"LogLinesWritten", logs.Length); var compressedSizeBytes = fileStream.Position; Tracer.TrackMetric(context, $"CompressedBytesWritten", compressedSizeBytes); var uncompressedSizeBytes = recordingStream.BytesWritten; Tracer.TrackMetric(context, $"UncompressedBytesWritten", uncompressedSizeBytes); return new Result <LogFile>(new LogFile() { Path = logFilePath, UncompressedSizeBytes = uncompressedSizeBytes, CompressedSizeBytes = compressedSizeBytes, }); },
public Task <Result <MachineState> > UpdateClusterStateAsync( OperationContext context, MachineState machineState = MachineState.Unknown, ClusterState?clusterState = null, Role?currentRole = null) { clusterState ??= ClusterState; // Due to initialization issues the instance level ClusterState still can be null. if (clusterState is null) { return(Task.FromResult(Result.FromErrorMessage <MachineState>("Failed to update cluster state because the existing cluster state is null."))); } var startMaxMachineId = clusterState.MaxMachineId; int postDbMaxMachineId = startMaxMachineId; int postGlobalMaxMachineId = startMaxMachineId; return(context.PerformOperationAsync( Tracer, async() => { var updateResult = await UpdateClusterStateCoreAsync(context, clusterState, machineState); postGlobalMaxMachineId = clusterState.MaxMachineId; if (currentRole == Role.Master && _configuration.UseBinManager) { Tracer.Info(context, $"Initializing bin manager"); clusterState.InitializeBinManagerIfNeeded(locationsPerBin: _configuration.ProactiveCopyLocationsThreshold, _clock, expiryTime: _configuration.PreferredLocationsExpiryTime); } return updateResult; }, extraEndMessage: result => $"[MaxMachineId=({startMaxMachineId} -> (Db={postDbMaxMachineId}, Global={postGlobalMaxMachineId}))]")); }