/// <nodoc />
private CheckpointState(Role role, DateTime epochStartCursorTime, MachineLocation master)
: this()
{
Role = role;
StartSequencePoint = new EventSequencePoint(epochStartCursorTime);
Master = master;
}
/// <summary>
/// Creates a checkpoint for a given sequence point.
/// </summary>
public Task <BoolResult> CreateCheckpointAsync(OperationContext context, EventSequencePoint sequencePoint)
{
context = context.CreateNested();
return(context.PerformOperationAsync(
_tracer,
async() =>
{
bool successfullyUpdatedIncrementalState = false;
try
{
// Creating a working temporary directory
using (new DisposableDirectory(_fileSystem, _checkpointStagingDirectory))
{
// Saving checkpoint for the database into the temporary folder
_database.SaveCheckpoint(context, _checkpointStagingDirectory).ThrowIfFailure();
if (_configuration.UseIncrementalCheckpointing)
{
successfullyUpdatedIncrementalState = await CreateCheckpointIncrementalAsync(context, sequencePoint, successfullyUpdatedIncrementalState);
}
else
{
await CreateFullCheckpointAsync(context, sequencePoint);
}
return BoolResult.Success;
}
}
finally
{
ClearIncrementalCheckpointStateIfNeeded(successfullyUpdatedIncrementalState);
}
}));
}
public CheckpointState CreateCheckpointState(IClock clock)
{
var index = _index++;
var checkpointId = "chkpt" + _index;
var sequencePoint = new EventSequencePoint(sequenceNumber: index);
return(new CheckpointState(sequencePoint, checkpointId, clock.UtcNow, M1));
}
/// <nodoc />
public CheckpointState(Role role, EventSequencePoint startSequencePoint, string checkpointId, DateTime checkpointTime)
{
Contract.Requires(!string.IsNullOrEmpty(checkpointId));
Role = role;
StartSequencePoint = startSequencePoint;
CheckpointId = checkpointId;
CheckpointTime = checkpointTime;
}
/// <nodoc />
public CheckpointState(Role role, EventSequencePoint startSequencePoint, string checkpointId, DateTime checkpointTime, MachineLocation producer)
{
Contract.Requires(!string.IsNullOrEmpty(checkpointId));
Role = role;
StartSequencePoint = startSequencePoint;
CheckpointId = checkpointId;
CheckpointTime = checkpointTime;
Producer = producer;
}
private bool IsBefore(EventData eventData, EventSequencePoint sequencePoint)
{
if (sequencePoint.SequenceNumber != null)
{
return(eventData.SystemProperties.SequenceNumber < sequencePoint.SequenceNumber.Value);
}
else
{
return(eventData.SystemProperties.EnqueuedTimeUtc < sequencePoint.EventStartCursorTimeUtc.Value);
}
}
/// <summary>
/// Creates a checkpoint for a given sequence point.
/// </summary>
public Task <BoolResult> CreateCheckpointAsync(OperationContext context, EventSequencePoint sequencePoint)
{
context = context.CreateNested();
string checkpointId = "Unknown";
long checkpointSize = 0;
return(context.PerformOperationAsync(
_tracer,
async() =>
{
bool successfullyUpdatedIncrementalState = false;
try
{
// Creating a working temporary directory
using (new DisposableDirectory(_fileSystem, _checkpointStagingDirectory))
{
// NOTE(jubayard): this needs to be done previous to checkpointing, because we always
// fetch the latest version's size in this way. This implies there may be some difference
// between the reported value and the actual size on disk: updates will get in in-between.
// The better alternative is to actually open the checkpoint and ask, but it seems like too
// much.
checkpointSize = _database.GetContentDatabaseSizeBytes().GetValueOrDefault(-1);
// Saving checkpoint for the database into the temporary folder
_database.SaveCheckpoint(context, _checkpointStagingDirectory).ThrowIfFailure();
if (_configuration.UseIncrementalCheckpointing)
{
checkpointId = await CreateCheckpointIncrementalAsync(context, sequencePoint);
successfullyUpdatedIncrementalState = true;
}
else
{
checkpointId = await CreateFullCheckpointAsync(context, sequencePoint);
}
return BoolResult.Success;
}
}
finally
{
ClearIncrementalCheckpointStateIfNeeded(context, successfullyUpdatedIncrementalState);
}
},
extraStartMessage: $"SequencePoint=[{sequencePoint}]",
extraEndMessage: result => $"SequencePoint=[{sequencePoint}] Id=[{checkpointId}] SizeMb=[{(checkpointSize < 0 ? checkpointSize:checkpointSize*1e-6)}]"));
}
/// <inheritdoc />
public BoolResult StartProcessing(OperationContext context, EventSequencePoint sequencePoint, IPartitionReceiveHandler processor)
{
using (_lock.AcquireWriteLock())
{
_handler = ev => Dispatch(ev, processor);
var events = _hub.SubscribeAndGetEventsStartingAtSequencePoint(sequencePoint, _handler);
foreach (var eventData in events)
{
_handler(eventData);
}
}
return(BoolResult.Success);
}
/// <inheritdoc />
protected override BoolResult DoStartProcessing(OperationContext context, EventSequencePoint sequencePoint)
{
using (_lock.AcquireWriteLock())
{
_processing = true;
while (_queue.TryTake(out var eventData))
{
DispatchAsync(context, eventData).GetAwaiter().GetResult();
Interlocked.Increment(ref _sequenceNumber);
}
}
return(BoolResult.Success);
}
public Task ReconciliationOverRealStorage()
{
var checkpointsKey = Guid.NewGuid().ToString();
// Copy and paste a real connection string here.
var storageConnectionString = string.Empty;
// Consider updating this directory if you want to keep data between invocations.
var workingDirectory = TestRootDirectoryPath;
var configuration = new LocalDiskCentralStoreConfiguration(
workingDirectory,
checkpointsKey);
var blobStoreConfiguration = new BlobCentralStoreConfiguration(
credentials: new AzureBlobStorageCredentials(storageConnectionString),
containerName: "checkpoints",
checkpointsKey: checkpointsKey);
var producerMachineLocation = new MachineLocation();
ConfigureWithOneMaster(s =>
{
s.ReconcileMode = ReconciliationMode.Once.ToString();
s.AzureStorageSecretName = Host.StoreSecret("StorageName", storageConnectionString);
});
return(RunTestAsync(
new Context(Logger),
2,
async context =>
{
var master = context.GetMaster();
var worker = context.GetFirstWorker();
var workerId = worker.LocalLocationStore.ClusterState.PrimaryMachineId;
var workerSession = context.Sessions[context.GetFirstWorkerIndex()];
var checkpointState = new CheckpointState(
Role.Worker,
EventSequencePoint.Parse("24382354"),
"MD5:8C4856EA13F6AD59B65D8F6781D2A2F9||DCS||incrementalCheckpoints/24382354.10a0ca0f-d63f-4992-a088-f67bd00abd8a.checkpointInfo.txt|Incremental",
DateTime.Now,
producerMachineLocation,
producerMachineLocation);
// Next heartbeat workers to restore checkpoint
await worker.LocalLocationStore.ProcessStateAsync(new OperationContext(context), checkpointState, inline: true, forceRestore: true).ShouldBeSuccess();
var reconcileResult = await worker.ReconcileAsync(context).ShouldBeSuccess();
Output.WriteLine($"Reconcile result: {reconcileResult}");
}));
}
private async Task CreateFullCheckpointAsync(OperationContext context, EventSequencePoint sequencePoint)
{
// Zipping the checkpoint
var targetZipFile = _checkpointStagingDirectory + ".zip";
File.Delete(targetZipFile);
ZipFile.CreateFromDirectory(_checkpointStagingDirectory.ToString(), targetZipFile);
// Track checkpoint size
var fileInfo = new System.IO.FileInfo(targetZipFile);
_tracer.TrackMetric(context, CheckpointSizeMetricName, fileInfo.Length);
var checkpointBlobName = $"checkpoints/{sequencePoint.SequenceNumber}.{Guid.NewGuid()}.zip";
var checkpointId = await _storage.UploadFileAsync(context, new AbsolutePath(targetZipFile), checkpointBlobName, garbageCollect : true).ThrowIfFailureAsync();
// Uploading the checkpoint
await _checkpointRegistry.RegisterCheckpointAsync(context, checkpointId, sequencePoint).ThrowIfFailure();
}
public Task ShutdownDoesNotProcessEnqueuedEvents()
{
// This test is "weird" in the sense that it relies on a "controlled" race condition. We need to check that
// the master does not wait for events to finish processing to complete its shutdown. The way we do this is
// force an arbitrary delay when processing events, and then make sure that we haven't processed any after
// shutdown.
//
// * If we do it with a cancellation token on shutdown started, then we have a race condition, because the
// token will be triggered, and the item could be processed before shutdown finishes.
// * If we do it with a separate lock, we'll need some sort of delay which will be equivalent to what we do
// now.
// * We can't use a cancellation token on shutdown finished, because shutdown awaits for the action blocks
// to complete, and this means we have a deadlock.
//
// By doing things this way, we are relying on the fact that it is unlikely for a thread to not run for a
// second. If we assume that's true, then the slowdown will resume after shutdown has started waiting, and
// we will do the appropriate fast-return path.
var configuration = new SlowedContentLocationEventStoreConfiguration()
{
Slowdown = TimeSpan.FromSeconds(1)
};
var eventHandler = new TestEventHandler();
return(WithContentLocationEventStore(async(tracingContext, clock, fileSystem, eventStore) =>
{
var context = new OperationContext(tracingContext);
var sequencePoint = new EventSequencePoint(clock.UtcNow);
eventStore.StartProcessing(context, sequencePoint).ShouldBeSuccess();
eventStore.AddLocations(context, MachineId.FromIndex(0), new[] { new ContentHashWithSize(ContentHash.Random(), 1) }).ShouldBeSuccess();
(await eventStore.ShutdownAsync(context)).ShouldBeSuccess();
eventHandler.EventsHandled.Should().Be(0);
}, configuration, eventHandler));
}
public Task <BoolResult> RegisterCheckpointAsync(OperationContext context, string checkpointId, EventSequencePoint sequencePoint)
{
return(context.PerformOperationWithTimeoutAsync(
Tracer,
async nestedContext =>
{
Contract.Assert(sequencePoint.SequenceNumber != null);
var checkpoint = new RedisCheckpointInfo(checkpointId, sequencePoint.SequenceNumber.Value, _clock.UtcNow, Configuration.PrimaryMachineLocation.ToString());
Tracer.Debug(nestedContext, $"Saving checkpoint '{checkpoint}' into the central store.");
var slotNumber = await _checkpointsKey.UseNonConcurrentReplicatedHashAsync(
nestedContext,
Configuration.RetryWindow,
RedisOperation.UploadCheckpoint,
(batch, key) => batch.AddCheckpointAsync(key, checkpoint, MaxCheckpointSlotCount),
timeout: Configuration.ClusterRedisOperationTimeout)
.ThrowIfFailureAsync();
Tracer.Debug(nestedContext, $"Saved checkpoint into slot '{slotNumber}'.");
return BoolResult.Success;
},
Counters[GlobalStoreCounters.RegisterCheckpoint],
timeout: Configuration.ClusterRedisOperationTimeout));
}
/// <summary>
/// Creates a checkpoint for a given sequence point.
/// </summary>
public Task <BoolResult> CreateCheckpointAsync(OperationContext context, EventSequencePoint sequencePoint)
{
context = context.CreateNested(nameof(CheckpointManager));
string checkpointId = "Unknown";
double contentColumnFamilySizeMb = -1;
double contentDataSizeMb = -1;
double metadataColumnFamilySizeMb = -1;
double metadataDataSizeMb = -1;
double sizeOnDiskMb = -1;
return(context.PerformOperationAsync(
_tracer,
async() =>
{
bool successfullyUpdatedIncrementalState = false;
try
{
// Creating a working temporary directory
using (new DisposableDirectory(_fileSystem, _checkpointStagingDirectory))
{
// Write out the time this checkpoint was generated to the database. This will be used by
// the workers in order to determine whether they should restore or not after restart. The
// checkpoint id is generated inside the upload methods, so we only generate the guid here.
// Since this is only used for reporting purposes, there's no harm in it.
var checkpointGuid = Guid.NewGuid();
DatabaseWriteCheckpointCreationTime(context, checkpointGuid.ToString(), DateTime.UtcNow);
// NOTE(jubayard): this needs to be done previous to checkpointing, because we always
// fetch the latest version's size in this way. This implies there may be some difference
// between the reported value and the actual size on disk: updates will get in in-between.
// The better alternative is to actually open the checkpoint and ask, but it seems like too
// much.
if (_database is RocksDbContentLocationDatabase rocksDb)
{
contentColumnFamilySizeMb = rocksDb.GetLongProperty(
RocksDbContentLocationDatabase.LongProperty.LiveFilesSizeBytes,
RocksDbContentLocationDatabase.Entity.ContentTracking).Select(x => x * 1e-6).GetValueOrDefault(-1);
contentDataSizeMb = rocksDb.GetLongProperty(
RocksDbContentLocationDatabase.LongProperty.LiveDataSizeBytes,
RocksDbContentLocationDatabase.Entity.ContentTracking).Select(x => x * 1e-6).GetValueOrDefault(-1);
metadataColumnFamilySizeMb = rocksDb.GetLongProperty(
RocksDbContentLocationDatabase.LongProperty.LiveFilesSizeBytes,
RocksDbContentLocationDatabase.Entity.Metadata).Select(x => x * 1e-6).GetValueOrDefault(-1);
metadataDataSizeMb = rocksDb.GetLongProperty(
RocksDbContentLocationDatabase.LongProperty.LiveDataSizeBytes,
RocksDbContentLocationDatabase.Entity.Metadata).Select(x => x * 1e-6).GetValueOrDefault(-1);
}
// Saving checkpoint for the database into the temporary folder
_database.SaveCheckpoint(context, _checkpointStagingDirectory).ThrowIfFailure();
try
{
sizeOnDiskMb = _fileSystem
.EnumerateFiles(_checkpointStagingDirectory, EnumerateOptions.Recurse)
.Sum(fileInfo => fileInfo.Length) * 1e-6;
}
catch (IOException e)
{
_tracer.Error(context, $"Error counting size of checkpoint's staging directory `{_checkpointStagingDirectory}`: {e}");
}
if (_configuration.UseIncrementalCheckpointing)
{
checkpointId = await CreateCheckpointIncrementalAsync(context, sequencePoint, checkpointGuid);
successfullyUpdatedIncrementalState = true;
}
else
{
checkpointId = await CreateFullCheckpointAsync(context, sequencePoint, checkpointGuid);
}
return BoolResult.Success;
}
}
finally
{
ClearIncrementalCheckpointStateIfNeeded(context, successfullyUpdatedIncrementalState);
}
},
private async Task <string> CreateCheckpointIncrementalAsync(OperationContext context, EventSequencePoint sequencePoint)
{
InitializeIncrementalCheckpointIfNeeded(restoring: false);
var incrementalCheckpointsPrefix = $"incrementalCheckpoints/{sequencePoint.SequenceNumber}.{Guid.NewGuid()}.";
// See the comment of _incrementalCheckpointInfo for the meaning of keys and values.
var newCheckpointInfo = new ConcurrentDictionary <string, string>(StringComparer.OrdinalIgnoreCase);
// Get files in checkpoint and apply changes to the incremental checkpoint directory (locally and in blob storage)
var files = _fileSystem.EnumerateFiles(_checkpointStagingDirectory, EnumerateOptions.Recurse).Select(s => s.FullPath).ToList();
await UploadFilesAsync(context, files, newCheckpointInfo, incrementalCheckpointsPrefix);
// Finalize by writing the checkpoint info into the incremental checkpoint directory and updating checkpoint registry and storage
WriteCheckpointInfo(_incrementalCheckpointInfoFile, newCheckpointInfo);
var checkpointId = await _storage.UploadFileAsync(context, _incrementalCheckpointInfoFile, incrementalCheckpointsPrefix + _incrementalCheckpointInfoFile.FileName, garbageCollect : true).ThrowIfFailureAsync();
// Add incremental suffix so consumer knows that the checkpoint is an incremental checkpoint
checkpointId += IncrementalCheckpointIdSuffix;
await _checkpointRegistry.RegisterCheckpointAsync(context, checkpointId, sequencePoint).ThrowIfFailure();
// Have to create a dictionary in .NET 4.5.1 because ConcurrentDictionary does not implement IReadOnlyDictionary there.
UpdateIncrementalCheckpointInfo(new Dictionary <string, string>(newCheckpointInfo));
return(checkpointId);
}
public Task <BoolResult> RegisterCheckpointAsync(OperationContext context, string checkpointId, EventSequencePoint sequencePoint)
{
return(context.PerformOperationAsync(
Tracer,
() =>
{
Contract.Assert(sequencePoint.SequenceNumber != null);
var checkpoint = new RedisCheckpointInfo(checkpointId, sequencePoint.SequenceNumber.Value, _clock.UtcNow, LocalMachineLocation.ToString());
Tracer.Debug(context, $"Saving checkpoint '{checkpoint}' into the central store.");
return ExecuteRedisAsync(context, async redisDb =>
{
var slotNumber = await redisDb.ExecuteBatchAsync(context, batch =>
{
return batch.AddCheckpointAsync(_checkpointsKey, checkpoint, MaxCheckpointSlotCount);
}, RedisOperation.UploadCheckpoint);
Tracer.Debug(context, $"Saved checkpoint into slot '{slotNumber}' on {GetDbName(redisDb)}.");
return BoolResult.Success;
});
},
Counters[GlobalStoreCounters.RegisterCheckpoint]));
}
/// <nodoc />
private CheckpointState(Role role, DateTime epochStartCursorTime)
: this()
{
Role = role;
StartSequencePoint = new EventSequencePoint(epochStartCursorTime);
}
private async Task <bool> CreateCheckpointIncrementalAsync(OperationContext context, EventSequencePoint sequencePoint, bool successfullyUpdatedIncrementalState)
{
InitializeIncrementalCheckpointIfNeeded();
BoolResult result = BoolResult.Success;
var incrementalCheckpointsPrefix = $"incrementalCheckpoints/{sequencePoint.SequenceNumber}.{Guid.NewGuid()}.";
var newCheckpointInfo = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
// Get files in checkpoint and apply changes to the incremental checkpoint directory (locally and in blob storage)
var files = _fileSystem.EnumerateFiles(_checkpointStagingDirectory, EnumerateOptions.Recurse).Select(s => s.FullPath).ToList();
foreach (var file in files)
{
var relativePath = file.Path.Substring(_checkpointStagingDirectory.Path.Length + 1);
var incrementalCheckpointFile = _incrementalCheckpointDirectory / relativePath;
if (_incrementalCheckpointInfo.TryGetValue(relativePath, out var storageId) && _database.IsImmutable(file) && _fileSystem.FileExists(incrementalCheckpointFile))
{
// File was present in last checkpoint. Just add it to the new incremental checkpoint info
await _storage.TouchBlobAsync(context, file, storageId, isUploader : true).ThrowIfFailure();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploadSkipped].Increment();
}
else
{
// File is new or mutable. Need to add to storage and update local incremental checkpoint
await HardlinkWithFallBackAsync(context, file, incrementalCheckpointFile);
storageId = await _storage.UploadFileAsync(context, file, incrementalCheckpointsPrefix + file.FileName).ThrowIfFailureAsync();
newCheckpointInfo[relativePath] = storageId;
Counters[ContentLocationStoreCounters.IncrementalCheckpointFilesUploaded].Increment();
}
}
// Finalize by writing the checkpoint info into the incremental checkpoint directory and updating checkpoint registry and storage
WriteCheckpointInfo(_incrementalCheckpointInfoFile, newCheckpointInfo);
var checkpointId = await _storage.UploadFileAsync(context, _incrementalCheckpointInfoFile, incrementalCheckpointsPrefix + _incrementalCheckpointInfoFile.FileName, garbageCollect : true).ThrowIfFailureAsync();
// Add incremental suffix so consumer knows that the checkpoint is an incremental checkpoint
checkpointId += IncrementalCheckpointIdSuffix;
await _checkpointRegistry.RegisterCheckpointAsync(context, checkpointId, sequencePoint).ThrowIfFailure();
UpdateIncrementalCheckpointInfo(newCheckpointInfo);
successfullyUpdatedIncrementalState = true;
return(successfullyUpdatedIncrementalState);
}
internal IReadOnlyList <EventData> SubscribeAndGetEventsStartingAtSequencePoint(EventSequencePoint sequencePoint, Action <EventData> handler)
{
lock (_syncLock)
{
OnEvent += handler;
return(_eventStream.SkipWhile(eventData => IsBefore(eventData, sequencePoint)).ToArray());
}
}
