public async Task CreateBatchAndUpdateTimeouts(RavenBatch batch)
{
var commands = batch.TimeoutIds.Select(timeoutId =>
{
return(new Raven3BatchCommand
{
Key = timeoutId,
Method = "EVAL",
DebugMode = false,
Patch = new Patch()
{
Script = $"this.OwningTimeoutManager = '{RavenConstants.MigrationOngoingPrefix}' + this.OwningTimeoutManager;",
Values = new { }
}
});
}).Cast <object>().ToList();
commands.Add(new
{
Method = "PUT",
Key = $"{RavenConstants.BatchPrefix}/{batch.Number}",
Document = batch,
Metadata = new object()
});
await PostToBulkDocs(commands);
}
public async Task CreateBatchAndUpdateTimeouts(RavenBatch batch)
{
var insertCommand = new PutCommand
{
Id = $"{RavenConstants.BatchPrefix}/{batch.Number}",
Type = "PUT",
ChangeVector = null,
Document = batch
};
var timeoutUpdateCommands = batch.TimeoutIds.Select(timeoutId => new PatchCommand
{
Id = timeoutId,
Type = "PATCH",
ChangeVector = null,
Patch = new Patch()
{
Script = $"this.OwningTimeoutManager = '{RavenConstants.MigrationOngoingPrefix}' + this.OwningTimeoutManager;",
Values = new { }
}
}).ToList();
var commands = new List <object>
{
insertCommand
};
commands.AddRange(timeoutUpdateCommands);
await PostToBulkDocs(commands);
}
public async Task DeleteBatchAndUpdateTimeouts(RavenBatch batch)
{
var deleteCommand = GetDeleteCommand($"{RavenConstants.BatchPrefix}/{batch.Number}");
var commands = batch.TimeoutIds.Select(timeoutId =>
{
return(new Raven3BatchCommand
{
Key = timeoutId,
Method = "EVAL",
DebugMode = false,
Patch = new Patch()
{
Script = $"this.OwningTimeoutManager = this.OwningTimeoutManager.substr({RavenConstants.MigrationOngoingPrefix.Length});",
Values = new { }
}
});
}).ToList();
commands.Add(deleteCommand);
await PostToBulkDocs(commands);
}
public async Task DeleteBatchAndUpdateTimeouts(RavenBatch batch)
{
var deleteCommand = GetDeleteCommand($"{RavenConstants.BatchPrefix}/{batch.Number}");
var timeoutUpdateCommands = batch.TimeoutIds.Select(timeoutId => new PatchCommand
{
Id = timeoutId,
Type = "PATCH",
ChangeVector = null,
Patch = new Patch()
{
Script = $"this.OwningTimeoutManager = this.OwningTimeoutManager.substr({RavenConstants.MigrationOngoingPrefix.Length});",
Values = new { }
}
}).ToList();
var commands = new List <object>
{
deleteCommand
};
commands.AddRange(timeoutUpdateCommands);
await PostToBulkDocs(commands);
}
async Task <List <RavenBatch> > PrepareBatchesWithIndexUsage(DateTime cutoffTime, string endpointName)
{
var batches = new List <RavenBatch>();
var batchesExisted = false;
var timeoutIds = new Dictionary <string, string>();
var nrOfTimeoutsRetrieved = 0;
var initialIndexEtag = string.Empty;
var nrOfTimeouts = 0;
var tcs = new CancellationTokenSource();
var printTask = Task.Run(
async() =>
{
while (!tcs.Token.IsCancellationRequested)
{
await Task.Delay(TimeSpan.FromSeconds(5), tcs.Token);
logger.LogInformation($"{nrOfTimeoutsRetrieved} of {nrOfTimeouts} have been scanned.");
}
}, CancellationToken.None);
bool DoesNotExistInBatchesFilter(TimeoutData td)
{
return(batches.SelectMany(x => x.TimeoutIds).All(t => t != td.Id));
}
bool ElegibleFilter(TimeoutData td)
{
return(td.OwningTimeoutManager.ToLower() == endpointName.ToLower() && td.Time >= cutoffTime && !td.OwningTimeoutManager.StartsWith(
RavenConstants.MigrationDonePrefix,
StringComparison.OrdinalIgnoreCase) && !timeoutIds.ContainsKey(td.Id));
}
var findMoreTimeouts = true;
var existingBatches = await ravenAdapter.GetDocuments <RavenBatch>(batch => true, RavenConstants.BatchPrefix);
if (existingBatches.Any())
{
var batch = existingBatches.First();
if (batch.CutoffDate != cutoffTime || batch.EndpointName.ToLower() != endpointName.ToLower())
{
throw new Exception($"Found remaining batches from previous run, using a different cutoff date or endpoint than current run. Please abort the previous migration with parameters Cutoffdate = {batch.CutoffDate} and EndpointName = {batch.EndpointName}");
}
logger.LogInformation($"Found existing batches, resuming prepare for endpoint {endpointName}");
batches.AddRange(existingBatches);
batchesExisted = true;
}
while (findMoreTimeouts)
{
var startFrom = nrOfTimeoutsRetrieved;
var timeoutsResult = await ravenAdapter.GetDocumentsByIndex <TimeoutData>(startFrom, TimeSpan.FromSeconds(5), (doc, id) => doc.Id = id);
if (string.IsNullOrEmpty(initialIndexEtag))
{
initialIndexEtag = timeoutsResult.IndexETag;
nrOfTimeouts = timeoutsResult.NrOfDocuments;
}
if (timeoutsResult.IsStale || timeoutsResult.IndexETag != initialIndexEtag)
{
throw new Exception("We ran into a stale index while trying to prepare the batches to migrate. This means that there are still endpoints running that are using the legacy Timeout Manager connected to this storage. Please shut them down and try again.");
}
nrOfTimeoutsRetrieved += timeoutsResult.Documents.Count;
var eligibleTimeouts = timeoutsResult.Documents.Where(ElegibleFilter).ToList();
if (batchesExisted)
{
eligibleTimeouts = eligibleTimeouts.Where(DoesNotExistInBatchesFilter).ToList();
}
var timeouts = eligibleTimeouts.Select(x => x.Id);
foreach (var timeoutId in timeouts)
{
timeoutIds.Add(timeoutId, null);
}
if (timeoutsResult.Documents.Count == 0)
{
findMoreTimeouts = false;
}
}
tcs.Cancel();
try
{
await printTask;
}
catch (OperationCanceledException)
{
}
var timeoutsToProcess = timeoutIds.Select(x => x.Key).ToList();
var nrOfBatches = Math.Ceiling(timeoutsToProcess.Count / (decimal)RavenConstants.DefaultPagingSize);
for (var i = 0; i < nrOfBatches; i++)
{
var timeoutsInThisBatch = timeoutsToProcess.Skip(i * RavenConstants.DefaultPagingSize).Take(RavenConstants.DefaultPagingSize).ToList();
var batch = new RavenBatch(batches.Count + 1, BatchState.Pending, timeoutsInThisBatch.Count())
{
TimeoutIds = timeoutsInThisBatch.ToArray(),
CutoffDate = cutoffTime,
EndpointName = endpointName
};
await ravenAdapter.CreateBatchAndUpdateTimeouts(batch);
logger.LogInformation($"Batch {batch.Number} was created to handle {timeoutsInThisBatch.Count} timeouts");
batches.Add(batch);
}
return(batches);
}
async Task <List <RavenBatch> > PrepareBatchesWithoutIndexUsage(DateTime cutoffTime, string endpointName)
{
var batches = new List <RavenBatch>();
var batchesExisted = false;
bool ElegibleFilter(TimeoutData td)
{
return(td.OwningTimeoutManager.ToLower() == endpointName.ToLower() && td.Time >= cutoffTime && !td.OwningTimeoutManager.StartsWith(
RavenConstants.MigrationDonePrefix,
StringComparison.OrdinalIgnoreCase));
}
bool DoesNotExistInBatchesFilter(TimeoutData td)
{
return(batches.SelectMany(x => x.TimeoutIds).All(t => t != td.Id));
}
var findMoreTimeouts = true;
var iteration = 0;
var nrOfPages = 1;
var existingBatches = await ravenAdapter.GetDocuments <RavenBatch>(batch => true, RavenConstants.BatchPrefix, (batch, idSetter) => { });
if (existingBatches.Any())
{
var batch = existingBatches.First();
if (batch.CutoffDate != cutoffTime || batch.EndpointName.ToLower() != endpointName.ToLower())
{
throw new Exception($"Found remaining batches from previous run, using a different cutoff date or endpoint than current run. Please abort the previous migration with parameters Cutoffdate = {batch.CutoffDate} and EndpointName = {batch.EndpointName}");
}
logger.LogInformation($"Found existing batches, resuming prepare for endpoint {endpointName}");
batches.AddRange(existingBatches);
batchesExisted = true;
}
while (findMoreTimeouts)
{
var startFrom = iteration * nrOfPages * RavenConstants.DefaultPagingSize;
var timeouts = await ravenAdapter.GetPagedDocuments <TimeoutData>(timeoutDocumentPrefix, startFrom, (doc, id) => doc.Id = id, nrOfPages);
var elegibleTimeouts = timeouts.Where(ElegibleFilter).ToList();
if (batchesExisted)
{
elegibleTimeouts = elegibleTimeouts.Where(DoesNotExistInBatchesFilter).ToList();
}
if (elegibleTimeouts.Any())
{
var batch = new RavenBatch(batches.Count + 1, BatchState.Pending, elegibleTimeouts.Count())
{
TimeoutIds = elegibleTimeouts.Select(t => t.Id).ToArray(),
CutoffDate = cutoffTime,
EndpointName = endpointName
};
await ravenAdapter.CreateBatchAndUpdateTimeouts(batch);
logger.LogInformation($"Batch {batch.Number} was created to handle {elegibleTimeouts.Count} timeouts");
batches.Add(batch);
}
if (timeouts.Count == 0)
{
findMoreTimeouts = false;
}
iteration++;
}
return(batches);
}
