private async Task RollbackBatch(Batch batch, bool deleteDataset = false)
{
_logger.LogInformation("Rolling back batch '{BatchToken}'", batch.Token);
// I know we will have concurrency problems here because we could be in the middle of the rollback when another client calls for another one
// To mitigate this issue we are going to commit the status change of the batch as soon as possible
batch.Status = BatchStatus.Rolledback;
batch.End = DateTime.Now;
await _context.SaveChangesAsync();
await _context.Entry(batch).Collection(item => item.Entries).LoadAsync();
await _context.Entry(batch).Reference(item => item.Dataset).LoadAsync();
var ds = batch.Dataset;
await _context.Entry(ds).Reference(item => item.Organization).LoadAsync();
var org = ds.Organization;
if (deleteDataset)
{
await _datasetsManager.Delete(org.Slug, ds.Slug);
}
else
{
foreach (var entry in batch.Entries)
{
await _objectsManager.Delete(org.Slug, ds.Slug, entry.Path);
}
}
}
public async Task <IActionResult> Delete([FromRoute] string orgSlug, [FromRoute] string dsSlug, [FromForm] string path)
{
try
{
_logger.LogDebug("Objects controller Delete('{OrgSlug}', '{DsSlug}', '{Path}')", orgSlug, dsSlug, path);
await _objectsManager.Delete(orgSlug, dsSlug, path);
return(NoContent());
}
catch (Exception ex)
{
_logger.LogError(ex, "Exception in Objects controller Delete('{OrgSlug}', '{DsSlug}', '{Path}')", orgSlug, dsSlug, path);
return(ExceptionResult(ex));
}
}
public async Task <CleanupBatchesResultDto> CleanupBatches()
{
if (!await _authManager.IsUserAdmin())
{
throw new UnauthorizedException("Only admins can perform system related tasks");
}
var expiration = DateTime.Now - _settings.UploadBatchTimeout;
// I'm scared
var toRemove = (from batch in _context.Batches
.Include(b => b.Dataset.Organization)
.Include(b => b.Entries)
where batch.Status == BatchStatus.Committed ||
((batch.Status == BatchStatus.Rolledback || batch.Status == BatchStatus.Running) &&
batch.Entries.Max(entry => entry.AddedOn) < expiration)
select batch).ToArray();
var removed = new List <RemovedBatchDto>();
var errors = new List <RemoveBatchErrorDto>();
foreach (var batch in toRemove)
{
var ds = batch.Dataset;
var org = ds.Organization;
try
{
// Remove intermediate files
if (batch.Status is BatchStatus.Rolledback or BatchStatus.Running)
{
var entries = batch.Entries.ToArray();
foreach (var entry in entries)
{
_logger.LogInformation("Deleting '{EntryPath}' of '{OrgSlug}/{DsSlug}'", entry.Path, org.Slug, ds.Slug);
await _objectManager.Delete(org.Slug, ds.Slug, entry.Path);
}
var ddb = _ddbManager.Get(org.Slug, ds.InternalRef);
// Remove empty ddb
if (!(await ddb.SearchAsync("*", true)).Any())
{
_ddbManager.Delete(org.Slug, ds.InternalRef);
}
}
_context.Batches.Remove(batch);
await _context.SaveChangesAsync();
removed.Add(new RemovedBatchDto
{
Status = batch.Status,
Start = batch.Start,
End = batch.End,
Token = batch.Token,
UserName = batch.UserName,
Dataset = ds.Slug,
Organization = org.Slug
});
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot remove batch '{BatchToken}'", batch.Token);
errors.Add(new RemoveBatchErrorDto
{
Message = ex.Message,
Token = batch.Token,
Dataset = ds.Slug,
Organization = org.Slug
});
}
}
return(new CleanupBatchesResultDto
{
RemovedBatches = removed.ToArray(),
RemoveBatchErrors = errors.ToArray()
});
}