private async Task RollbackBatch(Batch batch, bool deleteDataset = false) { _logger.LogInformation("Rolling back batch '{BatchToken}'", batch.Token); // I know we will have concurrency problems here because we could be in the middle of the rollback when another client calls for another one // To mitigate this issue we are going to commit the status change of the batch as soon as possible batch.Status = BatchStatus.Rolledback; batch.End = DateTime.Now; await _context.SaveChangesAsync(); await _context.Entry(batch).Collection(item => item.Entries).LoadAsync(); await _context.Entry(batch).Reference(item => item.Dataset).LoadAsync(); var ds = batch.Dataset; await _context.Entry(ds).Reference(item => item.Organization).LoadAsync(); var org = ds.Organization; if (deleteDataset) { await _datasetsManager.Delete(org.Slug, ds.Slug); } else { foreach (var entry in batch.Entries) { await _objectsManager.Delete(org.Slug, ds.Slug, entry.Path); } } }
public async Task <IActionResult> Delete([FromRoute] string orgSlug, [FromRoute] string dsSlug, [FromForm] string path) { try { _logger.LogDebug("Objects controller Delete('{OrgSlug}', '{DsSlug}', '{Path}')", orgSlug, dsSlug, path); await _objectsManager.Delete(orgSlug, dsSlug, path); return(NoContent()); } catch (Exception ex) { _logger.LogError(ex, "Exception in Objects controller Delete('{OrgSlug}', '{DsSlug}', '{Path}')", orgSlug, dsSlug, path); return(ExceptionResult(ex)); } }
public async Task <CleanupBatchesResultDto> CleanupBatches() { if (!await _authManager.IsUserAdmin()) { throw new UnauthorizedException("Only admins can perform system related tasks"); } var expiration = DateTime.Now - _settings.UploadBatchTimeout; // I'm scared var toRemove = (from batch in _context.Batches .Include(b => b.Dataset.Organization) .Include(b => b.Entries) where batch.Status == BatchStatus.Committed || ((batch.Status == BatchStatus.Rolledback || batch.Status == BatchStatus.Running) && batch.Entries.Max(entry => entry.AddedOn) < expiration) select batch).ToArray(); var removed = new List <RemovedBatchDto>(); var errors = new List <RemoveBatchErrorDto>(); foreach (var batch in toRemove) { var ds = batch.Dataset; var org = ds.Organization; try { // Remove intermediate files if (batch.Status is BatchStatus.Rolledback or BatchStatus.Running) { var entries = batch.Entries.ToArray(); foreach (var entry in entries) { _logger.LogInformation("Deleting '{EntryPath}' of '{OrgSlug}/{DsSlug}'", entry.Path, org.Slug, ds.Slug); await _objectManager.Delete(org.Slug, ds.Slug, entry.Path); } var ddb = _ddbManager.Get(org.Slug, ds.InternalRef); // Remove empty ddb if (!(await ddb.SearchAsync("*", true)).Any()) { _ddbManager.Delete(org.Slug, ds.InternalRef); } } _context.Batches.Remove(batch); await _context.SaveChangesAsync(); removed.Add(new RemovedBatchDto { Status = batch.Status, Start = batch.Start, End = batch.End, Token = batch.Token, UserName = batch.UserName, Dataset = ds.Slug, Organization = org.Slug }); } catch (Exception ex) { _logger.LogError(ex, "Cannot remove batch '{BatchToken}'", batch.Token); errors.Add(new RemoveBatchErrorDto { Message = ex.Message, Token = batch.Token, Dataset = ds.Slug, Organization = org.Slug }); } } return(new CleanupBatchesResultDto { RemovedBatches = removed.ToArray(), RemoveBatchErrors = errors.ToArray() }); }