public async Task <DatasetDto> AddNew(string orgSlug, DatasetNewDto dataset)
{
var org = await _utils.GetOrganization(orgSlug);
if (!await _authManager.IsOwnerOrAdmin(org))
{
throw new UnauthorizedException("You are not authorized to add datasets to this organization");
}
if (dataset == null)
{
throw new BadRequestException("Dataset is null");
}
if (dataset.Slug == null)
{
throw new BadRequestException("Dataset slug is null");
}
if (!dataset.Slug.IsValidSlug())
{
throw new BadRequestException("Dataset slug is invalid");
}
if (_context.Datasets.Any(item => item.Slug == dataset.Slug && item.Organization.Slug == orgSlug))
{
throw new BadRequestException("Dataset with this slug already exists");
}
var ds = new Dataset
{
Slug = dataset.Slug,
Organization = org,
InternalRef = Guid.NewGuid(),
CreationDate = DateTime.Now
};
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
ddb.Meta.GetSafe().Name = dataset.Name ?? dataset.Slug;
var attributes = await ddb.GetAttributesAsync();
if (dataset.IsPublic.HasValue)
{
attributes.IsPublic = dataset.IsPublic.Value;
}
org.Datasets.Add(ds);
await _context.SaveChangesAsync();
return(ds.ToDto(await ddb.GetInfoAsync()));
}
public async Task <CleanupDatasetResultDto> CleanupEmptyDatasets()
{
if (!await _authManager.IsUserAdmin())
{
throw new UnauthorizedException("Only admins can perform system related tasks");
}
var datasets = _context.Datasets.Include(ds => ds.Organization).ToArray();
_logger.LogInformation("Found {DatasetsCount} with objects count zero", datasets.Length);
var deleted = new List <string>();
var notDeleted = new List <CleanupDatasetErrorDto>();
foreach (var ds in datasets)
{
_logger.LogInformation("Analyzing dataset {OrgSlug}/{DsSlug}", ds.Organization.Slug, ds.Slug);
try
{
// Check if objects count is ok
var ddb = _ddbManager.Get(ds.Organization.Slug, ds.InternalRef);
var entries = (await ddb.SearchAsync("*", true))?.ToArray();
if (entries == null || !entries.Any())
{
_context.Remove(ds);
await _context.SaveChangesAsync();
deleted.Add(ds.Slug);
_logger.LogInformation("Deleted");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot remove dataset '{DsSlug}'", ds.Slug);
notDeleted.Add(new CleanupDatasetErrorDto
{
Dataset = ds.Slug,
Organization = ds.Organization.Slug,
Message = ex.Message
});
}
}
return(new CleanupDatasetResultDto
{
RemoveDatasetErrors = notDeleted.ToArray(),
RemovedDatasets = deleted.ToArray()
});
}
public async Task <ShareInitResultDto> Initialize(ShareInitDto parameters)
{
if (parameters == null)
{
throw new BadRequestException("Invalid parameters");
}
var currentUser = await _authManager.GetCurrentUser();
if (currentUser == null)
{
throw new UnauthorizedException("Invalid user");
}
// Check if user has enough space to upload any file
await _utils.CheckCurrentUserStorage();
Dataset dataset;
TagDto tag = parameters.Tag.ToTag();
// No organization requested
if (tag.OrganizationSlug == null)
{
if (tag.DatasetSlug != null)
{
throw new BadRequestException("Cannot specify a dataset without an organization");
}
string orgSlug;
_logger.LogInformation("No organization and dataset specified");
var nameSlug = currentUser.UserName.ToSlug();
// We start from the principle that a default user organization begins with the username in slug form
// If we notice that this strategy is weak we have to add a new field to the db entry
var userOrganization = _context.Organizations.FirstOrDefault(item =>
item.OwnerId == currentUser.Id && item.Name.StartsWith(nameSlug));
// Some idiot removed its own organization, maybe we should prevent this, btw nevermind: let's take care of it
if (userOrganization == null)
{
// This section of code can be extracted and put in a separated utils method
orgSlug = _utils.GetFreeOrganizationSlug(currentUser.UserName);
_logger.LogInformation("No default user organization found, adding a new one: '{OrgSlug}'",
orgSlug);
var org = await _organizationsManager.AddNew(new OrganizationDto
{
Name = currentUser.UserName,
IsPublic = false,
CreationDate = DateTime.Now,
Owner = currentUser.Id,
Slug = orgSlug
});
userOrganization = _context.Organizations.First(item => item.Slug == orgSlug);
}
orgSlug = userOrganization.Slug;
_logger.LogInformation("Using default user organization '{OrgSlug}'", orgSlug);
var dsSlug = GetUniqueDatasetSlug();
_logger.LogInformation("Generated unique dataset slug '{DsSlug}'", dsSlug);
await _datasetsManager.AddNew(orgSlug, new DatasetNewDto
{
Slug = dsSlug,
Name = parameters.DatasetName,
IsPublic = true
});
dataset = await _utils.GetDataset(orgSlug, dsSlug);
_logger.LogInformation("Created new dataset '{DsSlug}', creating batch", dsSlug);
}
else
{
// Check if the requested organization exists
var organization = await _utils.GetOrganization(tag.OrganizationSlug, true);
if (organization == null)
{
throw new BadRequestException($"Cannot find organization '{tag.OrganizationSlug}'");
}
_logger.LogInformation("Organization found");
// If no dataset is specified, we create a new one with a random slug
if (tag.DatasetSlug == null)
{
var dsSlug = GetUniqueDatasetSlug();
_logger.LogInformation("Generated unique dataset slug '{DsSlug}'", dsSlug);
await _datasetsManager.AddNew(tag.OrganizationSlug, new DatasetNewDto
{
Slug = dsSlug,
Name = parameters.DatasetName,
IsPublic = true
});
dataset = _context.Datasets.First(ds => ds.Slug == dsSlug);
_logger.LogInformation("Dataset created");
}
else
{
dataset = await _utils.GetDataset(tag.OrganizationSlug, tag.DatasetSlug, true);
// Create dataset if not exists
if (dataset == null)
{
_logger.LogInformation("Dataset '{DatasetSlug}' not found, creating it", tag.DatasetSlug);
await _datasetsManager.AddNew(tag.OrganizationSlug, new DatasetNewDto
{
Slug = tag.DatasetSlug,
Name = parameters.DatasetName,
IsPublic = true
});
_logger.LogInformation("Dataset created");
dataset = _context.Datasets.First(ds => ds.Slug == tag.DatasetSlug);
}
else
{
_logger.LogInformation("Checking for running batches");
await _context.Entry(dataset).Collection(item => item.Batches).LoadAsync();
var runningBatches = dataset.Batches.Where(item => item.End == null).ToArray();
if (runningBatches.Any())
{
_logger.LogInformation(
"Found '{RunningBatchesCount}' running batch(es), stopping and rolling back before starting a new one",
runningBatches.Length);
foreach (var b in runningBatches)
{
await RollbackBatch(b);
}
}
}
}
}
var batch = new Batch
{
Dataset = dataset,
Start = DateTime.Now,
Token = _batchTokenGenerator.GenerateToken(),
UserName = await _authManager.SafeGetCurrentUserName(),
Status = BatchStatus.Running
};
_logger.LogInformation("Adding new batch for user '{BatchUserName}' with token '{BatchToken}'",
batch.UserName, batch.Token);
await _context.Batches.AddAsync(batch);
await _context.SaveChangesAsync();
_logger.LogInformation("Batch created, it is now possible to upload files");
return(new ShareInitResultDto
{
Token = batch.Token
// NOTE: Maybe useful in the future
// Tag = tag
});
}
public async Task <OrganizationDto> AddNew(OrganizationDto organization, bool skipAuthCheck = false)
{
// TODO: To change when implementing anonymous users
if (!skipAuthCheck)
{
var currentUser = await _authManager.GetCurrentUser();
if (currentUser == null)
{
throw new UnauthorizedException("Invalid user");
}
if (!organization.Slug.IsValidSlug())
{
throw new BadRequestException("Invalid organization orgSlug");
}
var existingOrg = _context.Organizations.FirstOrDefault(item => item.Slug == organization.Slug);
if (existingOrg != null)
{
throw new ConflictException("The organization already exists");
}
if (!await _authManager.IsUserAdmin())
{
// If the owner is specified it should be the current user
if (organization.Owner != null && organization.Owner != currentUser.Id)
{
throw new UnauthorizedException(
"Cannot create a new organization that belongs to a different user");
}
// The current user is the owner
organization.Owner = currentUser.Id;
}
else
{
// If no owner specified, the owner is the current user
if (organization.Owner == null)
{
organization.Owner = currentUser.Id;
}
else
{
// Otherwise check if user exists
if (!await _authManager.UserExists(organization.Owner))
{
throw new BadRequestException($"Cannot find user with name '{organization.Owner}'");
}
}
}
}
else
{
if (!organization.Slug.IsValidSlug())
{
throw new BadRequestException("Invalid organization orgSlug");
}
}
var org = organization.ToEntity();
org.CreationDate = DateTime.Now;
await _context.Organizations.AddAsync(org);
await _context.SaveChangesAsync();
return(org.ToDto());
}