public async Task Upload(string orgSlug, string dsSlug, string path, string token, Stream stream)
{
if (string.IsNullOrWhiteSpace(path))
{
throw new ArgumentException("Path cannot be empty");
}
if (!stream.CanRead)
{
throw new ArgumentException("Stream is null or is not readable");
}
var ds = await _utils.GetDataset(orgSlug, dsSlug);
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to upload to this dataset");
}
// Check if user has enough space to upload this
await _utils.CheckCurrentUserStorage(stream.Length);
if (path.Contains(".."))
{
throw new InvalidOperationException("Path cannot contain dot notation");
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
var baseTempFolder = ddb.GetTmpFolder("push-" + token);
if (!Directory.Exists(baseTempFolder))
{
throw new InvalidOperationException("Cannot upload file before initializing push");
}
var addTempFolder = Path.Combine(baseTempFolder, AddsTempFolder);
Directory.CreateDirectory(addTempFolder);
// Calculate new file path
var filePath = Path.Combine(addTempFolder, path);
// Ensure subfolder exists
var parentPath = Path.GetDirectoryName(filePath);
if (parentPath != null)
{
Directory.CreateDirectory(parentPath);
}
// Save file in temp folder
await using var file = File.OpenWrite(filePath);
await stream.CopyToAsync(file);
}
public async Task <ShareInitResultDto> Initialize(ShareInitDto parameters)
{
if (parameters == null)
{
throw new BadRequestException("Invalid parameters");
}
var currentUser = await _authManager.GetCurrentUser();
if (currentUser == null)
{
throw new UnauthorizedException("Invalid user");
}
// Check if user has enough space to upload any file
await _utils.CheckCurrentUserStorage();
Dataset dataset;
TagDto tag = parameters.Tag.ToTag();
// No organization requested
if (tag.OrganizationSlug == null)
{
if (tag.DatasetSlug != null)
{
throw new BadRequestException("Cannot specify a dataset without an organization");
}
string orgSlug;
_logger.LogInformation("No organization and dataset specified");
var nameSlug = currentUser.UserName.ToSlug();
// We start from the principle that a default user organization begins with the username in slug form
// If we notice that this strategy is weak we have to add a new field to the db entry
var userOrganization = _context.Organizations.FirstOrDefault(item =>
item.OwnerId == currentUser.Id && item.Name.StartsWith(nameSlug));
// Some idiot removed its own organization, maybe we should prevent this, btw nevermind: let's take care of it
if (userOrganization == null)
{
// This section of code can be extracted and put in a separated utils method
orgSlug = _utils.GetFreeOrganizationSlug(currentUser.UserName);
_logger.LogInformation("No default user organization found, adding a new one: '{OrgSlug}'",
orgSlug);
var org = await _organizationsManager.AddNew(new OrganizationDto
{
Name = currentUser.UserName,
IsPublic = false,
CreationDate = DateTime.Now,
Owner = currentUser.Id,
Slug = orgSlug
});
userOrganization = _context.Organizations.First(item => item.Slug == orgSlug);
}
orgSlug = userOrganization.Slug;
_logger.LogInformation("Using default user organization '{OrgSlug}'", orgSlug);
var dsSlug = GetUniqueDatasetSlug();
_logger.LogInformation("Generated unique dataset slug '{DsSlug}'", dsSlug);
await _datasetsManager.AddNew(orgSlug, new DatasetNewDto
{
Slug = dsSlug,
Name = parameters.DatasetName,
IsPublic = true
});
dataset = await _utils.GetDataset(orgSlug, dsSlug);
_logger.LogInformation("Created new dataset '{DsSlug}', creating batch", dsSlug);
}
else
{
// Check if the requested organization exists
var organization = await _utils.GetOrganization(tag.OrganizationSlug, true);
if (organization == null)
{
throw new BadRequestException($"Cannot find organization '{tag.OrganizationSlug}'");
}
_logger.LogInformation("Organization found");
// If no dataset is specified, we create a new one with a random slug
if (tag.DatasetSlug == null)
{
var dsSlug = GetUniqueDatasetSlug();
_logger.LogInformation("Generated unique dataset slug '{DsSlug}'", dsSlug);
await _datasetsManager.AddNew(tag.OrganizationSlug, new DatasetNewDto
{
Slug = dsSlug,
Name = parameters.DatasetName,
IsPublic = true
});
dataset = _context.Datasets.First(ds => ds.Slug == dsSlug);
_logger.LogInformation("Dataset created");
}
else
{
dataset = await _utils.GetDataset(tag.OrganizationSlug, tag.DatasetSlug, true);
// Create dataset if not exists
if (dataset == null)
{
_logger.LogInformation("Dataset '{DatasetSlug}' not found, creating it", tag.DatasetSlug);
await _datasetsManager.AddNew(tag.OrganizationSlug, new DatasetNewDto
{
Slug = tag.DatasetSlug,
Name = parameters.DatasetName,
IsPublic = true
});
_logger.LogInformation("Dataset created");
dataset = _context.Datasets.First(ds => ds.Slug == tag.DatasetSlug);
}
else
{
_logger.LogInformation("Checking for running batches");
await _context.Entry(dataset).Collection(item => item.Batches).LoadAsync();
var runningBatches = dataset.Batches.Where(item => item.End == null).ToArray();
if (runningBatches.Any())
{
_logger.LogInformation(
"Found '{RunningBatchesCount}' running batch(es), stopping and rolling back before starting a new one",
runningBatches.Length);
foreach (var b in runningBatches)
{
await RollbackBatch(b);
}
}
}
}
}
var batch = new Batch
{
Dataset = dataset,
Start = DateTime.Now,
Token = _batchTokenGenerator.GenerateToken(),
UserName = await _authManager.SafeGetCurrentUserName(),
Status = BatchStatus.Running
};
_logger.LogInformation("Adding new batch for user '{BatchUserName}' with token '{BatchToken}'",
batch.UserName, batch.Token);
await _context.Batches.AddAsync(batch);
await _context.SaveChangesAsync();
_logger.LogInformation("Batch created, it is now possible to upload files");
return(new ShareInitResultDto
{
Token = batch.Token
// NOTE: Maybe useful in the future
// Tag = tag
});
}
public async Task <EntryDto> AddNew(string orgSlug, string dsSlug, string path, Stream stream = null)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug);
_logger.LogInformation("In AddNew('{OrgSlug}/{DsSlug}')", orgSlug, dsSlug);
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to edit dataset");
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
// If it's a folder
if (stream == null)
{
if (await ddb.EntryExistsAsync(path))
{
throw new InvalidOperationException("Cannot create a folder on another entry");
}
if (path == DDB.DatabaseFolderName)
{
throw new InvalidOperationException($"'{DDB.DatabaseFolderName}' is a reserved folder name");
}
_logger.LogInformation("Adding folder to DDB");
// Add to DDB
await ddb.AddAsync(path);
_logger.LogInformation("Added to DDB");
return(new EntryDto
{
Path = path,
Type = EntryType.Directory,
Size = 0
});
}
// Check user storage space
await _utils.CheckCurrentUserStorage(stream.Length);
var localFilePath = ddb.GetLocalPath(path);
CommonUtils.EnsureSafePath(localFilePath);
_logger.LogInformation("Local file path is '{LocalFilePath}'", localFilePath);
// Write down the file
await using (var localFileStream = File.OpenWrite(localFilePath))
await stream.CopyToAsync(localFileStream);
_logger.LogInformation("File saved, adding to DDB");
ddb.AddRaw(localFilePath);
_logger.LogInformation("Added to DDB, checking entry now...");
var entry = await ddb.GetEntryAsync(path);
if (entry == null)
{
throw new InvalidOperationException("Cannot find just added file!");
}
_logger.LogInformation("Entry OK");
if (await ddb.IsBuildableAsync(entry.Path))
{
_logger.LogInformation("This item is buildable, build it!");
var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildWrapper(ddb, path, false, null));
_logger.LogInformation("Background job id is {JobId}", jobId);
}
else if (await ddb.IsBuildPendingAsync())
{
_logger.LogInformation("Items are pending build, retriggering build");
var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildPendingWrapper(ddb, null));
_logger.LogInformation("Background job id is {JobId}", jobId);
}
return(entry.ToDto());
}