public async Task Commit(string orgSlug, string dsSlug, string token)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug);
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to commit to this dataset");
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
var baseTempFolder = ddb.GetTmpFolder("push-" + token);
var stampFilePath = Path.Combine(baseTempFolder, StampFileName);
var ourStampFilePath = Path.Combine(baseTempFolder, OurStampFileName);
var addTempFolder = Path.Combine(baseTempFolder, AddsTempFolder);
var metaFile = Path.Combine(baseTempFolder, MetaFile);
// Check push folder integrity
if (!File.Exists(stampFilePath))
{
throw new InvalidOperationException("Stamp not found");
}
if (!File.Exists(ourStampFilePath))
{
throw new InvalidOperationException("Our stamp not found");
}
var stamp = JsonConvert.DeserializeObject <Stamp>(await File.ReadAllTextAsync(stampFilePath));
var ourStamp = JsonConvert.DeserializeObject <Stamp>(await File.ReadAllTextAsync(ourStampFilePath));
if (ourStamp == null)
{
throw new InvalidOperationException("Our stamp is invalid (cannot deserialize)");
}
// Check that our stamp has not changed! If it has, another client
// might have performed changes that could conflict with our operation
// TODO: we could check for conflicts rather than failing and continue
// the operation if no conflicts are detected.
var currentStamp = DDBWrapper.GetStamp(ddb.DatasetFolderPath);
if (currentStamp.Checksum != ourStamp.Checksum)
{
throw new InvalidOperationException("The dataset has been changed by another user while pushing. Please try again!");
}
// Recompute delta
var delta = DDBWrapper.Delta(stamp, currentStamp);
// Create hard links for local files
var _ = DDBWrapper.ComputeDeltaLocals(delta, ddb.DatasetFolderPath, addTempFolder);
foreach (var add in delta.Adds.Where(item => item.Hash.Length > 0))
{
if (!File.Exists(Path.Combine(addTempFolder, add.Path)))
{
throw new InvalidOperationException($"Cannot commit: missing '{add.Path}'");
}
}
// Read meta dump
string metaDump = null;
if (File.Exists(metaFile))
{
metaDump = File.ReadAllText(metaFile);
}
// Applies delta
var conflicts = DDBWrapper.ApplyDelta(delta, addTempFolder, ddb.DatasetFolderPath, MergeStrategy.KeepTheirs, metaDump);
if (conflicts.Count > 0)
{
// This should never happen, since we merge conflicts using keep theirs
throw new InvalidOperationException("Merge conflicts detected, try pulling first.");
}
// Delete temp folder
Directory.Delete(baseTempFolder, true);
// Build items
foreach (var item in delta.Adds)
{
if (await ddb.IsBuildableAsync(item.Path))
{
_backgroundJob.Enqueue(() =>
HangfireUtils.BuildWrapper(ddb, item.Path, false, null));
}
}
if (await ddb.IsBuildPendingAsync())
{
_logger.LogInformation("Items are pending build, retriggering build");
var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildPendingWrapper(ddb, null));
_logger.LogInformation("Background job id is {JobId}", jobId);
}
}
public async Task <EntryDto> AddNew(string orgSlug, string dsSlug, string path, Stream stream = null)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug);
_logger.LogInformation("In AddNew('{OrgSlug}/{DsSlug}')", orgSlug, dsSlug);
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to edit dataset");
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
// If it's a folder
if (stream == null)
{
if (await ddb.EntryExistsAsync(path))
{
throw new InvalidOperationException("Cannot create a folder on another entry");
}
if (path == DDB.DatabaseFolderName)
{
throw new InvalidOperationException($"'{DDB.DatabaseFolderName}' is a reserved folder name");
}
_logger.LogInformation("Adding folder to DDB");
// Add to DDB
await ddb.AddAsync(path);
_logger.LogInformation("Added to DDB");
return(new EntryDto
{
Path = path,
Type = EntryType.Directory,
Size = 0
});
}
// Check user storage space
await _utils.CheckCurrentUserStorage(stream.Length);
var localFilePath = ddb.GetLocalPath(path);
CommonUtils.EnsureSafePath(localFilePath);
_logger.LogInformation("Local file path is '{LocalFilePath}'", localFilePath);
// Write down the file
await using (var localFileStream = File.OpenWrite(localFilePath))
await stream.CopyToAsync(localFileStream);
_logger.LogInformation("File saved, adding to DDB");
ddb.AddRaw(localFilePath);
_logger.LogInformation("Added to DDB, checking entry now...");
var entry = await ddb.GetEntryAsync(path);
if (entry == null)
{
throw new InvalidOperationException("Cannot find just added file!");
}
_logger.LogInformation("Entry OK");
if (await ddb.IsBuildableAsync(entry.Path))
{
_logger.LogInformation("This item is buildable, build it!");
var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildWrapper(ddb, path, false, null));
_logger.LogInformation("Background job id is {JobId}", jobId);
}
else if (await ddb.IsBuildPendingAsync())
{
_logger.LogInformation("Items are pending build, retriggering build");
var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildPendingWrapper(ddb, null));
_logger.LogInformation("Background job id is {JobId}", jobId);
}
return(entry.ToDto());
}