public async Task <PushInitResultDto> Init(string orgSlug, string dsSlug, string checksum, StampDto stamp)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug, true);
var validateChecksum = false;
if (ds is null)
{
_logger.LogInformation("Dataset does not exist, creating it");
await _datasetsManager.AddNew(orgSlug, new DatasetNewDto
{
Name = dsSlug,
Slug = dsSlug
});
_logger.LogInformation("New dataset {OrgSlug}/{DsSlug} created", orgSlug, dsSlug);
ds = await _utils.GetDataset(orgSlug, dsSlug);
}
else
{
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to init push");
}
validateChecksum = true;
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
Stamp ourStamp = null;
if (validateChecksum)
{
if (string.IsNullOrEmpty(checksum))
{
throw new InvalidOperationException("Checksum parameter missing (dataset exists)");
}
// Is a pull required? The checksum passed by client is the checksum of the stamp
// of the last sync. If it's different, the client should pull first.
ourStamp = DDBWrapper.GetStamp(ddb.DatasetFolderPath);
if (ourStamp.Checksum != checksum)
{
return(new PushInitResultDto
{
PullRequired = true
});
}
}
ourStamp ??= DDBWrapper.GetStamp(ddb.DatasetFolderPath);
// Perform delta with our ddb
var delta = DDBWrapper.Delta(new Stamp
{
Checksum = stamp.Checksum,
Entries = stamp.Entries,
Meta = stamp.Meta
}, ourStamp);
// Compute locals
var locals = DDBWrapper.ComputeDeltaLocals(delta, ddb.DatasetFolderPath);
// Generate UUID
var uuid = Guid.NewGuid().ToString();
// Create tmp folder
var baseTempFolder = ddb.GetTmpFolder("push-" + uuid);
// Save incoming stamp as well as our stamp in temp folder
await File.WriteAllTextAsync(Path.Combine(baseTempFolder, StampFileName),
JsonConvert.SerializeObject(stamp));
await File.WriteAllTextAsync(Path.Combine(baseTempFolder, OurStampFileName),
JsonConvert.SerializeObject(ourStamp));
// Return missing files list (excluding folders)
return(new PushInitResultDto
{
Token = uuid,
NeededFiles = delta.Adds
.Where(item => item.Hash.Length > 0 && !locals.ContainsKey(item.Hash))
.Select(item => item.Path)
.ToArray(),
NeededMeta = delta.MetaAdds.ToArray(),
PullRequired = false
});
}
public async Task Commit(string orgSlug, string dsSlug, string token)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug);
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to commit to this dataset");
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
var baseTempFolder = ddb.GetTmpFolder("push-" + token);
var stampFilePath = Path.Combine(baseTempFolder, StampFileName);
var ourStampFilePath = Path.Combine(baseTempFolder, OurStampFileName);
var addTempFolder = Path.Combine(baseTempFolder, AddsTempFolder);
var metaFile = Path.Combine(baseTempFolder, MetaFile);
// Check push folder integrity
if (!File.Exists(stampFilePath))
{
throw new InvalidOperationException("Stamp not found");
}
if (!File.Exists(ourStampFilePath))
{
throw new InvalidOperationException("Our stamp not found");
}
var stamp = JsonConvert.DeserializeObject <Stamp>(await File.ReadAllTextAsync(stampFilePath));
var ourStamp = JsonConvert.DeserializeObject <Stamp>(await File.ReadAllTextAsync(ourStampFilePath));
if (ourStamp == null)
{
throw new InvalidOperationException("Our stamp is invalid (cannot deserialize)");
}
// Check that our stamp has not changed! If it has, another client
// might have performed changes that could conflict with our operation
// TODO: we could check for conflicts rather than failing and continue
// the operation if no conflicts are detected.
var currentStamp = DDBWrapper.GetStamp(ddb.DatasetFolderPath);
if (currentStamp.Checksum != ourStamp.Checksum)
{
throw new InvalidOperationException("The dataset has been changed by another user while pushing. Please try again!");
}
// Recompute delta
var delta = DDBWrapper.Delta(stamp, currentStamp);
// Create hard links for local files
var _ = DDBWrapper.ComputeDeltaLocals(delta, ddb.DatasetFolderPath, addTempFolder);
foreach (var add in delta.Adds.Where(item => item.Hash.Length > 0))
{
if (!File.Exists(Path.Combine(addTempFolder, add.Path)))
{
throw new InvalidOperationException($"Cannot commit: missing '{add.Path}'");
}
}
// Read meta dump
string metaDump = null;
if (File.Exists(metaFile))
{
metaDump = File.ReadAllText(metaFile);
}
// Applies delta
var conflicts = DDBWrapper.ApplyDelta(delta, addTempFolder, ddb.DatasetFolderPath, MergeStrategy.KeepTheirs, metaDump);
if (conflicts.Count > 0)
{
// This should never happen, since we merge conflicts using keep theirs
throw new InvalidOperationException("Merge conflicts detected, try pulling first.");
}
// Delete temp folder
Directory.Delete(baseTempFolder, true);
// Build items
foreach (var item in delta.Adds)
{
if (await ddb.IsBuildableAsync(item.Path))
{
_backgroundJob.Enqueue(() =>
HangfireUtils.BuildWrapper(ddb, item.Path, false, null));
}
}
if (await ddb.IsBuildPendingAsync())
{
_logger.LogInformation("Items are pending build, retriggering build");
var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildPendingWrapper(ddb, null));
_logger.LogInformation("Background job id is {JobId}", jobId);
}
}
