public async Task <IEnumerable <BatchDto> > ListBatches(string orgSlug, string dsSlug)
{
await _utils.GetOrganization(orgSlug);
var dataset = await _utils.GetDataset(orgSlug, dsSlug);
_logger.LogInformation("Listing batches of '{OrgSlug}/{DsSlug}'", orgSlug, dsSlug);
var batches = from batch in _context.Batches
.Include(x => x.Entries)
.Include(x => x.Dataset)
where batch.Dataset.Id == dataset.Id
select new BatchDto
{
End = batch.End,
Start = batch.Start,
Token = batch.Token,
UserName = batch.UserName,
Status = batch.Status,
Entries = from entry in batch.Entries
select new BatchEntryDto
{
Hash = entry.Hash,
Type = entry.Type,
Size = entry.Size,
AddedOn = entry.AddedOn,
Path = entry.Path
}
};
return(batches);
}
public async Task <DatasetDto> Get(string orgSlug, string dsSlug)
{
var dataset = await _utils.GetDataset(orgSlug, dsSlug);
var ddb = _ddbManager.Get(orgSlug, dataset.InternalRef);
return(dataset.ToDto(await ddb.GetInfoAsync()));
}
public async Task <MetaDto> Add(string orgSlug, string dsSlug, string key, string data, string path = null)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug);
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to add meta");
}
_logger.LogInformation("In Add('{OrgSlug}/{DsSlug}', {Key}, {Path})", orgSlug, dsSlug, key, path);
if (string.IsNullOrWhiteSpace(key))
{
throw new ArgumentException("Key should not be null or empty");
}
if (data == null)
{
throw new ArgumentException("Data should not be null");
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
if (path != null && !await ddb.EntryExistsAsync(path))
{
throw new ArgumentException($"Path '{path}' does not exist");
}
return(ddb.Meta.Add(key, data, path).ToDto());
}
public async Task <IEnumerable <EntryDto> > List(string orgSlug, string dsSlug, string path = null,
bool recursive = false, EntryType?type = null)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug);
_logger.LogInformation("In List('{OrgSlug}/{DsSlug}')", orgSlug, dsSlug);
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
_logger.LogInformation("Searching in '{Path}'", path);
var entities = await ddb.SearchAsync(path, recursive);
if (type != null)
{
entities = entities.Where(item => item.Type == type);
}
var files = entities.Select(item => item.ToDto()).ToArray();
_logger.LogInformation("Found {FilesCount} objects", files.Length);
return(files);
}
public async Task <PushInitResultDto> Init(string orgSlug, string dsSlug, string checksum, StampDto stamp)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug, true);
var validateChecksum = false;
if (ds is null)
{
_logger.LogInformation("Dataset does not exist, creating it");
await _datasetsManager.AddNew(orgSlug, new DatasetNewDto
{
Name = dsSlug,
Slug = dsSlug
});
_logger.LogInformation("New dataset {OrgSlug}/{DsSlug} created", orgSlug, dsSlug);
ds = await _utils.GetDataset(orgSlug, dsSlug);
}
else
{
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to init push");
}
validateChecksum = true;
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
Stamp ourStamp = null;
if (validateChecksum)
{
if (string.IsNullOrEmpty(checksum))
{
throw new InvalidOperationException("Checksum parameter missing (dataset exists)");
}
// Is a pull required? The checksum passed by client is the checksum of the stamp
// of the last sync. If it's different, the client should pull first.
ourStamp = DDBWrapper.GetStamp(ddb.DatasetFolderPath);
if (ourStamp.Checksum != checksum)
{
return(new PushInitResultDto
{
PullRequired = true
});
}
}
ourStamp ??= DDBWrapper.GetStamp(ddb.DatasetFolderPath);
// Perform delta with our ddb
var delta = DDBWrapper.Delta(new Stamp
{
Checksum = stamp.Checksum,
Entries = stamp.Entries,
Meta = stamp.Meta
}, ourStamp);
// Compute locals
var locals = DDBWrapper.ComputeDeltaLocals(delta, ddb.DatasetFolderPath);
// Generate UUID
var uuid = Guid.NewGuid().ToString();
// Create tmp folder
var baseTempFolder = ddb.GetTmpFolder("push-" + uuid);
// Save incoming stamp as well as our stamp in temp folder
await File.WriteAllTextAsync(Path.Combine(baseTempFolder, StampFileName),
JsonConvert.SerializeObject(stamp));
await File.WriteAllTextAsync(Path.Combine(baseTempFolder, OurStampFileName),
JsonConvert.SerializeObject(ourStamp));
// Return missing files list (excluding folders)
return(new PushInitResultDto
{
Token = uuid,
NeededFiles = delta.Adds
.Where(item => item.Hash.Length > 0 && !locals.ContainsKey(item.Hash))
.Select(item => item.Path)
.ToArray(),
NeededMeta = delta.MetaAdds.ToArray(),
PullRequired = false
});
}
