public async Task <IEnumerable <BatchDto> > ListBatches(string orgSlug, string dsSlug) { await _utils.GetOrganization(orgSlug); var dataset = await _utils.GetDataset(orgSlug, dsSlug); _logger.LogInformation("Listing batches of '{OrgSlug}/{DsSlug}'", orgSlug, dsSlug); var batches = from batch in _context.Batches .Include(x => x.Entries) .Include(x => x.Dataset) where batch.Dataset.Id == dataset.Id select new BatchDto { End = batch.End, Start = batch.Start, Token = batch.Token, UserName = batch.UserName, Status = batch.Status, Entries = from entry in batch.Entries select new BatchEntryDto { Hash = entry.Hash, Type = entry.Type, Size = entry.Size, AddedOn = entry.AddedOn, Path = entry.Path } }; return(batches); }
public async Task <DatasetDto> Get(string orgSlug, string dsSlug) { var dataset = await _utils.GetDataset(orgSlug, dsSlug); var ddb = _ddbManager.Get(orgSlug, dataset.InternalRef); return(dataset.ToDto(await ddb.GetInfoAsync())); }
public async Task <MetaDto> Add(string orgSlug, string dsSlug, string key, string data, string path = null) { var ds = await _utils.GetDataset(orgSlug, dsSlug); if (!await _authManager.IsOwnerOrAdmin(ds)) { throw new UnauthorizedException("The current user is not allowed to add meta"); } _logger.LogInformation("In Add('{OrgSlug}/{DsSlug}', {Key}, {Path})", orgSlug, dsSlug, key, path); if (string.IsNullOrWhiteSpace(key)) { throw new ArgumentException("Key should not be null or empty"); } if (data == null) { throw new ArgumentException("Data should not be null"); } var ddb = _ddbManager.Get(orgSlug, ds.InternalRef); if (path != null && !await ddb.EntryExistsAsync(path)) { throw new ArgumentException($"Path '{path}' does not exist"); } return(ddb.Meta.Add(key, data, path).ToDto()); }
public async Task <IEnumerable <EntryDto> > List(string orgSlug, string dsSlug, string path = null, bool recursive = false, EntryType?type = null) { var ds = await _utils.GetDataset(orgSlug, dsSlug); _logger.LogInformation("In List('{OrgSlug}/{DsSlug}')", orgSlug, dsSlug); var ddb = _ddbManager.Get(orgSlug, ds.InternalRef); _logger.LogInformation("Searching in '{Path}'", path); var entities = await ddb.SearchAsync(path, recursive); if (type != null) { entities = entities.Where(item => item.Type == type); } var files = entities.Select(item => item.ToDto()).ToArray(); _logger.LogInformation("Found {FilesCount} objects", files.Length); return(files); }
public async Task <PushInitResultDto> Init(string orgSlug, string dsSlug, string checksum, StampDto stamp) { var ds = await _utils.GetDataset(orgSlug, dsSlug, true); var validateChecksum = false; if (ds is null) { _logger.LogInformation("Dataset does not exist, creating it"); await _datasetsManager.AddNew(orgSlug, new DatasetNewDto { Name = dsSlug, Slug = dsSlug }); _logger.LogInformation("New dataset {OrgSlug}/{DsSlug} created", orgSlug, dsSlug); ds = await _utils.GetDataset(orgSlug, dsSlug); } else { if (!await _authManager.IsOwnerOrAdmin(ds)) { throw new UnauthorizedException("The current user is not allowed to init push"); } validateChecksum = true; } var ddb = _ddbManager.Get(orgSlug, ds.InternalRef); Stamp ourStamp = null; if (validateChecksum) { if (string.IsNullOrEmpty(checksum)) { throw new InvalidOperationException("Checksum parameter missing (dataset exists)"); } // Is a pull required? The checksum passed by client is the checksum of the stamp // of the last sync. If it's different, the client should pull first. ourStamp = DDBWrapper.GetStamp(ddb.DatasetFolderPath); if (ourStamp.Checksum != checksum) { return(new PushInitResultDto { PullRequired = true }); } } ourStamp ??= DDBWrapper.GetStamp(ddb.DatasetFolderPath); // Perform delta with our ddb var delta = DDBWrapper.Delta(new Stamp { Checksum = stamp.Checksum, Entries = stamp.Entries, Meta = stamp.Meta }, ourStamp); // Compute locals var locals = DDBWrapper.ComputeDeltaLocals(delta, ddb.DatasetFolderPath); // Generate UUID var uuid = Guid.NewGuid().ToString(); // Create tmp folder var baseTempFolder = ddb.GetTmpFolder("push-" + uuid); // Save incoming stamp as well as our stamp in temp folder await File.WriteAllTextAsync(Path.Combine(baseTempFolder, StampFileName), JsonConvert.SerializeObject(stamp)); await File.WriteAllTextAsync(Path.Combine(baseTempFolder, OurStampFileName), JsonConvert.SerializeObject(ourStamp)); // Return missing files list (excluding folders) return(new PushInitResultDto { Token = uuid, NeededFiles = delta.Adds .Where(item => item.Hash.Length > 0 && !locals.ContainsKey(item.Hash)) .Select(item => item.Path) .ToArray(), NeededMeta = delta.MetaAdds.ToArray(), PullRequired = false }); }