public async Task <CleanupDatasetResultDto> CleanupEmptyDatasets()
{
if (!await _authManager.IsUserAdmin())
{
throw new UnauthorizedException("Only admins can perform system related tasks");
}
var datasets = _context.Datasets.Include(ds => ds.Organization).ToArray();
_logger.LogInformation("Found {DatasetsCount} with objects count zero", datasets.Length);
var deleted = new List <string>();
var notDeleted = new List <CleanupDatasetErrorDto>();
foreach (var ds in datasets)
{
_logger.LogInformation("Analyzing dataset {OrgSlug}/{DsSlug}", ds.Organization.Slug, ds.Slug);
try
{
// Check if objects count is ok
var ddb = _ddbManager.Get(ds.Organization.Slug, ds.InternalRef);
var entries = (await ddb.SearchAsync("*", true))?.ToArray();
if (entries == null || !entries.Any())
{
_context.Remove(ds);
await _context.SaveChangesAsync();
deleted.Add(ds.Slug);
_logger.LogInformation("Deleted");
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot remove dataset '{DsSlug}'", ds.Slug);
notDeleted.Add(new CleanupDatasetErrorDto
{
Dataset = ds.Slug,
Organization = ds.Organization.Slug,
Message = ex.Message
});
}
}
return(new CleanupDatasetResultDto
{
RemoveDatasetErrors = notDeleted.ToArray(),
RemovedDatasets = deleted.ToArray()
});
}
public async Task <MetaDto> Add(string orgSlug, string dsSlug, string key, string data, string path = null)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug);
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to add meta");
}
_logger.LogInformation("In Add('{OrgSlug}/{DsSlug}', {Key}, {Path})", orgSlug, dsSlug, key, path);
if (string.IsNullOrWhiteSpace(key))
{
throw new ArgumentException("Key should not be null or empty");
}
if (data == null)
{
throw new ArgumentException("Data should not be null");
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
if (path != null && !await ddb.EntryExistsAsync(path))
{
throw new ArgumentException($"Path '{path}' does not exist");
}
return(ddb.Meta.Add(key, data, path).ToDto());
}
public async Task CopyToAsync(Stream stream)
{
// If there is just one file we return it
if (_descriptorType == FileDescriptorType.Single)
{
var filePath = _paths.First();
_logger.LogInformation("Only one path found: '{FilePath}'", filePath);
var localPath = _ddb.GetLocalPath(filePath);
await using var fileStream = new FileStream(localPath, FileMode.Open, FileAccess.Read);
await fileStream.CopyToAsync(stream);
}
// Otherwise we zip everything together and return the package
else
{
using var archive = new ZipArchive(stream, ZipArchiveMode.Create, true);
foreach (var path in _paths)
{
_logger.LogInformation("Zipping: '{Path}'", path);
var entry = archive.CreateEntry(path, CommonUtils.GetCompressionLevel(path));
await using var entryStream = entry.Open();
var localPath = _ddb.GetLocalPath(path);
await using var fileStream = new FileStream(localPath, FileMode.Open, FileAccess.Read);
await fileStream.CopyToAsync(entryStream);
}
// We treat folders separately because if they are empty they would not be included in the archive
if (_folders != null)
{
foreach (var folder in _folders)
{
archive.CreateEntry(folder + "/");
}
}
// Include ddb folder
if (_descriptorType == FileDescriptorType.Dataset)
{
var ddb = _ddbManager.Get(_orgSlug, _internalRef);
archive.CreateEntryFromAny(Path.Combine(ddb.DatasetFolderPath, DDB.DatabaseFolderName), string.Empty, new[] { ddb.BuildFolderPath });
}
}
}
public FileStreamDescriptor(string name, string contentType, string orgSlug, Guid internalRef, string[] paths, string[] folders,
FileDescriptorType descriptorType, ILogger <ObjectsManager> logger, IDdbManager ddbManager)
{
_orgSlug = orgSlug;
_internalRef = internalRef;
_paths = paths;
_folders = folders;
_descriptorType = descriptorType;
_logger = logger;
_ddbManager = ddbManager;
Name = name;
ContentType = contentType;
_ddb = ddbManager.Get(orgSlug, internalRef);
}
public async Task <HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
var tempOrg = "test-" + Guid.NewGuid();
var tempDs = Guid.NewGuid();
var data = new Dictionary <string, object>
{
{ "TempOrg", tempOrg },
{ "TempDs", tempDs.ToString() },
{ "Provider", _ddbManager.GetType().FullName }
};
var ddb = _ddbManager.Get(tempOrg, tempDs);
try
{
var version = ddb.Version;
if (string.IsNullOrWhiteSpace(version))
{
return(HealthCheckResult.Unhealthy("Cannot get ddb version", null, data));
}
data.Add("DdbVersion", version);
var entries = await ddb.SearchAsync(null, true, cancellationToken);
if (entries == null || entries.Any())
{
return(HealthCheckResult.Unhealthy("Something wrong with ddb behaviour", null, data));
}
return(HealthCheckResult.Healthy("Ddb is working properly", data));
}
catch (Exception ex)
{
return(HealthCheckResult.Unhealthy("Exception while testing ddb: " + ex.Message, ex, data));
}
finally
{
_ddbManager.Delete(tempOrg, tempDs);
}
}
public async Task <IEnumerable <DatasetDto> > List(string orgSlug)
{
var org = await _utils.GetOrganization(orgSlug);
var res = new List <DatasetDto>();
foreach (var ds in org.Datasets.ToArray())
{
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
var info = await ddb.GetInfoAsync();
var attributes = new EntryProperties(info.Properties);
res.Add(new DatasetDto
{
Slug = ds.Slug,
CreationDate = ds.CreationDate,
Properties = attributes.Properties,
Size = info.Size
});
}
return(res.ToArray());
}
public async Task <IEnumerable <EntryDto> > List(string orgSlug, string dsSlug, string path = null,
bool recursive = false, EntryType?type = null)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug);
_logger.LogInformation("In List('{OrgSlug}/{DsSlug}')", orgSlug, dsSlug);
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
_logger.LogInformation("Searching in '{Path}'", path);
var entities = await ddb.SearchAsync(path, recursive);
if (type != null)
{
entities = entities.Where(item => item.Type == type);
}
var files = entities.Select(item => item.ToDto()).ToArray();
_logger.LogInformation("Found {FilesCount} objects", files.Length);
return(files);
}
public async Task <PushInitResultDto> Init(string orgSlug, string dsSlug, string checksum, StampDto stamp)
{
var ds = await _utils.GetDataset(orgSlug, dsSlug, true);
var validateChecksum = false;
if (ds is null)
{
_logger.LogInformation("Dataset does not exist, creating it");
await _datasetsManager.AddNew(orgSlug, new DatasetNewDto
{
Name = dsSlug,
Slug = dsSlug
});
_logger.LogInformation("New dataset {OrgSlug}/{DsSlug} created", orgSlug, dsSlug);
ds = await _utils.GetDataset(orgSlug, dsSlug);
}
else
{
if (!await _authManager.IsOwnerOrAdmin(ds))
{
throw new UnauthorizedException("The current user is not allowed to init push");
}
validateChecksum = true;
}
var ddb = _ddbManager.Get(orgSlug, ds.InternalRef);
Stamp ourStamp = null;
if (validateChecksum)
{
if (string.IsNullOrEmpty(checksum))
{
throw new InvalidOperationException("Checksum parameter missing (dataset exists)");
}
// Is a pull required? The checksum passed by client is the checksum of the stamp
// of the last sync. If it's different, the client should pull first.
ourStamp = DDBWrapper.GetStamp(ddb.DatasetFolderPath);
if (ourStamp.Checksum != checksum)
{
return(new PushInitResultDto
{
PullRequired = true
});
}
}
ourStamp ??= DDBWrapper.GetStamp(ddb.DatasetFolderPath);
// Perform delta with our ddb
var delta = DDBWrapper.Delta(new Stamp
{
Checksum = stamp.Checksum,
Entries = stamp.Entries,
Meta = stamp.Meta
}, ourStamp);
// Compute locals
var locals = DDBWrapper.ComputeDeltaLocals(delta, ddb.DatasetFolderPath);
// Generate UUID
var uuid = Guid.NewGuid().ToString();
// Create tmp folder
var baseTempFolder = ddb.GetTmpFolder("push-" + uuid);
// Save incoming stamp as well as our stamp in temp folder
await File.WriteAllTextAsync(Path.Combine(baseTempFolder, StampFileName),
JsonConvert.SerializeObject(stamp));
await File.WriteAllTextAsync(Path.Combine(baseTempFolder, OurStampFileName),
JsonConvert.SerializeObject(ourStamp));
// Return missing files list (excluding folders)
return(new PushInitResultDto
{
Token = uuid,
NeededFiles = delta.Adds
.Where(item => item.Hash.Length > 0 && !locals.ContainsKey(item.Hash))
.Select(item => item.Path)
.ToArray(),
NeededMeta = delta.MetaAdds.ToArray(),
PullRequired = false
});
}
public async Task <Dataset> GetDataset(string orgSlug, string dsSlug, bool retNullIfNotFound = false,
bool checkOwnership = true)
{
if (string.IsNullOrWhiteSpace(dsSlug))
{
throw new BadRequestException("Missing dataset id");
}
if (!dsSlug.IsValidSlug())
{
throw new BadRequestException("Invalid dataset id");
}
if (string.IsNullOrWhiteSpace(orgSlug))
{
throw new BadRequestException("Missing organization id");
}
if (!orgSlug.IsValidSlug())
{
throw new BadRequestException("Invalid organization id");
}
var org = _context.Organizations
.Include(item => item.Datasets)
.FirstOrDefault(item => item.Slug == orgSlug);
if (org == null)
{
throw new NotFoundException("Organization not found");
}
var dataset = org.Datasets.FirstOrDefault(item => item.Slug == dsSlug);
if (dataset == null)
{
if (retNullIfNotFound)
{
return(null);
}
throw new NotFoundException("Cannot find dataset");
}
var ddb = _ddbManager.Get(orgSlug, dataset.InternalRef);
var attributes = await ddb.GetAttributesAsync();
if (!attributes.IsPublic && !await _authManager.IsUserAdmin() && checkOwnership)
{
var currentUser = await _authManager.GetCurrentUser();
if (currentUser == null)
{
throw new UnauthorizedException("Invalid user");
}
if (org.OwnerId != null && org.OwnerId != currentUser.Id && org.Users.All(usr => usr.UserId != currentUser.Id))
{
throw new UnauthorizedException("The current user does not have access to this dataset");
}
}
return(dataset);
}