public async Task Commit(string orgSlug, string dsSlug, string token) { var ds = await _utils.GetDataset(orgSlug, dsSlug); if (!await _authManager.IsOwnerOrAdmin(ds)) { throw new UnauthorizedException("The current user is not allowed to commit to this dataset"); } var ddb = _ddbManager.Get(orgSlug, ds.InternalRef); var baseTempFolder = ddb.GetTmpFolder("push-" + token); var stampFilePath = Path.Combine(baseTempFolder, StampFileName); var ourStampFilePath = Path.Combine(baseTempFolder, OurStampFileName); var addTempFolder = Path.Combine(baseTempFolder, AddsTempFolder); var metaFile = Path.Combine(baseTempFolder, MetaFile); // Check push folder integrity if (!File.Exists(stampFilePath)) { throw new InvalidOperationException("Stamp not found"); } if (!File.Exists(ourStampFilePath)) { throw new InvalidOperationException("Our stamp not found"); } var stamp = JsonConvert.DeserializeObject <Stamp>(await File.ReadAllTextAsync(stampFilePath)); var ourStamp = JsonConvert.DeserializeObject <Stamp>(await File.ReadAllTextAsync(ourStampFilePath)); if (ourStamp == null) { throw new InvalidOperationException("Our stamp is invalid (cannot deserialize)"); } // Check that our stamp has not changed! If it has, another client // might have performed changes that could conflict with our operation // TODO: we could check for conflicts rather than failing and continue // the operation if no conflicts are detected. var currentStamp = DDBWrapper.GetStamp(ddb.DatasetFolderPath); if (currentStamp.Checksum != ourStamp.Checksum) { throw new InvalidOperationException("The dataset has been changed by another user while pushing. Please try again!"); } // Recompute delta var delta = DDBWrapper.Delta(stamp, currentStamp); // Create hard links for local files var _ = DDBWrapper.ComputeDeltaLocals(delta, ddb.DatasetFolderPath, addTempFolder); foreach (var add in delta.Adds.Where(item => item.Hash.Length > 0)) { if (!File.Exists(Path.Combine(addTempFolder, add.Path))) { throw new InvalidOperationException($"Cannot commit: missing '{add.Path}'"); } } // Read meta dump string metaDump = null; if (File.Exists(metaFile)) { metaDump = File.ReadAllText(metaFile); } // Applies delta var conflicts = DDBWrapper.ApplyDelta(delta, addTempFolder, ddb.DatasetFolderPath, MergeStrategy.KeepTheirs, metaDump); if (conflicts.Count > 0) { // This should never happen, since we merge conflicts using keep theirs throw new InvalidOperationException("Merge conflicts detected, try pulling first."); } // Delete temp folder Directory.Delete(baseTempFolder, true); // Build items foreach (var item in delta.Adds) { if (await ddb.IsBuildableAsync(item.Path)) { _backgroundJob.Enqueue(() => HangfireUtils.BuildWrapper(ddb, item.Path, false, null)); } } if (await ddb.IsBuildPendingAsync()) { _logger.LogInformation("Items are pending build, retriggering build"); var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildPendingWrapper(ddb, null)); _logger.LogInformation("Background job id is {JobId}", jobId); } }
public async Task <EntryDto> AddNew(string orgSlug, string dsSlug, string path, Stream stream = null) { var ds = await _utils.GetDataset(orgSlug, dsSlug); _logger.LogInformation("In AddNew('{OrgSlug}/{DsSlug}')", orgSlug, dsSlug); if (!await _authManager.IsOwnerOrAdmin(ds)) { throw new UnauthorizedException("The current user is not allowed to edit dataset"); } var ddb = _ddbManager.Get(orgSlug, ds.InternalRef); // If it's a folder if (stream == null) { if (await ddb.EntryExistsAsync(path)) { throw new InvalidOperationException("Cannot create a folder on another entry"); } if (path == DDB.DatabaseFolderName) { throw new InvalidOperationException($"'{DDB.DatabaseFolderName}' is a reserved folder name"); } _logger.LogInformation("Adding folder to DDB"); // Add to DDB await ddb.AddAsync(path); _logger.LogInformation("Added to DDB"); return(new EntryDto { Path = path, Type = EntryType.Directory, Size = 0 }); } // Check user storage space await _utils.CheckCurrentUserStorage(stream.Length); var localFilePath = ddb.GetLocalPath(path); CommonUtils.EnsureSafePath(localFilePath); _logger.LogInformation("Local file path is '{LocalFilePath}'", localFilePath); // Write down the file await using (var localFileStream = File.OpenWrite(localFilePath)) await stream.CopyToAsync(localFileStream); _logger.LogInformation("File saved, adding to DDB"); ddb.AddRaw(localFilePath); _logger.LogInformation("Added to DDB, checking entry now..."); var entry = await ddb.GetEntryAsync(path); if (entry == null) { throw new InvalidOperationException("Cannot find just added file!"); } _logger.LogInformation("Entry OK"); if (await ddb.IsBuildableAsync(entry.Path)) { _logger.LogInformation("This item is buildable, build it!"); var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildWrapper(ddb, path, false, null)); _logger.LogInformation("Background job id is {JobId}", jobId); } else if (await ddb.IsBuildPendingAsync()) { _logger.LogInformation("Items are pending build, retriggering build"); var jobId = _backgroundJob.Enqueue(() => HangfireUtils.BuildPendingWrapper(ddb, null)); _logger.LogInformation("Background job id is {JobId}", jobId); } return(entry.ToDto()); }