public void Should_resume_to_rename_file_if_appropriate_config_exists() { var client = NewAsyncClient(); var rfs = GetFileSystem(); string fileName = FileHeader.Canonize("file.bin"); string rename = FileHeader.Canonize("renamed.bin"); client.UploadAsync(fileName, new RandomStream(1)).Wait(); // create config to say to the server that rename operation performed last time were not finished var renameOpConfig = RavenFileNameHelper.RenameOperationConfigNameForFile(fileName); var renameOperation = new RenameFileOperation(fileName, rename, client.GetAsync(new[] { fileName }).Result[0].Etag, new RavenJObject()); rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(renameOpConfig, renameOperation)); rfs.Files.ResumeFileRenamingAsync().Wait(); IEnumerable <string> configNames = null; rfs.Storage.Batch(accessor => configNames = accessor.GetConfigNames(0, 10).ToArray()); Assert.DoesNotContain(renameOpConfig, configNames); var renamedMetadata = client.GetMetadataForAsync(rename).Result; Assert.NotNull(renamedMetadata); var results = client.SearchOnDirectoryAsync("/").Result; // make sure that indexes are updated Assert.Equal(1, results.FileCount); Assert.Equal(rename, results.Files[0].FullPath); }
public async Task Should_resume_file_renaming_from_client() { var client = NewAsyncClient(); var rfs = GetFileSystem(); string fileName = FileHeader.Canonize("file.bin"); string rename = FileHeader.Canonize("renamed.bin"); await client.UploadAsync(fileName, new RandomStream(1)); // create config to say to the server that rename operation performed last time were not finished var renameOpConfig = RavenFileNameHelper.RenameOperationConfigNameForFile(fileName); var renameOperation = new RenameFileOperation(fileName, rename, client.GetAsync(new[] { fileName }).Result[0].Etag, new RavenJObject()); rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(renameOpConfig, renameOperation)); await client.Storage.RetryRenamingAsync(); IEnumerable <string> configNames = await client.Configuration.GetKeyNamesAsync(); Assert.DoesNotContain(renameOpConfig, configNames); var renamedMetadata = await client.GetMetadataForAsync(rename); Assert.NotNull(renamedMetadata); }
public async Task resumed_rename_operation_needs_to_take_into_account_file_etag_to_avoid_renaming_next_version_of_file() { string name = FileHeader.Canonize("file.bin"); string renamed = FileHeader.Canonize("renamed.bin"); using (var store = NewStore()) { var rfs = GetFileSystem(); await store.AsyncFilesCommands.UploadAsync(name, new MemoryStream(), new RavenJObject { { "version", 1 } }); // instead of this: // await store.AsyncFilesCommands.RenameAsync(fileName, newName); // let's create a config to indicate rename operation - for example restart in the middle could happen var renameOpConfig = RavenFileNameHelper.RenameOperationConfigNameForFile(name); var renameOperation = new RenameFileOperation(name, renamed, (await store.AsyncFilesCommands.GetAsync(new [] { name }))[0].Etag, new RavenJObject { { "version", 1 } }); rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(renameOpConfig, renameOperation)); // upload new file under the same name, before ResumeFileRenamingAsync is called await store.AsyncFilesCommands.UploadAsync(name, new MemoryStream(), new RavenJObject { { "version", 2 } }); await rfs.Files.ResumeFileRenamingAsync(); var version2 = await store.AsyncFilesCommands.GetMetadataForAsync(name); Assert.NotNull(version2); Assert.Equal(2, version2["version"]); Assert.DoesNotContain(RavenFileNameHelper.RenameOperationConfigNameForFile(renameOperation.Name), await store.AsyncFilesCommands.Configuration.GetKeyNamesAsync()); } }
public async Task Should_resume_file_renaming_from_client() { var client = NewAsyncClient(); var rfs = GetRavenFileSystem(); const string fileName = "file.bin"; const string rename = "renamed.bin"; await client.UploadAsync(fileName, new RandomStream(1)); // create config to say to the server that rename operation performed last time were not finished var renameOpConfig = RavenFileNameHelper.RenameOperationConfigNameForFile(fileName); var renameOperation = new RenameFileOperation { Name = fileName, Rename = rename, MetadataAfterOperation = new RavenJObject().WithETag(Guid.Empty) }; rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(renameOpConfig, renameOperation)); await client.Storage.RetryRenamingAsync(); IEnumerable <string> configNames = await client.Configuration.GetKeyNamesAsync(); Assert.DoesNotContain(renameOpConfig, configNames); var renamedMetadata = await client.GetMetadataForAsync(rename); Assert.NotNull(renamedMetadata); }
public HttpResponseMessage Patch(string name, string rename) { name = FileHeader.Canonize(name); rename = FileHeader.Canonize(rename); var etag = GetEtag(); Storage.Batch(accessor => { FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name); var existingFile = accessor.ReadFile(name); if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker)) { throw new FileNotFoundException(); } var renamingFile = accessor.ReadFile(rename); if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false) { throw new FileExistsException("Cannot rename because file " + rename + " already exists"); } var metadata = existingFile.Metadata; if (etag != null && existingFile.Etag != etag) { throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag") { ActualETag = existingFile.Etag, ExpectedETag = etag } } ; Historian.UpdateLastModified(metadata); var operation = new RenameFileOperation { FileSystem = FileSystem.Name, Name = name, Rename = rename, MetadataAfterOperation = metadata }; accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation)); accessor.PulseTransaction(); // commit rename operation config Files.ExecuteRenameOperation(operation); }); Log.Debug("File '{0}' was renamed to '{1}'", name, rename); FileSystem.Synchronizations.StartSynchronizeDestinationsInBackground(); return(GetEmptyMessage(HttpStatusCode.NoContent)); }
public HttpResponseMessage Patch(string name, string rename) { name = FileHeader.Canonize(name); rename = FileHeader.Canonize(rename); var etag = GetEtag(); if (rename.Length > SystemParameters.KeyMost) { Log.Debug("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename); return(GetMessageWithString(string.Format("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename), HttpStatusCode.BadRequest)); } Storage.Batch(accessor => { FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name); var existingFile = accessor.ReadFile(name); if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker)) { throw new FileNotFoundException(); } var renamingFile = accessor.ReadFile(rename); if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false) { throw new FileExistsException("Cannot rename because file " + rename + " already exists"); } var metadata = existingFile.Metadata; if (etag != null && existingFile.Etag != etag) { throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag") { ActualETag = existingFile.Etag, ExpectedETag = etag } } ; Historian.UpdateLastModified(metadata); var operation = new RenameFileOperation(name, rename, existingFile.Etag, metadata); accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation)); accessor.PulseTransaction(); // commit rename operation config Files.ExecuteRenameOperation(operation); }); Log.Debug("File '{0}' was renamed to '{1}'", name, rename); SynchronizationTask.Context.NotifyAboutWork(); return(GetEmptyMessage(HttpStatusCode.NoContent)); }
public HttpResponseMessage Patch(string name, string rename) { name = RavenFileNameHelper.RavenPath(name); rename = RavenFileNameHelper.RavenPath(rename); try { ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor => { AssertFileIsNotBeingSynced(name, accessor, true); var metadata = accessor.GetFile(name, 0, 0).Metadata; if (metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker)) { throw new FileNotFoundException(); } var existingHeader = accessor.ReadFile(rename); if (existingHeader != null && !existingHeader.Metadata.ContainsKey(SynchronizationConstants.RavenDeleteMarker)) { throw new HttpResponseException( Request.CreateResponse(HttpStatusCode.Forbidden, new InvalidOperationException("Cannot rename because file " + rename + " already exists"))); } Historian.UpdateLastModified(metadata); var operation = new RenameFileOperation { FileSystem = FileSystem.Name, Name = name, Rename = rename, MetadataAfterOperation = metadata }; accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation)); accessor.PulseTransaction(); // commit rename operation config StorageOperationsTask.RenameFile(operation); }), ConcurrencyResponseException); } catch (FileNotFoundException) { log.Debug("Cannot rename a file '{0}' to '{1}' because a file was not found", name, rename); return(GetEmptyMessage(HttpStatusCode.NotFound)); } log.Debug("File '{0}' was renamed to '{1}'", name, rename); StartSynchronizeDestinationsInBackground(); return(GetMessageWithString("", HttpStatusCode.NoContent)); }
public void ExecuteRenameOperation(RenameFileOperation operation) { var configName = RavenFileNameHelper.RenameOperationConfigNameForFile(operation.Name); Storage.Batch(accessor => { AssertRenameOperationNotVetoed(operation.Name, operation.Rename); Publisher.Publish(new FileChangeNotification { File = operation.Name, Action = FileChangeAction.Renaming }); var previousRenameTombstone = accessor.ReadFile(operation.Rename); if (previousRenameTombstone != null && previousRenameTombstone.Metadata[SynchronizationConstants.RavenDeleteMarker] != null) { // if there is a tombstone delete it accessor.Delete(previousRenameTombstone.FullPath); } FileSystem.RenameTriggers.Apply(trigger => trigger.OnRename(operation.Name, operation.MetadataAfterOperation)); accessor.RenameFile(operation.Name, operation.Rename, true); accessor.UpdateFileMetadata(operation.Rename, operation.MetadataAfterOperation, null); FileSystem.RenameTriggers.Apply(trigger => trigger.AfterRename(operation.Name, operation.Rename, operation.MetadataAfterOperation)); // copy renaming file metadata and set special markers var tombstoneMetadata = new RavenJObject(operation.MetadataAfterOperation).WithRenameMarkers(operation.Rename); accessor.PutFile(operation.Name, 0, tombstoneMetadata, true); // put rename tombstone // let's bump renamed doc etag so it'll be greater than tombstone var touchResult = accessor.TouchFile(operation.Rename, null); accessor.DeleteConfig(configName); Search.Delete(operation.Name); Search.Index(operation.Rename, operation.MetadataAfterOperation, touchResult.Etag); }); Publisher.Publish(new ConfigurationChangeNotification { Name = configName, Action = ConfigurationChangeAction.Set }); Publisher.Publish(new FileChangeNotification { File = operation.Rename, Action = FileChangeAction.Renamed }); }
public void RenameFile(RenameFileOperation operation) { var configName = RavenFileNameHelper.RenameOperationConfigNameForFile(operation.Name); notificationPublisher.Publish(new FileChangeNotification { File = FilePathTools.Cannoicalise(operation.Name), Action = FileChangeAction.Renaming }); storage.Batch(accessor => { var previousRenameTombstone = accessor.ReadFile(operation.Rename); if (previousRenameTombstone != null && previousRenameTombstone.Metadata[SynchronizationConstants.RavenDeleteMarker] != null) { // if there is a tombstone delete it accessor.Delete(previousRenameTombstone.FullPath); } accessor.RenameFile(operation.Name, operation.Rename, true); accessor.UpdateFileMetadata(operation.Rename, operation.MetadataAfterOperation); // copy renaming file metadata and set special markers var tombstoneMetadata = new RavenJObject(operation.MetadataAfterOperation).WithRenameMarkers(operation.Rename); accessor.PutFile(operation.Name, 0, tombstoneMetadata, true); // put rename tombstone accessor.DeleteConfig(configName); search.Delete(operation.Name); search.Index(operation.Rename, operation.MetadataAfterOperation); }); notificationPublisher.Publish(new ConfigurationChangeNotification { Name = configName, Action = ConfigurationChangeAction.Set }); notificationPublisher.Publish(new FileChangeNotification { File = FilePathTools.Cannoicalise(operation.Rename), Action = FileChangeAction.Renamed }); }
public void Should_resume_to_rename_file_if_appropriate_config_exists() { var client = NewClient(); var rfs = GetRavenFileSystem(); const string fileName = "file.bin"; const string rename = "renamed.bin"; client.UploadAsync(fileName, new RandomStream(1)).Wait(); // create config to say to the server that rename operation performed last time were not finished var renameOpConfig = RavenFileNameHelper.RenameOperationConfigNameForFile(fileName); var renameOperation = new RenameFileOperation { Name = fileName, Rename = rename, MetadataAfterOperation = new RavenJObject().WithETag(Guid.Empty) }; rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(renameOpConfig, renameOperation)); rfs.StorageOperationsTask.ResumeFileRenamingAsync().Wait(); IEnumerable <string> configNames = null; rfs.Storage.Batch(accessor => configNames = accessor.GetConfigNames(0, 10).ToArray()); Assert.DoesNotContain(renameOpConfig, configNames); var renamedMetadata = client.GetMetadataForAsync(rename).Result; Assert.NotNull(renamedMetadata); var results = client.GetFilesAsync("/").Result; // make sure that indexes are updated Assert.Equal(1, results.FileCount); Assert.Equal(rename, results.Files[0].Name); }
public Task ResumeFileRenamingAsync() { var filesToRename = new List <RenameFileOperation>(); Storage.Batch(accessor => { var renameOpConfigs = accessor.GetConfigsStartWithPrefix(RavenFileNameHelper.RenameOperationConfigPrefix, 0, 10); filesToRename = renameOpConfigs.Select(config => config.JsonDeserialization <RenameFileOperation>()).ToList(); }); if (filesToRename.Count == 0) { return(Task.FromResult <object>(null)); } var tasks = new List <Task>(); foreach (var item in filesToRename) { var renameOperation = item; if (IsRenameInProgress(renameOperation.Name)) { continue; } FileHeader existingFile = null; Storage.Batch(accessor => existingFile = accessor.ReadFile(renameOperation.Name)); if (existingFile == null) { continue; } if (renameOperation.Etag != null && renameOperation.Etag != existingFile.Etag) { Storage.Batch(accessor => accessor.DeleteConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(renameOperation.Name))); continue; } Log.Debug("Starting to resume a rename operation of a file '{0}' to '{1}'", renameOperation.Name, renameOperation.Rename); var renameTask = Task.Run(() => { try { ExecuteRenameOperation(renameOperation); Log.Debug("File '{0}' was renamed to '{1}'", renameOperation.Name, renameOperation.Rename); } catch (Exception e) { Log.Warn(string.Format("Could not rename file '{0}' to '{1}'", renameOperation.Name, renameOperation.Rename), e); throw; } }); renameFileTasks.AddOrUpdate(renameOperation.Name, renameTask, (file, oldTask) => renameTask); tasks.Add(renameTask); } return(Task.WhenAll(tasks)); }