public void Should_create_apropriate_config_after_indicating_file_to_delete()
{
var filename = FileHeader.Canonize("toDelete.bin");
var client = NewAsyncClient();
var rfs = GetFileSystem();
client.UploadAsync(filename, new MemoryStream(new byte[] { 1, 2, 3, 4, 5 })).Wait();
rfs.Files.IndicateFileToDelete(filename, null);
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(filename))));
Assert.Equal(RavenFileNameHelper.DeletingFileName(filename), deleteFile.CurrentFileName);
Assert.Equal(filename, deleteFile.OriginalFileName);
}
public void IndicateFileToDelete(string fileName, Etag etag)
{
var deletingFileName = RavenFileNameHelper.DeletingFileName(fileName);
var fileExists = true;
Storage.Batch(accessor =>
{
AssertDeleteOperationNotVetoed(fileName);
var existingFile = accessor.ReadFile(fileName);
if (existingFile == null)
{
// do nothing if file does not exist
fileExists = false;
return;
}
if (existingFile.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if it is a tombstone drop it
accessor.Delete(fileName);
fileExists = false;
return;
}
if (etag != null && existingFile.Etag != etag)
{
throw new ConcurrencyException("Operation attempted on file '" + fileName + "' using a non current etag")
{
ActualETag = existingFile.Etag,
ExpectedETag = etag
}
}
;
var metadata = new RavenJObject(existingFile.Metadata).WithDeleteMarker();
var renameSucceeded = false;
int deleteAttempts = 0;
do
{
try
{
accessor.RenameFile(fileName, deletingFileName);
renameSucceeded = true;
}
catch (FileExistsException) // it means that .deleting file was already existed
{
var deletingFileHeader = accessor.ReadFile(deletingFileName);
if (deletingFileHeader != null && deletingFileHeader.Equals(existingFile))
{
fileExists = false; // the same file already marked as deleted no need to do it again
return;
}
if (deleteAttempts++ > 128)
{
Log.Warn("Could not rename a file '{0}' when a delete operation was performed", fileName);
throw;
}
// we need to use different name to do a file rename
deletingFileName = RavenFileNameHelper.DeletingFileName(fileName, RandomProvider.GetThreadRandom().Next());
}
} while (renameSucceeded == false);
accessor.UpdateFileMetadata(deletingFileName, metadata, null);
accessor.DecrementFileCount(deletingFileName);
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was renamed to '{1}' and marked as deleted", fileName, deletingFileName);
}
var configName = RavenFileNameHelper.DeleteOperationConfigNameForFile(deletingFileName);
var operation = new DeleteFileOperation {
OriginalFileName = fileName, CurrentFileName = deletingFileName
};
accessor.SetConfig(configName, JsonExtensions.ToJObject(operation));
FileSystem.DeleteTriggers.Apply(trigger => trigger.AfterDelete(fileName));
Publisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
Publisher.Publish(new FileChangeNotification {
File = fileName, Action = FileChangeAction.Delete
});
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was deleted", fileName);
}
});
if (fileExists)
{
Search.Delete(fileName);
Search.Delete(deletingFileName);
}
}
public void Should_not_delete_downloading_file_if_synchronization_retry_is_being_performed()
{
const string fileName = "file.bin";
var downloadingFileName = RavenFileNameHelper.DownloadingFileName(fileName);
var client = NewAsyncClient();
var rfs = GetFileSystem();
client.UploadAsync(fileName, new RandomStream(1)).Wait();
client.UploadAsync(downloadingFileName, new RandomStream(1)).Wait();
rfs.Files.IndicateFileToDelete(downloadingFileName, null);
rfs.Storage.Batch(accessor =>
accessor.SetConfigurationValue(RavenFileNameHelper.SyncLockNameForFile(fileName), LockFileTests.SynchronizationConfig(DateTime.UtcNow)));
rfs.Files.CleanupDeletedFilesAsync().Wait();
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(downloadingFileName))));
Assert.Equal(RavenFileNameHelper.DeletingFileName(downloadingFileName), deleteFile.CurrentFileName);
Assert.Equal(downloadingFileName, deleteFile.OriginalFileName);
}
public void Should_not_perform_file_delete_if_it_is_being_synced()
{
var filename = FileHeader.Canonize("file.bin");
var client = NewAsyncClient();
var rfs = GetFileSystem();
client.UploadAsync("file.bin", new MemoryStream(new byte[] { 1, 2, 3, 4, 5 })).Wait();
rfs.Files.IndicateFileToDelete(filename, null);
rfs.Storage.Batch(accessor =>
accessor.SetConfigurationValue(RavenFileNameHelper.SyncLockNameForFile(filename),
LockFileTests.SynchronizationConfig(DateTime.UtcNow)));
rfs.Files.CleanupDeletedFilesAsync().Wait();
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(filename))));
Assert.Equal(RavenFileNameHelper.DeletingFileName(filename), deleteFile.CurrentFileName);
Assert.Equal(filename, deleteFile.OriginalFileName);
}
public Task CleanupDeletedFilesAsync()
{
if (maxNumberOfConcurrentDeletionsInBackground.CurrentCount == 0)
{
return(new CompletedTask());
}
var filesToDelete = new List <DeleteFileOperation>();
Storage.Batch(accessor => filesToDelete = accessor.GetConfigsStartWithPrefix(RavenFileNameHelper.DeleteOperationConfigPrefix, 0, MaxNumberOfFilesToDeleteByCleanupTaskRun)
.Select(config => config.JsonDeserialization <DeleteFileOperation>())
.ToList());
if (filesToDelete.Count == 0)
{
return(new CompletedTask());
}
var tasks = new List <Task>();
foreach (var fileToDelete in filesToDelete)
{
var deletingFileName = fileToDelete.CurrentFileName;
if (IsDeleteInProgress(deletingFileName))
{
continue;
}
if (IsUploadInProgress(fileToDelete.OriginalFileName))
{
continue;
}
if (IsSynchronizationInProgress(fileToDelete.OriginalFileName))
{
continue;
}
if (fileToDelete.OriginalFileName.EndsWith(RavenFileNameHelper.DownloadingFileSuffix)) // if it's .downloading file
{
if (IsSynchronizationInProgress(SynchronizedFileName(fileToDelete.OriginalFileName))) // and file is being synced
{
continue;
}
}
if (Log.IsDebugEnabled)
{
Log.Debug("Starting to delete file '{0}' from storage", deletingFileName);
}
var deleteTask = new Task(() =>
{
try
{
Storage.Batch(accessor => accessor.Delete(deletingFileName));
}
catch (Exception e)
{
var warnMessage = string.Format("Could not delete file '{0}' from storage", deletingFileName);
Log.Warn(warnMessage, e);
throw new InvalidOperationException(warnMessage, e);
}
var configName = RavenFileNameHelper.DeleteOperationConfigNameForFile(deletingFileName);
Storage.Batch(accessor => accessor.DeleteConfig(configName));
Publisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Delete
});
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was deleted from storage", deletingFileName);
}
});
deleteTask.ContinueWith(x =>
{
Task _;
deleteFileTasks.TryRemove(deletingFileName, out _);
maxNumberOfConcurrentDeletionsInBackground.Release();
});
maxNumberOfConcurrentDeletionsInBackground.Wait();
deleteTask.Start();
deleteFileTasks.AddOrUpdate(deletingFileName, deleteTask, (file, oldTask) => deleteTask);
tasks.Add(deleteTask);
}
return(Task.WhenAll(tasks));
}
public Task CleanupDeletedFilesAsync()
{
var filesToDelete = new List <DeleteFileOperation>();
storage.Batch(accessor => filesToDelete = accessor.GetConfigsStartWithPrefix(RavenFileNameHelper.DeleteOperationConfigPrefix, 0, 10)
.Select(config => config.JsonDeserialization <DeleteFileOperation>())
.ToList());
if (filesToDelete.Count == 0)
{
return(Task.FromResult <object>(null));
}
var tasks = new List <Task>();
foreach (var fileToDelete in filesToDelete)
{
var deletingFileName = fileToDelete.CurrentFileName;
if (IsDeleteInProgress(deletingFileName))
{
continue;
}
if (IsUploadInProgress(fileToDelete.OriginalFileName))
{
continue;
}
if (IsSynchronizationInProgress(fileToDelete.OriginalFileName))
{
continue;
}
if (fileToDelete.OriginalFileName.EndsWith(RavenFileNameHelper.DownloadingFileSuffix)) // if it's .downloading file
{
if (IsSynchronizationInProgress(SynchronizedFileName(fileToDelete.OriginalFileName))) // and file is being synced
{
continue;
}
}
Log.Debug("Starting to delete file '{0}' from storage", deletingFileName);
var deleteTask = Task.Run(() =>
{
try
{
ConcurrencyAwareExecutor.Execute(() => storage.Batch(accessor => accessor.Delete(deletingFileName)), retries: 1);
}
catch (Exception e)
{
Log.WarnException(string.Format("Could not delete file '{0}' from storage", deletingFileName), e);
return;
}
var configName = RavenFileNameHelper.DeleteOperationConfigNameForFile(deletingFileName);
storage.Batch(accessor => accessor.DeleteConfig(configName));
notificationPublisher.Publish(new ConfigurationChangeNotification
{
Name = configName,
Action = ConfigurationChangeAction.Delete
});
Log.Debug("File '{0}' was deleted from storage", deletingFileName);
});
deleteFileTasks.AddOrUpdate(deletingFileName, deleteTask, (file, oldTask) => deleteTask);
tasks.Add(deleteTask);
}
return(Task.WhenAll(tasks));
}
public void IndicateFileToDelete(string fileName)
{
var deletingFileName = RavenFileNameHelper.DeletingFileName(fileName);
var fileExists = true;
storage.Batch(accessor =>
{
var existingFileHeader = accessor.ReadFile(fileName);
if (existingFileHeader == null)
{
// do nothing if file does not exist
fileExists = false;
return;
}
if (existingFileHeader.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if it is a tombstone drop it
accessor.Delete(fileName);
fileExists = false;
return;
}
var metadata = new RavenJObject(existingFileHeader.Metadata).WithDeleteMarker();
var renameSucceeded = false;
int deleteVersion = 0;
do
{
try
{
accessor.RenameFile(fileName, deletingFileName);
renameSucceeded = true;
}
catch (FileExistsException) // it means that .deleting file was already existed
{
var deletingFileHeader = accessor.ReadFile(deletingFileName);
if (deletingFileHeader != null && deletingFileHeader.Equals(existingFileHeader))
{
fileExists = false; // the same file already marked as deleted no need to do it again
return;
}
// we need to use different name to do a file rename
deleteVersion++;
deletingFileName = RavenFileNameHelper.DeletingFileName(fileName, deleteVersion);
}
} while (!renameSucceeded && deleteVersion < 128);
if (renameSucceeded)
{
accessor.UpdateFileMetadata(deletingFileName, metadata);
accessor.DecrementFileCount(deletingFileName);
Log.Debug(string.Format("File '{0}' was renamed to '{1}' and marked as deleted", fileName, deletingFileName));
var configName = RavenFileNameHelper.DeleteOperationConfigNameForFile(deletingFileName);
var operation = new DeleteFileOperation {
OriginalFileName = fileName, CurrentFileName = deletingFileName
};
accessor.SetConfig(configName, JsonExtensions.ToJObject(operation));
notificationPublisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
}
else
{
Log.Warn("Could not rename a file '{0}' when a delete operation was performed", fileName);
}
});
if (fileExists)
{
search.Delete(fileName);
search.Delete(deletingFileName);
}
}