public async Task Should_remove_deleting_file_and_its_pages_after_storage_cleanup()
{
const int numberOfPages = 10;
var client = NewAsyncClient();
var rfs = GetFileSystem();
var bytes = new byte[numberOfPages * StorageConstants.MaxPageSize];
new Random().NextBytes(bytes);
await client.UploadAsync("toDelete.bin", new MemoryStream(bytes));
rfs.Files.IndicateFileToDelete(FileHeader.Canonize("toDelete.bin"), null);
await rfs.Files.CleanupDeletedFilesAsync();
Assert.Throws(typeof(FileNotFoundException),
() => rfs.Storage.Batch(accessor => accessor.GetFile(RavenFileNameHelper.DeletingFileName("toDelete.bin"), 0, 10)));
for (var i = 1; i <= numberOfPages; i++)
{
var pageId = 0;
var i1 = i;
rfs.Storage.Batch(accessor => pageId = accessor.ReadPage(i1, null));
Assert.Equal(-1, pageId); // if page does not exist we return -1
}
}
public void Should_not_delete_downloading_file_if_synchronization_retry_is_being_performed()
{
const string fileName = "file.bin";
var downloadingFileName = RavenFileNameHelper.DownloadingFileName(fileName);
var client = NewAsyncClient();
var rfs = GetFileSystem();
client.UploadAsync(fileName, new RandomStream(1)).Wait();
client.UploadAsync(downloadingFileName, new RandomStream(1)).Wait();
rfs.Files.IndicateFileToDelete(downloadingFileName, null);
rfs.Storage.Batch(accessor =>
accessor.SetConfigurationValue(RavenFileNameHelper.SyncLockNameForFile(fileName), LockFileTests.SynchronizationConfig(DateTime.UtcNow)));
rfs.Files.CleanupDeletedFilesAsync().Wait();
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(downloadingFileName))));
Assert.Equal(RavenFileNameHelper.DeletingFileName(downloadingFileName), deleteFile.CurrentFileName);
Assert.Equal(downloadingFileName, deleteFile.OriginalFileName);
}
public async Task Upload_before_performing_cleanup_do_a_rename_by_adding_version_number()
{
var client = NewAsyncClient();
var rfs = GetFileSystem();
await client.UploadAsync("file.bin", new RandomStream(1));
// this upload should indicate old file to delete
await client.UploadAsync("file.bin", new RandomStream(1));
// upload again - note that actual file delete was not performed yet
await client.UploadAsync("file.bin", new RandomStream(1));
List <string> configNames = null;
rfs.Storage.Batch(
accessor =>
configNames =
accessor.GetConfigNames(0, 10).ToArray().Where(x => x.StartsWith(RavenFileNameHelper.DeleteOperationConfigPrefix)).OrderBy(x => x.Length).ToList());
Assert.Equal(2, configNames.Count);
Assert.Equal(RavenFileNameHelper.DeleteOperationConfigPrefix + RavenFileNameHelper.DeletingFileName("file.bin"), configNames[0]);
Assert.True(configNames[1].StartsWith(RavenFileNameHelper.DeleteOperationConfigPrefix + "/file.bin") &&
configNames[1].EndsWith(RavenFileNameHelper.DeletingFileSuffix)); // number in the middle is used to avoid duplicates
}
public void Should_not_perform_file_delete_if_it_is_being_synced()
{
var filename = FileHeader.Canonize("file.bin");
var client = NewAsyncClient();
var rfs = GetFileSystem();
client.UploadAsync("file.bin", new MemoryStream(new byte[] { 1, 2, 3, 4, 5 })).Wait();
rfs.Files.IndicateFileToDelete(filename, null);
rfs.Storage.Batch(accessor =>
accessor.SetConfigurationValue(RavenFileNameHelper.SyncLockNameForFile(filename),
LockFileTests.SynchronizationConfig(DateTime.UtcNow)));
rfs.Files.CleanupDeletedFilesAsync().Wait();
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(filename))));
Assert.Equal(RavenFileNameHelper.DeletingFileName(filename), deleteFile.CurrentFileName);
Assert.Equal(filename, deleteFile.OriginalFileName);
}
public async Task Upload_before_performing_cleanup_do_a_rename_by_adding_version_number()
{
var client = NewAsyncClient();
var rfs = GetFileSystem();
await client.UploadAsync("file.bin", new RandomStream(1));
// this upload should indicate old file to delete
await client.UploadAsync("file.bin", new RandomStream(1));
// upload again - note that actual file delete was not performed yet
await client.UploadAsync("file.bin", new RandomStream(1));
List <string> configNames = null;
rfs.Storage.Batch(
accessor =>
configNames =
accessor.GetConfigNames(0, 10).ToArray().Where(x => x.StartsWith(RavenFileNameHelper.DeleteOperationConfigPrefix)).ToList());
Assert.Equal(2, configNames.Count());
foreach (var configName in configNames)
{
Assert.True(RavenFileNameHelper.DeleteOperationConfigPrefix + RavenFileNameHelper.DeletingFileName("file.bin") == configName ||
RavenFileNameHelper.DeleteOperationConfigPrefix + RavenFileNameHelper.DeletingFileName("file.bin1") == configName); // 1 indicate delete version
}
}
public async Task Can_force_storage_cleanup_from_client()
{
var client = NewAsyncClient();
await client.UploadAsync("toDelete.bin", new MemoryStream(new byte[] { 1, 2, 3, 4, 5 }));
await client.DeleteAsync("toDelete.bin");
await client.Storage.CleanUpAsync();
var configNames = await client.Configuration.GetKeyNamesAsync();
Assert.DoesNotContain(
RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName("toDelete.bin")),
configNames);
}
public void Should_create_apropriate_config_after_indicating_file_to_delete()
{
var client = NewAsyncClient();
var rfs = GetRavenFileSystem();
client.UploadAsync("toDelete.bin", new MemoryStream(new byte[] { 1, 2, 3, 4, 5 })).Wait();
rfs.StorageOperationsTask.IndicateFileToDelete("toDelete.bin");
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName("toDelete.bin"))));
Assert.Equal(RavenFileNameHelper.DeletingFileName("toDelete.bin"), deleteFile.CurrentFileName);
Assert.Equal("toDelete.bin", deleteFile.OriginalFileName);
}
public async Task Should_remove_file_deletion_config_after_storage_cleanup()
{
var client = NewAsyncClient();
var rfs = GetFileSystem();
await client.UploadAsync("toDelete.bin", new MemoryStream(new byte[] { 1, 2, 3, 4, 5 }));
rfs.Files.IndicateFileToDelete("toDelete.bin", null);
await rfs.Files.CleanupDeletedFilesAsync();
IEnumerable <string> configNames = null;
rfs.Storage.Batch(accessor => configNames = accessor.GetConfigNames(0, 10).ToArray());
Assert.DoesNotContain(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName("toDelete.bin")), configNames);
}
public void IndicateFileToDelete(string fileName)
{
var deletingFileName = RavenFileNameHelper.DeletingFileName(fileName);
var fileExists = true;
storage.Batch(accessor =>
{
var existingFileHeader = accessor.ReadFile(fileName);
if (existingFileHeader == null)
{
// do nothing if file does not exist
fileExists = false;
return;
}
if (existingFileHeader.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if it is a tombstone drop it
accessor.Delete(fileName);
fileExists = false;
return;
}
var metadata = new RavenJObject(existingFileHeader.Metadata).WithDeleteMarker();
var renameSucceeded = false;
int deleteVersion = 0;
do
{
try
{
accessor.RenameFile(fileName, deletingFileName);
renameSucceeded = true;
}
catch (FileExistsException) // it means that .deleting file was already existed
{
var deletingFileHeader = accessor.ReadFile(deletingFileName);
if (deletingFileHeader != null && deletingFileHeader.Equals(existingFileHeader))
{
fileExists = false; // the same file already marked as deleted no need to do it again
return;
}
// we need to use different name to do a file rename
deleteVersion++;
deletingFileName = RavenFileNameHelper.DeletingFileName(fileName, deleteVersion);
}
} while (!renameSucceeded && deleteVersion < 128);
if (renameSucceeded)
{
accessor.UpdateFileMetadata(deletingFileName, metadata);
accessor.DecrementFileCount(deletingFileName);
Log.Debug(string.Format("File '{0}' was renamed to '{1}' and marked as deleted", fileName, deletingFileName));
var configName = RavenFileNameHelper.DeleteOperationConfigNameForFile(deletingFileName);
var operation = new DeleteFileOperation {
OriginalFileName = fileName, CurrentFileName = deletingFileName
};
accessor.SetConfig(configName, JsonExtensions.ToJObject(operation));
notificationPublisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
}
else
{
Log.Warn("Could not rename a file '{0}' when a delete operation was performed", fileName);
}
});
if (fileExists)
{
search.Delete(fileName);
search.Delete(deletingFileName);
}
}
public void IndicateFileToDelete(string fileName, Etag etag)
{
var deletingFileName = RavenFileNameHelper.DeletingFileName(fileName);
var fileExists = true;
Storage.Batch(accessor =>
{
AssertDeleteOperationNotVetoed(fileName);
var existingFile = accessor.ReadFile(fileName);
if (existingFile == null)
{
// do nothing if file does not exist
fileExists = false;
return;
}
if (existingFile.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if it is a tombstone drop it
accessor.Delete(fileName);
fileExists = false;
return;
}
if (etag != null && existingFile.Etag != etag)
{
throw new ConcurrencyException("Operation attempted on file '" + fileName + "' using a non current etag")
{
ActualETag = existingFile.Etag,
ExpectedETag = etag
}
}
;
var metadata = new RavenJObject(existingFile.Metadata).WithDeleteMarker();
var renameSucceeded = false;
int deleteVersion = 0;
do
{
try
{
accessor.RenameFile(fileName, deletingFileName);
renameSucceeded = true;
}
catch (FileExistsException) // it means that .deleting file was already existed
{
var deletingFileHeader = accessor.ReadFile(deletingFileName);
if (deletingFileHeader != null && deletingFileHeader.Equals(existingFile))
{
fileExists = false; // the same file already marked as deleted no need to do it again
return;
}
// we need to use different name to do a file rename
deleteVersion++;
deletingFileName = RavenFileNameHelper.DeletingFileName(fileName, deleteVersion);
}
} while (!renameSucceeded && deleteVersion < 128);
if (renameSucceeded)
{
accessor.UpdateFileMetadata(deletingFileName, metadata, null);
accessor.DecrementFileCount(deletingFileName);
Log.Debug("File '{0}' was renamed to '{1}' and marked as deleted", fileName, deletingFileName);
var configName = RavenFileNameHelper.DeleteOperationConfigNameForFile(deletingFileName);
var operation = new DeleteFileOperation {
OriginalFileName = fileName, CurrentFileName = deletingFileName
};
accessor.SetConfig(configName, JsonExtensions.ToJObject(operation));
FileSystem.DeleteTriggers.Apply(trigger => trigger.AfterDelete(fileName));
Publisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
Publisher.Publish(new FileChangeNotification {
File = fileName, Action = FileChangeAction.Delete
});
Log.Debug("File '{0}' was deleted", fileName);
}
else
{
Log.Warn("Could not rename a file '{0}' when a delete operation was performed", fileName);
}
});
if (fileExists)
{
Search.Delete(fileName);
Search.Delete(deletingFileName);
}
}