public async Task Should_remove_file_deletion_config_after_storage_cleanup()
{
var client = NewAsyncClient();
var rfs = GetFileSystem();
await client.UploadAsync("toDelete.bin", new MemoryStream(new byte[] { 1, 2, 3, 4, 5 }));
rfs.Files.IndicateFileToDelete("toDelete.bin", null);
await rfs.Files.CleanupDeletedFilesAsync();
IEnumerable <string> configNames = null;
rfs.Storage.Batch(accessor => configNames = accessor.GetConfigNames(0, 10).ToArray());
Assert.DoesNotContain(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName("toDelete.bin")), configNames);
}
public void Create(string fileName, ConflictItem conflict)
{
RavenJObject metadata = null;
storage.Batch(
accessor =>
{
metadata = accessor.GetFile(fileName, 0, 0).Metadata;
accessor.SetConfig(RavenFileNameHelper.ConflictConfigNameForFile(fileName), JsonExtensions.ToJObject(conflict));
metadata[SynchronizationConstants.RavenSynchronizationConflict] = true;
accessor.UpdateFileMetadata(fileName, metadata);
});
if (metadata != null)
{
index.Index(fileName, metadata);
}
}
public void RenameFile(RenameFileOperation operation)
{
var configName = RavenFileNameHelper.RenameOperationConfigNameForFile(operation.Name);
notificationPublisher.Publish(new FileChangeNotification
{
File = FilePathTools.Cannoicalise(operation.Name),
Action = FileChangeAction.Renaming
});
storage.Batch(accessor =>
{
var previousRenameTombstone = accessor.ReadFile(operation.Rename);
if (previousRenameTombstone != null &&
previousRenameTombstone.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if there is a tombstone delete it
accessor.Delete(previousRenameTombstone.FullPath);
}
accessor.RenameFile(operation.Name, operation.Rename, true);
accessor.UpdateFileMetadata(operation.Rename, operation.MetadataAfterOperation);
// copy renaming file metadata and set special markers
var tombstoneMetadata = new RavenJObject(operation.MetadataAfterOperation).WithRenameMarkers(operation.Rename);
accessor.PutFile(operation.Name, 0, tombstoneMetadata, true); // put rename tombstone
accessor.DeleteConfig(configName);
search.Delete(operation.Name);
search.Index(operation.Rename, operation.MetadataAfterOperation);
});
notificationPublisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
notificationPublisher.Publish(new FileChangeNotification
{
File = FilePathTools.Cannoicalise(operation.Rename),
Action = FileChangeAction.Renamed
});
}
public void Should_create_apropriate_config_after_indicating_file_to_delete()
{
var filename = FileHeader.Canonize("toDelete.bin");
var client = NewAsyncClient();
var rfs = GetFileSystem();
client.UploadAsync(filename, new MemoryStream(new byte[] { 1, 2, 3, 4, 5 })).Wait();
rfs.Files.IndicateFileToDelete(filename, null);
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(filename))));
Assert.Equal(RavenFileNameHelper.DeletingFileName(filename), deleteFile.CurrentFileName);
Assert.Equal(filename, deleteFile.OriginalFileName);
}
private SynchronizationReport GetSynchronizationReport(string fileName)
{
SynchronizationReport preResult = null;
Storage.Batch(
accessor =>
{
try
{
var name = RavenFileNameHelper.SyncResultNameForFile(fileName);
preResult = accessor.GetConfig(name).JsonDeserialization <SynchronizationReport>();
}
catch (FileNotFoundException)
{
// just ignore
}
});
return(preResult);
}
private void StrategyAsGetRemote(string fileName)
{
Storage.Batch(
accessor =>
{
var localMetadata = accessor.GetFile(fileName, 0, 0).Metadata;
var conflictConfigName = RavenFileNameHelper.ConflictConfigNameForFile(fileName);
var conflictItem = accessor.GetConfig(conflictConfigName).JsonDeserialization <ConflictItem>();
var conflictResolution = new ConflictResolution
{
Strategy = ConflictResolutionStrategy.RemoteVersion,
RemoteServerId = conflictItem.RemoteHistory.Last().ServerId,
Version = conflictItem.RemoteHistory.Last().Version,
};
localMetadata[SynchronizationConstants.RavenSynchronizationConflictResolution] = JsonExtensions.ToJObject(conflictResolution);
accessor.UpdateFileMetadata(fileName, localMetadata);
});
}
private IEnumerable <SynchronizationDetails> GetSyncingConfigurations(SynchronizationDestination destination)
{
var configObjects = new List <SynchronizationDetails>();
try
{
storage.Batch(
accessor =>
{
configObjects = accessor.GetConfigsStartWithPrefix(RavenFileNameHelper.SyncNameForFile(string.Empty, destination.Url), 0, 100)
.Select(config => config.JsonDeserialization <SynchronizationDetails>())
.ToList();
});
}
catch (Exception e)
{
Log.WarnException(string.Format("Could not get syncing configurations for a destination {0}", destination), e);
}
return(configObjects);
}
private void DeleteFiles(IEnumerable <string> keys, int totalResults, Action <string> progress)
{
Storage.Batch(accessor =>
{
var files = keys.Select(accessor.ReadFile);
foreach (var fileWithIndex in files.Select((value, i) => new { i, value }))
{
var file = fileWithIndex.value;
var fileName = file.FullPath;
try
{
Synchronizations.AssertFileIsNotBeingSynced(fileName);
}
catch (Exception)
{
//ignore files that are being synced
continue;
}
var metadata = file.Metadata;
if (metadata == null || metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
continue;
}
Historian.Update(fileName, metadata);
Files.IndicateFileToDelete(fileName, null);
// don't create a tombstone for .downloading file
if (!fileName.EndsWith(RavenFileNameHelper.DownloadingFileSuffix))
{
Files.PutTombstone(fileName, metadata);
accessor.DeleteConfig(RavenFileNameHelper.ConflictConfigNameForFile(fileName)); // delete conflict item too
}
progress(string.Format("File {0}/{1} was deleted, name: '{2}'.", fileWithIndex.i, totalResults, fileName));
}
});
}
public void Should_resume_to_rename_file_if_appropriate_config_exists()
{
var client = NewAsyncClient();
var rfs = GetRavenFileSystem();
const string fileName = "file.bin";
const string rename = "renamed.bin";
client.UploadAsync(fileName, new RandomStream(1)).Wait();
// create config to say to the server that rename operation performed last time were not finished
var renameOpConfig = RavenFileNameHelper.RenameOperationConfigNameForFile(fileName);
var renameOperation = new RenameFileOperation
{
Name = fileName,
Rename = rename,
MetadataAfterOperation = new RavenJObject().WithETag(Guid.Empty)
};
rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(renameOpConfig, renameOperation));
rfs.StorageOperationsTask.ResumeFileRenamingAsync().Wait();
IEnumerable <string> configNames = null;
rfs.Storage.Batch(accessor => configNames = accessor.GetConfigNames(0, 10).ToArray());
Assert.DoesNotContain(renameOpConfig, configNames);
var renamedMetadata = client.GetMetadataForAsync(rename).Result;
Assert.NotNull(renamedMetadata);
var results = client.SearchOnDirectoryAsync("/").Result; // make sure that indexes are updated
Assert.Equal(1, results.FileCount);
Assert.Equal(rename, results.Files[0].Name);
}
public void Should_synchronize_just_metadata()
{
var content = new MemoryStream(new byte[] {1, 2, 3, 4});
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
sourceClient.UploadAsync("test.bin", content, new RavenJObject { { "difference", "metadata" } }).Wait();
content.Position = 0;
destinationClient.UploadAsync("test.bin", content).Wait();
var report = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
var conflictItem = destinationClient.Configuration.GetKeyAsync<ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin")).Result;
Assert.Null(conflictItem);
Assert.Equal(SynchronizationType.MetadataUpdate, report.Type);
var destinationMetadata = destinationClient.GetMetadataForAsync("test.bin").Result;
Assert.Equal("metadata", destinationMetadata.Value<string>("difference"));
}
private void CreateSyncingConfiguration(string fileName, Guid etag, string destinationFileSystemUrl, SynchronizationType synchronizationType)
{
try
{
var name = RavenFileNameHelper.SyncNameForFile(fileName, destinationFileSystemUrl);
var details = new SynchronizationDetails
{
DestinationUrl = destinationFileSystemUrl,
FileName = fileName,
FileETag = etag,
Type = synchronizationType
};
storage.Batch(accessor => accessor.SetConfig(name, JsonExtensions.ToJObject(details)));
}
catch (Exception e)
{
Log.WarnException(
string.Format("Could not create syncing configurations for a file {0} and destination {1}", fileName, destinationFileSystemUrl),
e);
}
}
public void Should_not_perform_file_delete_if_it_is_being_synced()
{
var client = NewAsyncClient();
var rfs = GetRavenFileSystem();
client.UploadAsync("file.bin", new MemoryStream(new byte[] { 1, 2, 3, 4, 5 })).Wait();
rfs.StorageOperationsTask.IndicateFileToDelete("file.bin");
rfs.Storage.Batch(accessor =>
accessor.SetConfigurationValue(RavenFileNameHelper.SyncLockNameForFile("file.bin"),
LockFileTests.SynchronizationConfig(DateTime.UtcNow)));
rfs.StorageOperationsTask.CleanupDeletedFilesAsync().Wait();
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName("file.bin"))));
Assert.Equal(RavenFileNameHelper.DeletingFileName("file.bin"), deleteFile.CurrentFileName);
Assert.Equal("file.bin", deleteFile.OriginalFileName);
}
public HttpResponseMessage Post(string name)
{
name = RavenFileNameHelper.RavenPath(name);
var headers = this.GetFilteredMetadataFromHeaders(InnerHeaders);
Historian.UpdateLastModified(headers);
Historian.Update(name, headers);
try
{
ConcurrencyAwareExecutor.Execute(() =>
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(name, accessor, true);
accessor.UpdateFileMetadata(name, headers);
}), ConcurrencyResponseException);
}
catch (FileNotFoundException)
{
log.Debug("Cannot update metadata because file '{0}' was not found", name);
return(GetEmptyMessage(HttpStatusCode.NotFound));
}
Search.Index(name, headers);
Publisher.Publish(new FileChangeNotification {
File = FilePathTools.Cannoicalise(name), Action = FileChangeAction.Update
});
StartSynchronizeDestinationsInBackground();
log.Debug("Metadata of a file '{0}' was updated", name);
//Hack needed by jquery on the client side. We need to find a better solution for this
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public async Task Should_report_that_file_is_broken_if_last_synchronization_set_exception()
{
var destinationClient = NewAsyncClient(0);
var sampleGuid = Guid.NewGuid();
var failureSynchronization = new SynchronizationReport("test.bin", sampleGuid, SynchronizationType.Unknown)
{
Exception = new Exception("There was an exception in last synchronization.")
};
await destinationClient.Configuration.SetKeyAsync(RavenFileNameHelper.SyncResultNameForFile("test.bin"), failureSynchronization);
var confirmations = await destinationClient.Synchronization
.GetConfirmationForFilesAsync(new List <Tuple <string, Guid> >
{
new Tuple <string, Guid>(
"test.bin", sampleGuid)
});
Assert.Equal(1, confirmations.Count());
Assert.Equal(FileStatus.Broken, confirmations[0].Status);
Assert.Equal("test.bin", confirmations[0].FileName);
}
public void ExecuteCopyOperation(CopyFileOperation operation)
{
var configName = RavenFileNameHelper.CopyOperationConfigNameForFile(operation.SourceFilename, operation.TargetFilename);
Storage.Batch(accessor =>
{
AssertPutOperationNotVetoed(operation.TargetFilename, operation.MetadataAfterOperation);
var targetTombstrone = accessor.ReadFile(operation.TargetFilename);
if (targetTombstrone != null &&
targetTombstrone.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if there is a tombstone delete it
accessor.Delete(targetTombstrone.FullPath);
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(operation.TargetFilename, operation.MetadataAfterOperation));
accessor.CopyFile(operation.SourceFilename, operation.TargetFilename, true);
var putResult = accessor.UpdateFileMetadata(operation.TargetFilename, operation.MetadataAfterOperation, null);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(operation.TargetFilename, null, operation.MetadataAfterOperation));
accessor.DeleteConfig(configName);
Search.Index(operation.TargetFilename, operation.MetadataAfterOperation, putResult.Etag);
});
Publisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
Publisher.Publish(new FileChangeNotification {
File = operation.TargetFilename, Action = FileChangeAction.Add
});
}
public static FileVersioningConfiguration GetVersioningConfiguration(this IStorageActionsAccessor accessor, string filePath)
{
FileVersioningConfiguration fileVersioningConfiguration;
var directoryName = RavenFileNameHelper.RavenDirectory(Path.GetDirectoryName(filePath));
while (string.IsNullOrEmpty(directoryName) == false && directoryName != "/")
{
var configurationName = "Raven/Versioning/" + directoryName.TrimStart('/');
if (TryGetDeserializedConfig(accessor, configurationName, out fileVersioningConfiguration))
{
return(fileVersioningConfiguration);
}
directoryName = RavenFileNameHelper.RavenDirectory(Path.GetDirectoryName(directoryName));
}
if (TryGetDeserializedConfig(accessor, DefaultConfigurationName, out fileVersioningConfiguration))
{
return(fileVersioningConfiguration);
}
return(null);
}
public HttpResponseMessage Delete(string name)
{
name = FileHeader.Canonize(name);
Storage.Batch(accessor =>
{
Synchronizations.AssertFileIsNotBeingSynced(name);
var fileAndPages = accessor.GetFile(name, 0, 0);
var metadata = fileAndPages.Metadata;
if (metadata == null)
{
throw new FileNotFoundException();
}
if (metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
throw new FileNotFoundException();
}
Historian.Update(name, metadata);
Files.IndicateFileToDelete(name, GetEtag());
if (name.EndsWith(RavenFileNameHelper.DownloadingFileSuffix) == false) // don't create a tombstone for .downloading file
{
Files.PutTombstone(name, metadata);
accessor.DeleteConfig(RavenFileNameHelper.ConflictConfigNameForFile(name)); // delete conflict item too
}
});
SynchronizationTask.Context.NotifyAboutWork();
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public Task CleanupDeletedFilesAsync()
{
var filesToDelete = new List <DeleteFileOperation>();
storage.Batch(accessor => filesToDelete = accessor.GetConfigsStartWithPrefix(RavenFileNameHelper.DeleteOperationConfigPrefix, 0, 10)
.Select(config => config.JsonDeserialization <DeleteFileOperation>())
.ToList());
if (filesToDelete.Count == 0)
{
return(Task.FromResult <object>(null));
}
var tasks = new List <Task>();
foreach (var fileToDelete in filesToDelete)
{
var deletingFileName = fileToDelete.CurrentFileName;
if (IsDeleteInProgress(deletingFileName))
{
continue;
}
if (IsUploadInProgress(fileToDelete.OriginalFileName))
{
continue;
}
if (IsSynchronizationInProgress(fileToDelete.OriginalFileName))
{
continue;
}
if (fileToDelete.OriginalFileName.EndsWith(RavenFileNameHelper.DownloadingFileSuffix)) // if it's .downloading file
{
if (IsSynchronizationInProgress(SynchronizedFileName(fileToDelete.OriginalFileName))) // and file is being synced
{
continue;
}
}
Log.Debug("Starting to delete file '{0}' from storage", deletingFileName);
var deleteTask = Task.Run(() =>
{
try
{
ConcurrencyAwareExecutor.Execute(() => storage.Batch(accessor => accessor.Delete(deletingFileName)), retries: 1);
}
catch (Exception e)
{
Log.WarnException(string.Format("Could not delete file '{0}' from storage", deletingFileName), e);
return;
}
var configName = RavenFileNameHelper.DeleteOperationConfigNameForFile(deletingFileName);
storage.Batch(accessor => accessor.DeleteConfig(configName));
notificationPublisher.Publish(new ConfigurationChangeNotification
{
Name = configName,
Action = ConfigurationChangeAction.Delete
});
Log.Debug("File '{0}' was deleted from storage", deletingFileName);
});
deleteFileTasks.AddOrUpdate(deletingFileName, deleteTask, (file, oldTask) => deleteTask);
tasks.Add(deleteTask);
}
return(Task.WhenAll(tasks));
}
public async Task Source_should_save_configuration_record_after_synchronization()
{
var sourceClient = NewAsyncClient(0);
var sourceContent = new RandomStream(10000);
var destinationClient = NewAsyncClient(1);
await sourceClient.UploadAsync("test.bin", sourceContent);
await sourceClient.Synchronization.SetDestinationsAsync(destinationClient.ToSynchronizationDestination());
await sourceClient.Synchronization.SynchronizeAsync();
var fullDstUrl = destinationClient.ToSynchronizationDestination().FileSystemUrl;
var synchronizationDetails = sourceClient.Configuration.GetKeyAsync <SynchronizationDetails>(RavenFileNameHelper.SyncNameForFile("test.bin", fullDstUrl)).Result;
Assert.Equal("test.bin", synchronizationDetails.FileName);
Assert.Equal(fullDstUrl, synchronizationDetails.DestinationUrl);
Assert.NotEqual(Guid.Empty, synchronizationDetails.FileETag);
Assert.Equal(SynchronizationType.ContentUpdate, synchronizationDetails.Type);
}
public void UnlockByDeletingSyncConfiguration(string fileName, IStorageActionsAccessor accessor)
{
accessor.DeleteConfig(RavenFileNameHelper.SyncLockNameForFile(fileName));
log.Debug("File '{0}' was unlocked", fileName);
}
public async Task Should_just_rename_file_in_synchronization_process()
{
var content = new MemoryStream(new byte[] { 1, 2, 3, 4 });
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
await sourceClient.UploadAsync("test.bin", content, new RavenJObject { { "key", "value" } });
content.Position = 0;
await destinationClient.UploadAsync("test.bin", content, new RavenJObject { { "key", "value" } });
await sourceClient.RenameAsync("test.bin", "renamed.bin");
// we need to indicate old file name, otherwise content update would be performed because renamed file does not exist on dest
var report = SyncTestUtils.ResolveConflictAndSynchronize(sourceClient, destinationClient, "test.bin");
Assert.Equal(SynchronizationType.Rename, report.Type);
var conflictItem = destinationClient.Configuration.GetKeyAsync <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin")).Result;
Assert.Null(conflictItem);
var testMetadata = await destinationClient.GetMetadataForAsync("test.bin");
var renamedMetadata = await destinationClient.GetMetadataForAsync("renamed.bin");
Assert.Null(testMetadata);
Assert.NotNull(renamedMetadata);
var result = await destinationClient.SearchOnDirectoryAsync("/");
Assert.Equal(1, result.FileCount);
Assert.Equal("renamed.bin", result.Files[0].Name);
}
public async Task Source_should_delete_configuration_record_if_destination_confirm_that_file_is_safe()
{
var sourceClient = NewAsyncClient(0);
var sourceContent = new RandomStream(10000);
var destinationClient = (IAsyncFilesCommandsImpl)NewAsyncClient(1);
await sourceClient.UploadAsync("test.bin", sourceContent);
await sourceClient.Synchronization.SetDestinationsAsync(destinationClient.ToSynchronizationDestination());
await sourceClient.Synchronization.SynchronizeAsync();
// start synchronization again to force confirmation by source
await sourceClient.Synchronization.SynchronizeAsync();
var shouldBeNull = await sourceClient.Configuration.GetKeyAsync <SynchronizationDetails>(RavenFileNameHelper.SyncNameForFile("test.bin", destinationClient.ServerUrl));
Assert.Null(shouldBeNull);
}
public void Should_not_delete_downloading_file_if_synchronization_retry_is_being_performed()
{
const string fileName = "file.bin";
var downloadingFileName = RavenFileNameHelper.DownloadingFileName(fileName);
var client = NewAsyncClient();
var rfs = GetFileSystem();
client.UploadAsync(fileName, new RandomStream(1)).Wait();
client.UploadAsync(downloadingFileName, new RandomStream(1)).Wait();
rfs.Files.IndicateFileToDelete(downloadingFileName, null);
rfs.Storage.Batch(accessor =>
accessor.SetConfigurationValue(RavenFileNameHelper.SyncLockNameForFile(fileName), LockFileTests.SynchronizationConfig(DateTime.UtcNow)));
rfs.Files.CleanupDeletedFilesAsync().Wait();
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(downloadingFileName))));
Assert.Equal(RavenFileNameHelper.DeletingFileName(downloadingFileName), deleteFile.CurrentFileName);
Assert.Equal(downloadingFileName, deleteFile.OriginalFileName);
}
public HttpResponseMessage ResolveConflict(string filename, ConflictResolutionStrategy strategy)
{
var canonicalFilename = FileHeader.Canonize(filename);
Log.Debug("Resolving conflict of a file '{0}' by using {1} strategy", filename, strategy);
switch (strategy)
{
case ConflictResolutionStrategy.CurrentVersion:
Storage.Batch(accessor =>
{
var localMetadata = accessor.GetFile(canonicalFilename, 0, 0).Metadata;
var conflict = accessor.GetConfigurationValue <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile(canonicalFilename));
ConflictResolver.ApplyCurrentStrategy(canonicalFilename, conflict, localMetadata);
accessor.UpdateFileMetadata(canonicalFilename, localMetadata);
ConflictArtifactManager.Delete(canonicalFilename, accessor);
});
Publisher.Publish(new ConflictNotification
{
FileName = filename,
Status = ConflictStatus.Resolved
});
break;
case ConflictResolutionStrategy.RemoteVersion:
Storage.Batch(accessor =>
{
var localMetadata = accessor.GetFile(canonicalFilename, 0, 0).Metadata;
var conflict = accessor.GetConfig(RavenFileNameHelper.ConflictConfigNameForFile(canonicalFilename)).JsonDeserialization <ConflictItem>();
ConflictResolver.ApplyRemoteStrategy(canonicalFilename, conflict, localMetadata);
accessor.UpdateFileMetadata(canonicalFilename, localMetadata);
// ConflictArtifactManager.Delete(canonicalFilename, accessor); - intentionally not deleting, conflict item will be removed when a remote file is put
});
Task.Run(() => SynchronizationTask.SynchronizeDestinationsAsync(true));
break;
default:
throw new NotSupportedException(string.Format("{0} is not the valid strategy to resolve a conflict", strategy));
}
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
private void SaveSynchronizationReport(string fileName, IStorageActionsAccessor accessor, SynchronizationReport report)
{
var name = RavenFileNameHelper.SyncResultNameForFile(fileName);
accessor.SetConfig(name, JsonExtensions.ToJObject(report));
}
public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName)
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault();
var canonicalFilename = FileHeader.Canonize(fileName);
var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate);
Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo);
StorageStream localFile = null;
var isNewFile = false;
var isConflictResolved = false;
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
var localMetadata = GetLocalMetadata(canonicalFilename);
if (localMetadata != null)
{
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
localFile = StorageStream.Reading(Storage, canonicalFilename);
}
else
{
isNewFile = true;
}
Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata);
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName);
await Request.Content.ReadAsMultipartAsync(provider);
Log.Debug("Multipart content of a file '{0}' was processed/read", fileName);
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Flush();
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata));
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
accessor.RenameFile(tempFileName, canonicalFilename);
Search.Delete(tempFileName);
Search.Index(canonicalFilename, sourceMetadata);
});
if (isNewFile)
{
Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName);
}
else
{
Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName);
}
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
}
}
finally
{
if (localFile != null)
{
localFile.Dispose();
}
}
if (report.Exception == null)
{
Log.Debug(
"File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
else
{
Log.WarnException(
string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo),
report.Exception);
}
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update);
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
return(GetMessageWithObject(report));
}
public async void Should_synchronize_to_all_destinations()
{
var sourceContent = SyncTestUtils.PrepareSourceStream(10000);
sourceContent.Position = 0;
var sourceClient = NewAsyncClient(0);
var destination1Client = NewAsyncClient(1);
var destination2Client = NewAsyncClient(2);
var destination1Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
var destination2Content = new RandomlyModifiedStream(sourceContent, 0.01);
sourceContent.Position = 0;
await destination1Client.UploadAsync("test.bin", destination1Content);
await destination2Client.UploadAsync("test.bin", destination2Content);
sourceContent.Position = 0;
await sourceClient.UploadAsync("test.bin", sourceContent);
sourceContent.Position = 0;
sourceClient.Synchronization.SetDestinationsAsync(destination1Client.ToSynchronizationDestination(), destination2Client.ToSynchronizationDestination()).Wait();
var destinationSyncResults = sourceClient.Synchronization.SynchronizeAsync().Result;
// we expect conflicts after first attempt of synchronization
Assert.Equal(2, destinationSyncResults.Length);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[0].Reports.ToArray()[0].Exception.Message);
Assert.Equal("File test.bin is conflicted", destinationSyncResults[1].Reports.ToArray()[0].Exception.Message);
await destination1Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion);
await destination2Client.Synchronization.ResolveConflictAsync("test.bin", ConflictResolutionStrategy.RemoteVersion);
destinationSyncResults = await sourceClient.Synchronization.SynchronizeAsync();
var conflictItem = await destination1Client.Configuration.GetKeyAsync <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin"));
Assert.Null(conflictItem);
conflictItem = await destination2Client.Configuration.GetKeyAsync <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile("test.bin"));
Assert.Null(conflictItem);
// check if reports match
Assert.Equal(2, destinationSyncResults.Length);
var result1 = destinationSyncResults[0].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result1.BytesCopied + result1.BytesTransfered);
Assert.Equal(SynchronizationType.ContentUpdate, result1.Type);
var result2 = destinationSyncResults[1].Reports.ToArray()[0];
Assert.Equal(sourceContent.Length, result2.BytesCopied + result2.BytesTransfered);
Assert.Equal(SynchronizationType.ContentUpdate, result2.Type);
// check content of files
string destination1Md5;
using (var resultFileContent = await destination1Client.DownloadAsync("test.bin"))
{
destination1Md5 = resultFileContent.GetMD5Hash();
}
string destination2Md5;
using (var resultFileContent = await destination2Client.DownloadAsync("test.bin"))
{
destination2Md5 = resultFileContent.GetMD5Hash();
}
sourceContent.Position = 0;
var sourceMd5 = sourceContent.GetMD5Hash();
Assert.Equal(sourceMd5, destination1Md5);
Assert.Equal(sourceMd5, destination2Md5);
Assert.Equal(destination1Md5, destination2Md5);
}
public HttpResponseMessage Copy(string name, string targetFilename)
{
name = FileHeader.Canonize(name);
targetFilename = FileHeader.Canonize(targetFilename);
var etag = GetEtag();
var retriesCount = 0;
while (true)
{
try
{
Storage.Batch(accessor =>
{
FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);
var existingFile = accessor.ReadFile(name);
if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker))
{
throw new FileNotFoundException();
}
var renamingFile = accessor.ReadFile(targetFilename);
if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false)
{
throw new FileExistsException("Cannot copy because file " + targetFilename + " already exists");
}
var metadata = existingFile.Metadata;
if (etag != null && existingFile.Etag != etag)
{
throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag")
{
ActualETag = existingFile.Etag,
ExpectedETag = etag
}
}
;
Historian.UpdateLastModified(metadata);
var operation = new CopyFileOperation
{
FileSystem = FileSystem.Name,
SourceFilename = name,
TargetFilename = targetFilename,
MetadataAfterOperation = metadata
};
accessor.SetConfig(RavenFileNameHelper.CopyOperationConfigNameForFile(name, targetFilename), JsonExtensions.ToJObject(operation));
var configName = RavenFileNameHelper.CopyOperationConfigNameForFile(operation.SourceFilename, operation.TargetFilename);
Files.AssertPutOperationNotVetoed(operation.TargetFilename, operation.MetadataAfterOperation);
var targetTombstrone = accessor.ReadFile(operation.TargetFilename);
if (targetTombstrone != null &&
targetTombstrone.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if there is a tombstone delete it
accessor.Delete(targetTombstrone.FullPath);
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(operation.TargetFilename, operation.MetadataAfterOperation));
accessor.CopyFile(operation.SourceFilename, operation.TargetFilename, true);
var putResult = accessor.UpdateFileMetadata(operation.TargetFilename, operation.MetadataAfterOperation, null);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(operation.TargetFilename, null, operation.MetadataAfterOperation));
accessor.DeleteConfig(configName);
Search.Index(operation.TargetFilename, operation.MetadataAfterOperation, putResult.Etag);
Publisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
Publisher.Publish(new FileChangeNotification {
File = operation.TargetFilename, Action = FileChangeAction.Add
});
});
break;
}
catch (ConcurrencyException)
{
// due to IncrementUsageCount performed concurrently on Voron storage
// Esent deals with that much better using Api.EscrowUpdate
if (retriesCount++ > 100)
{
throw;
}
Thread.Sleep(new Random().Next(1, 13));
}
}
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was copied to '{1}'", name, targetFilename);
}
SynchronizationTask.Context.NotifyAboutWork();
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public async Task Should_delete_sync_configuration_after_synchronization()
{
IAsyncFilesCommands destinationClient;
IAsyncFilesCommands sourceClient;
UploadFilesSynchronously(out sourceClient, out destinationClient);
await sourceClient.Synchronization.StartAsync("test.bin", destinationClient);
var config = await destinationClient.Configuration.GetKeyAsync <SynchronizationLock>(RavenFileNameHelper.SyncLockNameForFile("test.bin"));
Assert.Null(config);
}
public HttpResponseMessage Patch(string name, string rename)
{
name = FileHeader.Canonize(name);
rename = FileHeader.Canonize(rename);
var etag = GetEtag();
if (rename.Length > SystemParameters.KeyMost)
{
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename);
}
return(GetMessageWithString(string.Format("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename), HttpStatusCode.BadRequest));
}
Storage.Batch(accessor =>
{
FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);
var existingFile = accessor.ReadFile(name);
if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker))
{
throw new FileNotFoundException();
}
var renamingFile = accessor.ReadFile(rename);
if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false)
{
throw new FileExistsException("Cannot rename because file " + rename + " already exists");
}
var metadata = existingFile.Metadata;
if (etag != null && existingFile.Etag != etag)
{
throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag")
{
ActualETag = existingFile.Etag,
ExpectedETag = etag
}
}
;
Historian.UpdateLastModified(metadata);
var operation = new RenameFileOperation(name, rename, existingFile.Etag, metadata);
accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation));
accessor.PulseTransaction(); // commit rename operation config
Files.ExecuteRenameOperation(operation);
});
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was renamed to '{1}'", name, rename);
}
SynchronizationTask.Context.NotifyAboutWork();
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
