public void FinishSynchronization(string fileName, SynchronizationReport report, FileSystemInfo sourceFileSystem, Etag sourceFileETag)
{
try
{
// we want to execute those operation in a single batch but we also have to ensure that
// Raven/Synchronization/Sources/sourceServerId config is modified only by one finishing synchronization at the same time
synchronizationFinishLocks.GetOrAdd(sourceFileSystem.Id, new ReaderWriterLockSlim()).EnterWriteLock();
Storage.Batch(accessor =>
{
SaveSynchronizationReport(fileName, accessor, report);
FileLockManager.UnlockByDeletingSyncConfiguration(fileName, accessor);
if (report.Exception == null)
SaveSynchronizationSourceInformation(sourceFileSystem, sourceFileETag);
});
}
catch (Exception ex)
{
Log.ErrorException(string.Format("Failed to finish synchronization of a file '{0}' from {1}", fileName, sourceFileSystem), ex);
}
finally
{
synchronizationFinishLocks.GetOrAdd(sourceFileSystem.Id, new ReaderWriterLockSlim()).ExitWriteLock();
}
}
public async Task Should_report_that_file_is_broken_if_last_synchronization_set_exception()
{
var destinationClient = NewAsyncClient(0);
var sampleGuid = Guid.NewGuid();
var failureSynchronization = new SynchronizationReport("test.bin", sampleGuid, SynchronizationType.Unknown)
{
Exception = new Exception("There was an exception in last synchronization.")
};
await destinationClient.Configuration.SetKeyAsync(RavenFileNameHelper.SyncResultNameForFile("test.bin"), failureSynchronization);
var confirmations = await destinationClient.Synchronization
.GetConfirmationForFilesAsync(new List<Tuple<string, Etag>>
{
new Tuple<string, Etag>(
"test.bin", sampleGuid)
});
Assert.Equal(1, confirmations.Count());
Assert.Equal(FileStatus.Broken, confirmations[0].Status);
Assert.Equal(FileHeader.Canonize("test.bin"), confirmations[0].FileName);
}
private async Task<SynchronizationReport> PerformSynchronizationAsync(ISynchronizationServerClient synchronizationServerClient,
SynchronizationWorkItem work)
{
var destinationUrl = synchronizationServerClient.BaseUrl;
Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
work.GetType().Name, work.FileName, destinationUrl);
if (AvailableSynchronizationRequestsTo(destinationUrl) <= 0)
{
Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destinationUrl, work.FileName);
if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
{
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Enqueue,
Direction = SynchronizationDirection.Outgoing
});
}
return new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = new SynchronizationException(string.Format(
"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destinationUrl, work.FileName))
};
}
string fileName = work.FileName;
synchronizationQueue.SynchronizationStarted(work, destinationUrl);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Start,
Direction = SynchronizationDirection.Outgoing
});
SynchronizationReport report;
try
{
report = await work.PerformAsync(synchronizationServerClient).ConfigureAwait(false);
}
catch (Exception ex)
{
report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = ex,
};
}
var synchronizationCancelled = false;
if (report.Exception == null)
{
var moreDetails = string.Empty;
if (work.SynchronizationType == SynchronizationType.ContentUpdate)
{
moreDetails = string.Format(". {0} bytes were transferred and {1} bytes copied. Need list length was {2}",
report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
context.UpdateSuccessfulSynchronizationTime();
Log.Debug("{0} to {1} has finished successfully{2}", work, destinationUrl, moreDetails);
}
else
{
if (work.IsCancelled || report.Exception is TaskCanceledException)
{
synchronizationCancelled = true;
Log.DebugException(string.Format("{0} to {1} was canceled", work, destinationUrl), report.Exception);
}
else
{
Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl), report.Exception);
}
}
Queue.SynchronizationFinished(work, destinationUrl);
if (synchronizationCancelled == false)
CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destinationUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Finish,
Direction = SynchronizationDirection.Outgoing
});
return report;
}
public async Task<HttpResponseMessage> MultipartProceed(string fileSystemName)
{
if (!Request.Content.IsMimeMultipartContent())
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault();
var canonicalFilename = FileHeader.Canonize(fileName);
var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename);
var sourceServerInfo = InnerHeaders.Value<ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(InnerHeaders.GetValues(Constants.MetadataEtagField).First().Trim('\"'));
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate);
Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo);
StorageStream localFile = null;
var isNewFile = false;
var isConflictResolved = false;
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(InnerHeaders);
var localMetadata = GetLocalMetadata(canonicalFilename);
if (localMetadata != null)
{
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
localFile = StorageStream.Reading(Storage, canonicalFilename);
}
else
{
isNewFile = true;
}
Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata);
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName);
await Request.Content.ReadAsMultipartAsync(provider);
Log.Debug("Multipart content of a file '{0}' was processed/read", fileName);
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Flush();
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata));
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
accessor.RenameFile(tempFileName, canonicalFilename);
Search.Delete(tempFileName);
Search.Index(canonicalFilename, sourceMetadata);
});
if (isNewFile)
{
Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName);
}
else
{
Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName);
}
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
report.Exception = ex;
}
finally
{
if (localFile != null)
{
localFile.Dispose();
}
}
if (report.Exception == null)
{
Log.Debug(
"File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
else
{
Log.WarnException(
string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo),
report.Exception);
}
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update);
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
return GetMessageWithObject(report);
}
private void SaveSynchronizationReport(string fileName, IStorageActionsAccessor accessor, SynchronizationReport report)
{
var name = RavenFileNameHelper.SyncResultNameForFile(fileName);
accessor.SetConfig(name, JsonExtensions.ToJObject(report));
}
public HttpResponseMessage Rename(string fileSystemName, string fileName, string rename)
{
bool isConflictResolved = false;
var canonicalFilename = FileHeader.Canonize(fileName);
var canonicalRename = FileHeader.Canonize(rename);
var sourceServerInfo = InnerHeaders.Value<ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(InnerHeaders.GetValues(Constants.MetadataEtagField).First().Trim('\"'));
var sourceMetadata = GetFilteredMetadataFromHeaders(InnerHeaders);
Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName,
rename, sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Rename);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Rename);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
StorageOperationsTask.RenameFile(new RenameFileOperation
{
FileSystem = FileSystem.Name,
Name = canonicalFilename,
Rename = canonicalRename,
MetadataAfterOperation = sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers()
});
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(string.Format("Error was occurred during renaming synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
if (report.Exception == null)
Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo);
return GetMessageWithObject(report);
}
public HttpResponseMessage Delete(string fileSystemName, string fileName)
{
var canonicalFilename = FileHeader.Canonize(fileName);
var sourceServerInfo = InnerHeaders.Value<ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(InnerHeaders.GetValues(Constants.MetadataEtagField).First().Trim('\"'));
Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Delete);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Delete);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
if (localMetadata != null)
{
// REVIEW: Use InnerHeaders for consistency?
var sourceMetadata = GetFilteredMetadataFromHeaders(Request.Headers); // Request.Headers.FilterHeadersToObject();
bool isConflictResolved;
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
var tombstoneMetadata = new RavenJObject
{
{
SynchronizationConstants.RavenSynchronizationHistory,
localMetadata[SynchronizationConstants.RavenSynchronizationHistory]
},
{
SynchronizationConstants.RavenSynchronizationVersion,
localMetadata[SynchronizationConstants.RavenSynchronizationVersion]
},
{
SynchronizationConstants.RavenSynchronizationSource,
localMetadata[SynchronizationConstants.RavenSynchronizationSource]
}
}.WithDeleteMarker();
Historian.UpdateLastModified(tombstoneMetadata);
accessor.PutFile(canonicalFilename, 0, tombstoneMetadata, true);
});
PublishFileNotification(fileName, FileChangeAction.Delete);
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(string.Format("Error was occurred during deletion synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (report.Exception == null)
{
Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceServerInfo);
}
return this.GetMessageWithObject(report, HttpStatusCode.OK);
}
public HttpResponseMessage UpdateMetadata(string fileSystemName, string fileName)
{
bool isConflictResolved = false;
var canonicalFilename = FileHeader.Canonize(fileName);
var sourceServerInfo = InnerHeaders.Value<ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
// REVIEW: (Oren) It works, but it seems to me it is not an scalable solution.
var sourceFileETag = Guid.Parse(InnerHeaders.GetValues(Constants.MetadataEtagField).First().Trim('\"'));
Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName,
sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.MetadataUpdate);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.MetadataUpdate);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
var sourceMetadata = GetFilteredMetadataFromHeaders(InnerHeaders);
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Historian.UpdateLastModified(sourceMetadata);
Storage.Batch(accessor => accessor.UpdateFileMetadata(canonicalFilename, sourceMetadata));
Search.Index(canonicalFilename, sourceMetadata);
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
PublishFileNotification(fileName, FileChangeAction.Update);
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(
string.Format("Error was occurred during metadata synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved )
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
if (report.Exception == null)
{
Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceServerInfo);
}
return this.GetMessageWithObject(report, HttpStatusCode.OK);
}
public async Task<SynchronizationReport> Execute()
{
var report = new SynchronizationReport(fileName, sourceFileEtag, type);
try
{
AssertOperationAndLockFile();
NotifyStart();
Prepare();
var localMetadata = fs.Synchronizations.GetLocalMetadata(fileName);
bool conflictResolved;
AssertConflictDetection(localMetadata, out conflictResolved);
fs.SynchronizationTriggers.Apply(trigger => trigger.BeforeSynchronization(fileName, sourceMetadata, type));
dynamic afterSynchronizationTriggerData = null;
switch (type)
{
case SynchronizationType.Delete:
ExecuteDelete(localMetadata);
break;
case SynchronizationType.Rename:
ExecuteRename(Rename);
break;
case SynchronizationType.MetadataUpdate:
ExecuteMetadataUpdate();
break;
case SynchronizationType.ContentUpdate:
await ExecuteContentUpdate(localMetadata, report);
afterSynchronizationTriggerData = new
{
TempFileName = RavenFileNameHelper.DownloadingFileName(fileName)
};
break;
default:
throw new ArgumentOutOfRangeException("type", type.ToString());
}
fs.SynchronizationTriggers.Apply(trigger => trigger.AfterSynchronization(fileName, sourceMetadata, type, afterSynchronizationTriggerData));
if (conflictResolved)
{
fs.ConflictArtifactManager.Delete(fileName);
fs.Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(string.Format("Error was occurred during deletion synchronization of file '{0}' from {1}", fileName, sourceFs), ex);
}
}
fs.Synchronizations.FinishSynchronization(fileName, report, sourceFs, sourceFileEtag);
NotifyEnd();
return report;
}
private async Task ExecuteContentUpdate(RavenJObject localMetadata, SynchronizationReport report)
{
var tempFileName = RavenFileNameHelper.DownloadingFileName(fileName);
using (var localFile = localMetadata != null ? StorageStream.Reading(fs.Storage, fileName) : null)
{
fs.PutTriggers.Apply(trigger => trigger.OnPut(tempFileName, sourceMetadata));
fs.Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(fs, tempFileName, sourceMetadata);
fs.PutTriggers.Apply(trigger => trigger.AfterPut(tempFileName, null, sourceMetadata));
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName);
await MultipartContent.ReadAsMultipartAsync(provider);
Log.Debug("Multipart content of a file '{0}' was processed/read", fileName);
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Flush();
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
FileUpdateResult updateResult = null;
fs.Storage.Batch(accessor => updateResult = accessor.UpdateFileMetadata(tempFileName, sourceMetadata, null));
fs.Storage.Batch(accessor =>
{
using (fs.DisableAllTriggersForCurrentThread())
{
fs.Files.IndicateFileToDelete(fileName, null);
}
accessor.RenameFile(tempFileName, fileName);
fs.Search.Delete(tempFileName);
fs.Search.Index(fileName, sourceMetadata, updateResult.Etag);
});
if (localFile == null)
Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName);
else
Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName);
fs.Publisher.Publish(new FileChangeNotification { File = fileName, Action = localFile == null ? FileChangeAction.Add : FileChangeAction.Update });
}
}
