private async Task<SynchronizationReport> PerformSynchronizationAsync(SynchronizationClient destination,
SynchronizationWorkItem work)
{
Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
work.GetType().Name, work.FileName, destination.FileSystemUrl);
if (!CanSynchronizeTo(destination.FileSystemUrl))
{
Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destination.FileSystemUrl, work.FileName);
synchronizationQueue.EnqueueSynchronization(destination.FileSystemUrl, work);
return new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = new SynchronizationException(string.Format(
"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
destination.FileSystemUrl, work.FileName))
};
}
string fileName = work.FileName;
synchronizationQueue.SynchronizationStarted(work, destination.FileSystemUrl);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destination.FileSystemUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Start,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
SynchronizationReport report;
try
{
report = await work.PerformAsync(destination);
}
catch (Exception ex)
{
report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
{
Exception = ex,
};
}
var synchronizationCancelled = false;
if (report.Exception == null)
{
var moreDetails = string.Empty;
if (work.SynchronizationType == SynchronizationType.ContentUpdate)
{
moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}",
report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
UpdateSuccessfulSynchronizationTime();
Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destination.FileSystemUrl, moreDetails);
}
else
{
if (work.IsCancelled || report.Exception is TaskCanceledException)
{
synchronizationCancelled = true;
Log.DebugException(string.Format("{0} to {1} was cancelled", work, destination.FileSystemUrl), report.Exception);
}
else
{
Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destination.FileSystemUrl),
report.Exception);
}
}
Queue.SynchronizationFinished(work, destination.FileSystemUrl);
if (!synchronizationCancelled)
CreateSyncingConfiguration(fileName, work.FileETag, destination.FileSystemUrl, work.SynchronizationType);
publisher.Publish(new SynchronizationUpdateNotification
{
FileName = work.FileName,
DestinationFileSystemUrl = destination.FileSystemUrl,
SourceServerId = storage.Id,
SourceFileSystemUrl = FileSystemUrl,
Type = work.SynchronizationType,
Action = SynchronizationAction.Finish,
SynchronizationDirection = SynchronizationDirection.Outgoing
});
return report;
}
public async Task<HttpResponseMessage> MultipartProceed()
{
if (!Request.Content.IsMimeMultipartContent())
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault();
var tempFileName = RavenFileNameHelper.DownloadingFileName(fileName);
var sourceServerInfo = InnerHeaders.Value<ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = InnerHeaders.Value<Guid>("ETag");
var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.ContentUpdate);
Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName,
sourceFileETag, sourceServerInfo);
StorageStream localFile = null;
var isNewFile = false;
var isConflictResolved = false;
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(fileName, accessor);
FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor);
});
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(fileName, accessor));
NameValueCollection sourceMetadata = InnerHeaders.FilterHeaders();
var localMetadata = GetLocalMetadata(fileName);
if (localMetadata != null)
{
AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
localFile = StorageStream.Reading(Storage, fileName);
}
else
{
isNewFile = true;
}
Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWritting(Storage, Search, StorageOperationsTask,
tempFileName, sourceMetadata);
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
Log.Debug("Starting to process multipart content of a file '{0}'", fileName);
await Request.Content.ReadAsMultipartAsync(provider);
Log.Debug("Multipart content of a file '{0}' was processed", fileName);
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata));
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(fileName);
accessor.RenameFile(tempFileName, fileName);
Search.Delete(tempFileName);
Search.Index(fileName, sourceMetadata);
});
if (isNewFile)
{
Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes was updated.", tempFileName, fileName);
}
else
{
Log.Debug("Old file '{0}' was deleted. Indexes was updated.", fileName);
}
if (isConflictResolved)
{
ConflictArtifactManager.Delete(fileName);
Publisher.Publish(new ConflictResolved { FileName = fileName });
}
}
catch (Exception ex)
{
report.Exception = ex;
}
finally
{
if (localFile != null)
{
localFile.Dispose();
}
}
if (report.Exception == null)
{
Log.Debug(
"File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
else
{
Log.WarnException(
string.Format("Error has occured during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo),
report.Exception);
}
FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag);
PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update);
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
return Request.CreateResponse(HttpStatusCode.OK, report);
}
private void SaveSynchronizationReport(string fileName, IStorageActionsAccessor accessor, SynchronizationReport report)
{
var name = RavenFileNameHelper.SyncResultNameForFile(fileName);
accessor.SetConfig(name, report.AsConfig());
}
public HttpResponseMessage Delete(string fileName)
{
var sourceServerInfo = InnerHeaders.Value<ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = InnerHeaders.Value<Guid>("ETag");
Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName,
sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.Delete);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(fileName, accessor);
FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor);
});
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(fileName, accessor));
var localMetadata = GetLocalMetadata(fileName);
if (localMetadata != null)
{
var sourceMetadata = Request.Headers.FilterHeaders();
bool isConflictResolved;
AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(fileName);
var tombstoneMetadata = new NameValueCollection
{
{
SynchronizationConstants.RavenSynchronizationHistory,
localMetadata[SynchronizationConstants.RavenSynchronizationHistory]
},
{
SynchronizationConstants.RavenSynchronizationVersion,
localMetadata[SynchronizationConstants.RavenSynchronizationVersion]
},
{
SynchronizationConstants.RavenSynchronizationSource,
localMetadata[SynchronizationConstants.RavenSynchronizationSource]
}
}.WithDeleteMarker();
Historian.UpdateLastModified(tombstoneMetadata);
accessor.PutFile(fileName, 0, tombstoneMetadata, true);
});
PublishFileNotification(fileName, FileChangeAction.Delete);
}
}
catch (Exception ex)
{
report.Exception = ex;
Log.WarnException(
string.Format("Error was occured during deletion synchronization of file '{0}' from {1}", fileName,
sourceServerInfo), ex);
}
finally
{
FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag);
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
}
if (report.Exception == null)
{
Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceServerInfo);
}
return Request.CreateResponse(HttpStatusCode.OK, report);
}
public HttpResponseMessage Rename(string fileName, string rename)
{
var sourceServerInfo = InnerHeaders.Value<ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = InnerHeaders.Value<Guid>("ETag");
var sourceMetadata = InnerHeaders.FilterHeaders();
Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName,
rename, sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.Rename);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(fileName, accessor);
FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor);
});
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(fileName, accessor));
var localMetadata = GetLocalMetadata(fileName);
bool isConflictResolved;
AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
StorageOperationsTask.RenameFile(new RenameFileOperation
{
Name = fileName,
Rename = rename,
MetadataAfterOperation =
sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers()
});
}
catch (Exception ex)
{
report.Exception = ex;
Log.WarnException(
string.Format("Error was occured during renaming synchronization of file '{0}' from {1}", fileName,
sourceServerInfo), ex);
}
finally
{
FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag);
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
}
if (report.Exception == null)
Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo);
return Request.CreateResponse(HttpStatusCode.OK, report);
}
public HttpResponseMessage UpdateMetadata(string fileName)
{
var sourceServerInfo = InnerHeaders.Value<ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = InnerHeaders.Value<Guid>("ETag");
Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName,
sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.MetadataUpdate);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(fileName, accessor);
FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor);
});
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(fileName, accessor));
var localMetadata = GetLocalMetadata(fileName);
var sourceMetadata = InnerHeaders.FilterHeaders();
bool isConflictResolved;
AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Historian.UpdateLastModified(sourceMetadata);
Storage.Batch(accessor => accessor.UpdateFileMetadata(fileName, sourceMetadata));
Search.Index(fileName, sourceMetadata);
if (isConflictResolved)
{
ConflictArtifactManager.Delete(fileName);
Publisher.Publish(new ConflictResolved { FileName = fileName });
}
PublishFileNotification(fileName, FileChangeAction.Update);
}
catch (Exception ex)
{
report.Exception = ex;
Log.WarnException(
string.Format("Error was occured during metadata synchronization of file '{0}' from {1}", fileName,
sourceServerInfo), ex);
}
finally
{
FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag);
PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
}
if (report.Exception == null)
{
Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceServerInfo);
}
return Request.CreateResponse(HttpStatusCode.OK, report);
}
private void FinishSynchronization(string fileName, SynchronizationReport report, ServerInfo sourceServer,
Guid sourceFileETag)
{
try
{
// we want to execute those operation in a single batch but we also have to ensure that
// Raven/Synchronization/Sources/sourceServerId config is modified only by one finishing synchronization at the same time
SynchronizationFinishLocks.GetOrAdd(sourceServer.Id, new ReaderWriterLockSlim()).EnterWriteLock();
Storage.Batch(accessor =>
{
SaveSynchronizationReport(fileName, accessor, report);
FileLockManager.UnlockByDeletingSyncConfiguration(fileName, accessor);
if (report.Exception == null)
{
SaveSynchronizationSourceInformation(sourceServer, sourceFileETag, accessor);
}
});
}
catch (Exception ex)
{
Log.ErrorException(
string.Format("Failed to finish synchronization of a file '{0}' from {1}", fileName, sourceServer), ex);
}
finally
{
SynchronizationFinishLocks.GetOrAdd(sourceServer.Id, new ReaderWriterLockSlim()).ExitWriteLock();
}
}
private void SaveSynchronizationReport(string fileName, IStorageActionsAccessor accessor, SynchronizationReport report)
{
var name = RavenFileNameHelper.SyncResultNameForFile(fileName);
accessor.SetConfig(name, JsonExtensions.ToJObject(report));
}
public async Task Should_report_that_file_is_broken_if_last_synchronization_set_exception()
{
var destinationClient = NewClient(0);
var sampleGuid = Guid.NewGuid();
var failureSynchronization = new SynchronizationReport("test.bin", sampleGuid, SynchronizationType.Unknown)
{
Exception = new Exception("There was an exception in last synchronization.")
};
await destinationClient.Config.SetConfig(RavenFileNameHelper.SyncResultNameForFile("test.bin"), failureSynchronization);
var confirmations = await destinationClient.Synchronization
.ConfirmFilesAsync(new List<Tuple<string, Guid>>
{
new Tuple<string, Guid>(
"test.bin", sampleGuid)
});
confirmations = confirmations.ToList();
Assert.Equal(1, confirmations.Count());
Assert.Equal(FileStatus.Broken, confirmations.ToArray()[0].Status);
Assert.Equal("test.bin", confirmations.ToArray()[0].FileName);
}
public void Should_report_that_file_is_broken_if_last_synchronization_set_exception()
{
var destinationClient = NewClient(0);
var sampleGuid = Guid.NewGuid();
var failureSynchronization = new SynchronizationReport("test.bin", sampleGuid, SynchronizationType.Unknown)
{Exception = new Exception("There was an exception in last synchronization.")};
//var sb = new StringBuilder();
//var jw = new JsonTextWriter(new StringWriter(sb));
//new JsonSerializer().Serialize(jw, failureSynchronization);
//destinationClient.Config.SetConfig(RavenFileNameHelper.SyncResultNameForFile("test.bin"),
// new NameValueCollection() {{"value", sb.ToString()}}).Wait();
destinationClient.Config.SetConfig(RavenFileNameHelper.SyncResultNameForFile("test.bin"),
failureSynchronization.AsConfig()).Wait();
var confirmations =
destinationClient.Synchronization.ConfirmFilesAsync(new List<Tuple<string, Guid>>
{
new Tuple<string, Guid>(
"test.bin", sampleGuid)
}).Result.ToList();
Assert.Equal(1, confirmations.Count());
Assert.Equal(FileStatus.Broken, confirmations.ToArray()[0].Status);
Assert.Equal("test.bin", confirmations.ToArray()[0].FileName);
}