Ejemplo n.º 1
0
        public override Task<SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination)
		{
			FileAndPagesInformation fileAndPages = null;
			Storage.Batch(accessor => fileAndPages = accessor.GetFile(FileName, 0, 0));

            return destination.RenameAsync(FileName, rename, fileAndPages.Metadata, FileSystemInfo);
		}
Ejemplo n.º 2
0
 public RemoteRdcManager(IAsyncFilesSynchronizationCommands ravenFileSystemClient, ISignatureRepository localSignatureRepository,
                         ISignatureRepository remoteCacheSignatureRepository)
 {
     this.localSignatureRepository       = localSignatureRepository;
     this.remoteCacheSignatureRepository = remoteCacheSignatureRepository;
     this.ravenFileSystemClient          = ravenFileSystemClient;
 }
Ejemplo n.º 3
0
        protected async Task <SynchronizationReport> HandleConflict(IAsyncFilesSynchronizationCommands destination, ConflictItem conflict, ILog log)
        {
            var conflictResolutionStrategy = await destination.Commands.Synchronization.GetResolutionStrategyFromDestinationResolvers(conflict, FileMetadata);

            switch (conflictResolutionStrategy)
            {
            case ConflictResolutionStrategy.NoResolution:
                return(await ApplyConflictOnDestinationAsync(conflict, FileMetadata, destination, ServerInfo.FileSystemUrl, log));

            case ConflictResolutionStrategy.CurrentVersion:
                await ApplyConflictOnDestinationAsync(conflict, FileMetadata, destination, ServerInfo.FileSystemUrl, log);

                await destination.Commands.Synchronization.ResolveConflictAsync(FileName, conflictResolutionStrategy);

                return(new SynchronizationReport(FileName, FileETag, SynchronizationType));

            case ConflictResolutionStrategy.RemoteVersion:
                // we can push the file even though it conflicted, the conflict will be automatically resolved on the destination side
                return(null);

            default:
                return(new SynchronizationReport(FileName, FileETag, SynchronizationType)
                {
                    Exception = new SynchronizationException(string.Format("Unknown resulution stragegy: {0}", conflictResolutionStrategy)),
                });
            }
        }
Ejemplo n.º 4
0
        public override async Task <SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination)
        {
            AssertLocalFileExistsAndIsNotConflicted(FileMetadata);

            var destinationMetadata = await destination.Commands.GetMetadataForAsync(FileName);

            if (destinationMetadata == null)
            {
                // if file doesn't exist on destination server - upload it there
                return(await UploadToAsync(destination));
            }

            var destinationServerRdcStats = await destination.GetRdcStatsAsync();

            if (!IsRemoteRdcCompatible(destinationServerRdcStats))
            {
                throw new SynchronizationException("Incompatible RDC version detected on destination server");
            }

            var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.FileSystemUrl);

            if (conflict != null)
            {
                var report = await HandleConflict(destination, conflict, log);

                if (report != null)
                {
                    return(report);
                }
            }

            using (var localSignatureRepository = new StorageSignatureRepository(Storage, FileName))
                using (var remoteSignatureCache = new VolatileSignatureRepository(FileName))
                {
                    var localRdcManager       = new LocalRdcManager(localSignatureRepository, Storage, sigGenerator);
                    var destinationRdcManager = new RemoteRdcManager(destination, localSignatureRepository, remoteSignatureCache);

                    log.Debug("Starting to retrieve signatures of a local file '{0}'.", FileName);

                    Cts.Token.ThrowIfCancellationRequested();

                    // first we need to create a local file signatures before we synchronize with remote ones
                    var localSignatureManifest = await localRdcManager.GetSignatureManifestAsync(FileDataInfo);

                    log.Debug("Number of a local file '{0}' signatures was {1}.", FileName, localSignatureManifest.Signatures.Count);

                    if (localSignatureManifest.Signatures.Any())
                    {
                        var destinationSignatureManifest = await destinationRdcManager.SynchronizeSignaturesAsync(FileDataInfo, Cts.Token);

                        if (destinationSignatureManifest.Signatures.Any())
                        {
                            return(await SynchronizeTo(destination, localSignatureRepository, remoteSignatureCache, localSignatureManifest, destinationSignatureManifest));
                        }
                    }

                    return(await UploadToAsync(destination));
                }
        }
Ejemplo n.º 5
0
 public RemoteRdcManager(IAsyncFilesSynchronizationCommands commands,
                         ISignatureRepository localSignatureRepository,
                         ISignatureRepository remoteCacheSignatureRepository)
 {
     this.localSignatureRepository       = localSignatureRepository;
     this.remoteCacheSignatureRepository = remoteCacheSignatureRepository;
     this.commands = commands;
 }
Ejemplo n.º 6
0
        public RemoteRdcManager(IAsyncFilesSynchronizationCommands commands, 
                                ISignatureRepository localSignatureRepository,
								ISignatureRepository remoteCacheSignatureRepository)
		{            
			this.localSignatureRepository = localSignatureRepository;
			this.remoteCacheSignatureRepository = remoteCacheSignatureRepository;
			this.commands = commands;
		}
Ejemplo n.º 7
0
        public override Task <SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination)
        {
            FileAndPagesInformation fileAndPages = null;

            Storage.Batch(accessor => fileAndPages = accessor.GetFile(FileName, 0, 0));

            return(destination.RenameAsync(FileName, rename, fileAndPages.Metadata, ServerInfo));
        }
 public SynchronizationMultipartRequest(IAsyncFilesSynchronizationCommands destination, ServerInfo serverInfo, string fileName,
                                        RavenJObject sourceMetadata, Stream sourceStream, IList <RdcNeed> needList)
 {
     this.destination    = destination;
     this.serverInfo     = serverInfo;
     this.fileName       = fileName;
     this.sourceMetadata = sourceMetadata;
     this.sourceStream   = sourceStream;
     this.needList       = needList;
     syncingBoundary     = "syncing";
 }
        public SynchronizationMultipartRequest(IAsyncFilesSynchronizationCommands destination, FileSystemInfo fileSystemInfo, string fileName,
                                               RavenJObject sourceMetadata, Stream sourceStream, IList<RdcNeed> needList)
		{
			this.destination = destination;
			this.fileSystemInfo = fileSystemInfo;
			this.fileName = fileName;
			this.sourceMetadata = sourceMetadata;
			this.sourceStream = sourceStream;
			this.needList = needList;
			syncingBoundary = "syncing";
		}
Ejemplo n.º 10
0
        public override Task <SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination)
        {
            AssertLocalFileExistsAndIsNotConflicted(FileMetadata);

            var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.FileSystemUrl);

            if (conflict != null)
            {
                return(ApplyConflictOnDestinationAsync(conflict, FileMetadata, destination, ServerInfo.FileSystemUrl, log));
            }

            return(destination.UpdateMetadataAsync(FileName, FileMetadata, ServerInfo));
        }
Ejemplo n.º 11
0
        public override async Task<SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination)
        {
            AssertLocalFileExistsAndIsNotConflicted(FileMetadata);

            var destinationMetadata = await destination.Commands.GetMetadataForAsync(FileName);
            if (destinationMetadata == null)
            {
                // if file doesn't exist on destination server - upload it there
                return await UploadToAsync(destination);
            }

            var destinationServerRdcStats = await destination.GetRdcStatsAsync();
            if (!IsRemoteRdcCompatible(destinationServerRdcStats))
                throw new SynchronizationException("Incompatible RDC version detected on destination server");

            var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.FileSystemUrl);
	        if (conflict != null)
	        {
		        var report = await HandleConflict(destination, conflict, log);

		        if (report != null)
			        return report;
	        }

            using (var localSignatureRepository = new StorageSignatureRepository(Storage, FileName))
            using (var remoteSignatureCache = new VolatileSignatureRepository(FileName))
            {
                var localRdcManager = new LocalRdcManager(localSignatureRepository, Storage, sigGenerator);
                var destinationRdcManager = new RemoteRdcManager(destination, localSignatureRepository, remoteSignatureCache);

                log.Debug("Starting to retrieve signatures of a local file '{0}'.", FileName);

                Cts.Token.ThrowIfCancellationRequested();

                // first we need to create a local file signatures before we synchronize with remote ones
                var localSignatureManifest = await localRdcManager.GetSignatureManifestAsync(FileDataInfo);

                log.Debug("Number of a local file '{0}' signatures was {1}.", FileName, localSignatureManifest.Signatures.Count);

                if (localSignatureManifest.Signatures.Any())
                {
                    var destinationSignatureManifest = await destinationRdcManager.SynchronizeSignaturesAsync(FileDataInfo, Cts.Token);
                    if (destinationSignatureManifest.Signatures.Any())
                    {
                        return await SynchronizeTo(destination, localSignatureRepository, remoteSignatureCache, localSignatureManifest, destinationSignatureManifest);
                    }
                }

                return await UploadToAsync(destination);
            }
        }
Ejemplo n.º 12
0
        private Task <SynchronizationReport> PushByUsingMultipartRequest(IAsyncFilesSynchronizationCommands destination, Stream sourceFileStream,
                                                                         IList <RdcNeed> needList)
        {
            Cts.Token.ThrowIfCancellationRequested();

            multipartRequest = new SynchronizationMultipartRequest(destination, ServerInfo, FileName, FileMetadata, sourceFileStream, needList);

            var bytesToTransferCount = needList.Where(x => x.BlockType == RdcNeedType.Source).Sum(x => (double)x.BlockLength);

            log.Debug(
                "Synchronizing a file '{0}' (ETag {1}) to {2} by using multipart request. Need list length is {3}. Number of bytes that needs to be transfered is {4}",
                FileName, FileETag, destination, needList.Count, bytesToTransferCount);

            return(multipartRequest.PushChangesAsync(Cts.Token));
        }
        public override async Task<SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination)
		{
			AssertLocalFileExistsAndIsNotConflicted(FileMetadata);

			var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.FileSystemUrl);

	        if (conflict != null)
	        {
				var report = await HandleConflict(destination, conflict, log);

				if (report != null)
					return report;
	        }

            return await destination.UpdateMetadataAsync(FileName, FileMetadata, ServerInfo);
		}
Ejemplo n.º 14
0
        public override async Task <SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination)
        {
            AssertLocalFileExistsAndIsNotConflicted(FileMetadata);

            var conflict = CheckConflictWithDestination(FileMetadata, destinationMetadata, ServerInfo.FileSystemUrl);

            if (conflict != null)
            {
                var report = await HandleConflict(destination, conflict, log);

                if (report != null)
                {
                    return(report);
                }
            }

            return(await destination.UpdateMetadataAsync(FileName, FileMetadata, ServerInfo));
        }
Ejemplo n.º 15
0
        public async Task <SynchronizationReport> UploadToAsync(IAsyncFilesSynchronizationCommands destination)
        {
            using (var sourceFileStream = StorageStream.Reading(Storage, FileName))
            {
                var fileSize = sourceFileStream.Length;

                var onlySourceNeed = new List <RdcNeed>
                {
                    new RdcNeed
                    {
                        BlockType   = RdcNeedType.Source,
                        BlockLength = (ulong)fileSize,
                        FileOffset  = 0
                    }
                };

                return(await PushByUsingMultipartRequest(destination, sourceFileStream, onlySourceNeed));
            }
        }
Ejemplo n.º 16
0
        private async Task <SynchronizationReport> SynchronizeTo(IAsyncFilesSynchronizationCommands destination,
                                                                 ISignatureRepository localSignatureRepository,
                                                                 ISignatureRepository remoteSignatureRepository,
                                                                 SignatureManifest sourceSignatureManifest,
                                                                 SignatureManifest destinationSignatureManifest)
        {
            var seedSignatureInfo   = SignatureInfo.Parse(destinationSignatureManifest.Signatures.Last().Name);
            var sourceSignatureInfo = SignatureInfo.Parse(sourceSignatureManifest.Signatures.Last().Name);

            using (var localFile = StorageStream.Reading(Storage, FileName))
            {
                IList <RdcNeed> needList;
                using (var needListGenerator = new NeedListGenerator(remoteSignatureRepository, localSignatureRepository))
                {
                    needList = needListGenerator.CreateNeedsList(seedSignatureInfo, sourceSignatureInfo, Cts.Token);
                }

                return(await PushByUsingMultipartRequest(destination, localFile, needList));
            }
        }
Ejemplo n.º 17
0
        private async Task <SynchronizationConfirmation[]> ConfirmPushedFiles(IEnumerable <SynchronizationDetails> filesNeedConfirmation, IAsyncFilesSynchronizationCommands commands)
        {
            if (!filesNeedConfirmation.Any())
            {
                return(new SynchronizationConfirmation[0]);
            }

            return(await commands.GetConfirmationForFilesAsync(filesNeedConfirmation.Select(x => new Tuple <string, Guid>(x.FileName, x.FileETag))));
        }
Ejemplo n.º 18
0
        public RemoteSignaturePartialAccess(IAsyncFilesSynchronizationCommands synchronizationClient, string fileName)
		{
			this.synchronizationClient = synchronizationClient;
			_fileName = fileName;
		}
Ejemplo n.º 19
0
		private async Task<SynchronizationConfirmation[]> ConfirmPushedFiles(IList<SynchronizationDetails> filesNeedConfirmation, IAsyncFilesSynchronizationCommands commands)
		{
			if (!filesNeedConfirmation.Any())
				return new SynchronizationConfirmation[0];

            return await commands.GetConfirmationForFilesAsync(filesNeedConfirmation.Select(x => new Tuple<string, Etag>(x.FileName, x.FileETag)));
		}
Ejemplo n.º 20
0
        private async Task<SynchronizationReport> PerformSynchronizationAsync(IAsyncFilesSynchronizationCommands destination,
																			  SynchronizationWorkItem work)
		{
            var commands = (IAsyncFilesCommandsImpl)destination.Commands;
            string destinationUrl = commands.UrlFor();

			Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
                       work.GetType().Name, work.FileName, destinationUrl);

            if (!CanSynchronizeTo(destinationUrl))
			{
				Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
                          destinationUrl, work.FileName);

                if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
                {
                    publisher.Publish(new SynchronizationUpdateNotification
                    {
                        FileName = work.FileName,
                        DestinationFileSystemUrl = destinationUrl,
                        SourceServerId = storage.Id,
                        SourceFileSystemUrl = FileSystemUrl,
                        Type = work.SynchronizationType,
                        Action = SynchronizationAction.Enqueue,
                        Direction = SynchronizationDirection.Outgoing
                    });
                }

				return new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
				{
					Exception = new SynchronizationException(string.Format(
						"The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
                        destinationUrl, work.FileName))
				};
			}

			string fileName = work.FileName;
            synchronizationQueue.SynchronizationStarted(work, destinationUrl);
			publisher.Publish(new SynchronizationUpdateNotification
			{
				FileName = work.FileName,
                DestinationFileSystemUrl = destinationUrl,
				SourceServerId = storage.Id,
				SourceFileSystemUrl = FileSystemUrl,
				Type = work.SynchronizationType,
				Action = SynchronizationAction.Start,
				Direction = SynchronizationDirection.Outgoing
			});

			SynchronizationReport report;

			try
			{
				report = await work.PerformAsync(destination);
			}
			catch (Exception ex)
			{
				report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
				{
					Exception = ex,
				};
			}

			var synchronizationCancelled = false;

			if (report.Exception == null)
			{
				var moreDetails = string.Empty;

				if (work.SynchronizationType == SynchronizationType.ContentUpdate)
				{
					moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}",
												report.BytesTransfered, report.BytesCopied, report.NeedListLength);
				}

                UpdateSuccessfulSynchronizationTime();

                Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destinationUrl, moreDetails);
			}
			else
			{
				if (work.IsCancelled || report.Exception is TaskCanceledException)
				{
					synchronizationCancelled = true;
                    Log.DebugException(string.Format("{0} to {1} was canceled", work, destinationUrl), report.Exception);
				}
				else
				{
                    Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl), report.Exception);
				}
			}

            Queue.SynchronizationFinished(work, destinationUrl);

			if (!synchronizationCancelled)
                CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType);

			publisher.Publish(new SynchronizationUpdateNotification
			{
				FileName = work.FileName,
                DestinationFileSystemUrl = destinationUrl,
				SourceServerId = storage.Id,
				SourceFileSystemUrl = FileSystemUrl,
				Type = work.SynchronizationType,
				Action = SynchronizationAction.Finish,
				Direction = SynchronizationDirection.Outgoing
			});

			return report;
		}
Ejemplo n.º 21
0
        private IEnumerable<Task<SynchronizationReport>> SynchronizePendingFilesAsync(SynchronizationDestination destination, IAsyncFilesSynchronizationCommands destinationCommands, bool forceSyncingContinuation)
		{
            var commands = (IAsyncFilesCommandsImpl)destinationCommands.Commands;

            var destinationUrl = commands.UrlFor();

            for (var i = 0; i < AvailableSynchronizationRequestsTo(destinationUrl); i++)
			{
				SynchronizationWorkItem work;
                if (!synchronizationQueue.TryDequePendingSynchronization(destinationUrl, out work))
					break;

                if (synchronizationQueue.IsDifferentWorkForTheSameFileBeingPerformed(work, destinationUrl))
				{
					Log.Debug("There was an already being performed synchronization of a file '{0}' to {1}", work.FileName,
							  destinationCommands);

                    if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) // add it again at the end of the queue
                    {
                        // add it again at the end of the queue
                        publisher.Publish(new SynchronizationUpdateNotification
                        {
                            FileName = work.FileName,
                            DestinationFileSystemUrl = destinationUrl,
                            SourceServerId = storage.Id,
                            SourceFileSystemUrl = FileSystemUrl,
                            Type = work.SynchronizationType,
                            Action = SynchronizationAction.Enqueue,
                            Direction = SynchronizationDirection.Outgoing
                        });
                    }
				}
				else
				{
					var workTask = PerformSynchronizationAsync(destinationCommands, work);
					if (forceSyncingContinuation)
						workTask.ContinueWith(async t =>
						{
							if (CanSynchronizeTo(destinationUrl))
								await SynchronizeDestinationAsync(destination, true);
						});

                    yield return workTask;
				}
			}
		}
Ejemplo n.º 22
0
        private async Task EnqueueMissingUpdatesAsync(IAsyncFilesSynchronizationCommands destination,
													  SourceSynchronizationInformation lastEtag,
                                                      IList<FileHeader> needSyncingAgain)
		{
			LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
						 needSyncingAgain);

            var commands = (IAsyncFilesCommandsImpl)destination.Commands;

            var filesToSynchronization = new HashSet<FileHeader>(GetFilesToSynchronization(lastEtag, 100),
																 new FileHeaderNameEqualityComparer());

			LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
						 filesToSynchronization);

            foreach (FileHeader needSyncing in needSyncingAgain)
			{
				filesToSynchronization.Add(needSyncing);
			}

			var filteredFilesToSynchronization =
				filesToSynchronization.Where(
					x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();

			if (filesToSynchronization.Count > 0)
				LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);

            // Early break. There are no files to synchronize to the selected destination. 
			if (!filteredFilesToSynchronization.Any())
				return;

            var baseUrl = commands.UrlFor();

			foreach (var fileHeader in filteredFilesToSynchronization)
			{
                var file = fileHeader.FullPath;
				var localMetadata = GetLocalMetadata(file);

				RavenJObject destinationMetadata;

                try
                {
                    destinationMetadata = await destination.Commands.GetMetadataForAsync(file);
                }
                catch (Exception ex)
                {
                    Log.WarnException(
                        string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
                            baseUrl), ex);

                    continue;
                }                

				NoSyncReason reason;
                var work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
				if (work == null)
				{
                    Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, baseUrl, reason.GetDescription());

					if (reason == NoSyncReason.ContainedInDestinationHistory)
					{
                        var etag = localMetadata.Value<Guid>(Constants.MetadataEtagField);
                        await destination.IncrementLastETagAsync(storage.Id, baseUrl, etag);
                        RemoveSyncingConfiguration(file, baseUrl);
					}

					continue;
				}

                if (synchronizationQueue.EnqueueSynchronization(baseUrl, work))
                {
                    publisher.Publish(new SynchronizationUpdateNotification
                    {
                        FileName = work.FileName,
                        DestinationFileSystemUrl = baseUrl,
                        SourceServerId = storage.Id,
                        SourceFileSystemUrl = FileSystemUrl,
                        Type = work.SynchronizationType,
                        Action = SynchronizationAction.Enqueue,
                        Direction = SynchronizationDirection.Outgoing
                    });
                }
			}
		}
Ejemplo n.º 23
0
        private async Task<SynchronizationReport> SynchronizeTo(IAsyncFilesSynchronizationCommands destination,
																ISignatureRepository localSignatureRepository,
																ISignatureRepository remoteSignatureRepository,
																SignatureManifest sourceSignatureManifest,
																SignatureManifest destinationSignatureManifest)
		{
			var seedSignatureInfo = SignatureInfo.Parse(destinationSignatureManifest.Signatures.Last().Name);
			var sourceSignatureInfo = SignatureInfo.Parse(sourceSignatureManifest.Signatures.Last().Name);

			using (var localFile = StorageStream.Reading(Storage, FileName))
			{
				IList<RdcNeed> needList;
				using (var needListGenerator = new NeedListGenerator(remoteSignatureRepository, localSignatureRepository))
				{
					needList = needListGenerator.CreateNeedsList(seedSignatureInfo, sourceSignatureInfo, Cts.Token);
				}

				return await PushByUsingMultipartRequest(destination, localFile, needList);
			}
		}
Ejemplo n.º 24
0
        public async Task<SynchronizationReport> UploadToAsync(IAsyncFilesSynchronizationCommands destination)
		{
			using (var sourceFileStream = StorageStream.Reading(Storage, FileName))
			{
				var fileSize = sourceFileStream.Length;

				var onlySourceNeed = new List<RdcNeed>
					                     {
						                     new RdcNeed
							                     {
								                     BlockType = RdcNeedType.Source,
								                     BlockLength = (ulong) fileSize,
								                     FileOffset = 0
							                     }
					                     };

				return await PushByUsingMultipartRequest(destination, sourceFileStream, onlySourceNeed);
			}
		}
Ejemplo n.º 25
0
 private Task <SynchronizationConfirmation[]> ConfirmPushedFiles(IList <SynchronizationDetails> filesNeedConfirmation, IAsyncFilesSynchronizationCommands destinationClient)
 {
     if (filesNeedConfirmation.Count == 0)
     {
         return(new CompletedTask <SynchronizationConfirmation[]>(new SynchronizationConfirmation[0]));
     }
     return(destinationClient.GetConfirmationForFilesAsync(filesNeedConfirmation.Select(x => new Tuple <string, Guid>(x.FileName, x.FileETag))));
 }
Ejemplo n.º 26
0
        private Task<SynchronizationReport> PushByUsingMultipartRequest(IAsyncFilesSynchronizationCommands destination, Stream sourceFileStream,
																		IList<RdcNeed> needList)
		{
			Cts.Token.ThrowIfCancellationRequested();

			multipartRequest = new SynchronizationMultipartRequest(destination, ServerInfo, FileName, FileMetadata, sourceFileStream, needList);

			var bytesToTransferCount = needList.Where(x => x.BlockType == RdcNeedType.Source).Sum(x => (double)x.BlockLength);

			log.Debug(
				"Synchronizing a file '{0}' (ETag {1}) to {2} by using multipart request. Need list length is {3}. Number of bytes that needs to be transfered is {4}",
				FileName, FileETag, destination, needList.Count, bytesToTransferCount);

			return multipartRequest.PushChangesAsync(Cts.Token);
		}
Ejemplo n.º 27
0
        private async Task EnqueueMissingUpdatesAsync(IAsyncFilesSynchronizationCommands destination,
                                                      SourceSynchronizationInformation lastEtag,
                                                      IList <FileHeader> needSyncingAgain)
        {
            LogFilesInfo("There were {0} file(s) that needed synchronization because the previous one went wrong: {1}",
                         needSyncingAgain);

            var commands = (IAsyncFilesCommandsImpl)destination.Commands;

            var filesToSynchronization = new HashSet <FileHeader>(GetFilesToSynchronization(lastEtag, 100),
                                                                  new FileHeaderNameEqualityComparer());

            LogFilesInfo("There were {0} file(s) that needed synchronization because of greater ETag value: {1}",
                         filesToSynchronization);

            foreach (FileHeader needSyncing in needSyncingAgain)
            {
                filesToSynchronization.Add(needSyncing);
            }

            var filteredFilesToSynchronization =
                filesToSynchronization.Where(
                    x => synchronizationStrategy.Filter(x, lastEtag.DestinationServerId, filesToSynchronization)).ToList();

            if (filesToSynchronization.Count > 0)
            {
                LogFilesInfo("There were {0} file(s) that needed synchronization after filtering: {1}", filteredFilesToSynchronization);
            }

            // Early break. There are no files to synchronize to the selected destination.
            if (!filteredFilesToSynchronization.Any())
            {
                return;
            }

            var baseUrl = commands.UrlFor();

            foreach (var fileHeader in filteredFilesToSynchronization)
            {
                var file          = fileHeader.FullPath;
                var localMetadata = GetLocalMetadata(file);

                RavenJObject destinationMetadata;

                try
                {
                    destinationMetadata = await destination.Commands.GetMetadataForAsync(file);
                }
                catch (Exception ex)
                {
                    Log.WarnException(
                        string.Format("Could not retrieve a metadata of a file '{0}' from {1} in order to determine needed synchronization type", file,
                                      baseUrl), ex);

                    continue;
                }

                NoSyncReason reason;
                var          work = synchronizationStrategy.DetermineWork(file, localMetadata, destinationMetadata, FileSystemUrl, out reason);
                if (work == null)
                {
                    Log.Debug("File '{0}' were not synchronized to {1}. {2}", file, baseUrl, reason.GetDescription());

                    if (reason == NoSyncReason.ContainedInDestinationHistory)
                    {
                        var etag = localMetadata.Value <Guid>(Constants.MetadataEtagField);
                        await destination.IncrementLastETagAsync(storage.Id, baseUrl, etag);

                        RemoveSyncingConfiguration(file, baseUrl);
                    }

                    continue;
                }

                if (synchronizationQueue.EnqueueSynchronization(baseUrl, work))
                {
                    publisher.Publish(new SynchronizationUpdateNotification
                    {
                        FileName = work.FileName,
                        DestinationFileSystemUrl = baseUrl,
                        SourceServerId           = storage.Id,
                        SourceFileSystemUrl      = FileSystemUrl,
                        Type      = work.SynchronizationType,
                        Action    = SynchronizationAction.Enqueue,
                        Direction = SynchronizationDirection.Outgoing
                    });
                }
            }
        }
Ejemplo n.º 28
0
        private IEnumerable <Task <SynchronizationReport> > SynchronizePendingFilesAsync(IAsyncFilesSynchronizationCommands destination, bool forceSyncingContinuation)
        {
            var commands = (IAsyncFilesCommandsImpl)destination.Commands;

            var destinationUrl = commands.UrlFor();

            for (var i = 0; i < AvailableSynchronizationRequestsTo(destinationUrl); i++)
            {
                SynchronizationWorkItem work;
                if (!synchronizationQueue.TryDequePendingSynchronization(destinationUrl, out work))
                {
                    break;
                }

                if (synchronizationQueue.IsDifferentWorkForTheSameFileBeingPerformed(work, destinationUrl))
                {
                    Log.Debug("There was an already being performed synchronization of a file '{0}' to {1}", work.FileName,
                              destination);

                    if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work)) // add it again at the end of the queue
                    {
                        // add it again at the end of the queue
                        publisher.Publish(new SynchronizationUpdateNotification
                        {
                            FileName = work.FileName,
                            DestinationFileSystemUrl = destinationUrl,
                            SourceServerId           = storage.Id,
                            SourceFileSystemUrl      = FileSystemUrl,
                            Type      = work.SynchronizationType,
                            Action    = SynchronizationAction.Enqueue,
                            Direction = SynchronizationDirection.Outgoing
                        });
                    }
                }
                else
                {
                    var workTask = PerformSynchronizationAsync(destination, work);
                    if (forceSyncingContinuation)
                    {
                        workTask.ContinueWith(t => SynchronizePendingFilesAsync(destination, true).ToArray());
                    }

                    yield return(workTask);
                }
            }
        }
Ejemplo n.º 29
0
        private async Task <SynchronizationReport> PerformSynchronizationAsync(IAsyncFilesSynchronizationCommands destination,
                                                                               SynchronizationWorkItem work)
        {
            var    commands       = (IAsyncFilesCommandsImpl)destination.Commands;
            string destinationUrl = commands.UrlFor();

            Log.Debug("Starting to perform {0} for a file '{1}' and a destination server {2}",
                      work.GetType().Name, work.FileName, destinationUrl);

            if (!CanSynchronizeTo(destinationUrl))
            {
                Log.Debug("The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
                          destinationUrl, work.FileName);

                if (synchronizationQueue.EnqueueSynchronization(destinationUrl, work))
                {
                    publisher.Publish(new SynchronizationUpdateNotification
                    {
                        FileName = work.FileName,
                        DestinationFileSystemUrl = destinationUrl,
                        SourceServerId           = storage.Id,
                        SourceFileSystemUrl      = FileSystemUrl,
                        Type      = work.SynchronizationType,
                        Action    = SynchronizationAction.Enqueue,
                        Direction = SynchronizationDirection.Outgoing
                    });
                }

                return(new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
                {
                    Exception = new SynchronizationException(string.Format(
                                                                 "The limit of active synchronizations to {0} server has been achieved. Cannot process a file '{1}'.",
                                                                 destinationUrl, work.FileName))
                });
            }

            string fileName = work.FileName;

            synchronizationQueue.SynchronizationStarted(work, destinationUrl);
            publisher.Publish(new SynchronizationUpdateNotification
            {
                FileName = work.FileName,
                DestinationFileSystemUrl = destinationUrl,
                SourceServerId           = storage.Id,
                SourceFileSystemUrl      = FileSystemUrl,
                Type      = work.SynchronizationType,
                Action    = SynchronizationAction.Start,
                Direction = SynchronizationDirection.Outgoing
            });

            SynchronizationReport report;

            try
            {
                report = await work.PerformAsync(destination);
            }
            catch (Exception ex)
            {
                report = new SynchronizationReport(work.FileName, work.FileETag, work.SynchronizationType)
                {
                    Exception = ex,
                };
            }

            var synchronizationCancelled = false;

            if (report.Exception == null)
            {
                var moreDetails = string.Empty;

                if (work.SynchronizationType == SynchronizationType.ContentUpdate)
                {
                    moreDetails = string.Format(". {0} bytes were transfered and {1} bytes copied. Need list length was {2}",
                                                report.BytesTransfered, report.BytesCopied, report.NeedListLength);
                }

                UpdateSuccessfulSynchronizationTime();

                Log.Debug("{0} to {1} has finished successfully{2}", work.ToString(), destinationUrl, moreDetails);
            }
            else
            {
                if (work.IsCancelled || report.Exception is TaskCanceledException)
                {
                    synchronizationCancelled = true;
                    Log.DebugException(string.Format("{0} to {1} was canceled", work, destinationUrl), report.Exception);
                }
                else
                {
                    Log.WarnException(string.Format("{0} to {1} has finished with the exception", work, destinationUrl), report.Exception);
                }
            }

            Queue.SynchronizationFinished(work, destinationUrl);

            if (!synchronizationCancelled)
            {
                CreateSyncingConfiguration(fileName, work.FileETag, destinationUrl, work.SynchronizationType);
            }

            publisher.Publish(new SynchronizationUpdateNotification
            {
                FileName = work.FileName,
                DestinationFileSystemUrl = destinationUrl,
                SourceServerId           = storage.Id,
                SourceFileSystemUrl      = FileSystemUrl,
                Type      = work.SynchronizationType,
                Action    = SynchronizationAction.Finish,
                Direction = SynchronizationDirection.Outgoing
            });

            return(report);
        }
Ejemplo n.º 30
0
 public RemoteSignaturePartialAccess(IAsyncFilesSynchronizationCommands synchronizationClient, string fileName)
 {
     this.synchronizationClient = synchronizationClient;
     _fileName = fileName;
 }
Ejemplo n.º 31
0
		protected async Task<SynchronizationReport> HandleConflict(IAsyncFilesSynchronizationCommands destination, ConflictItem conflict, ILog log)
		{
			var conflictResolutionStrategy = await destination.Commands.Synchronization.GetResolutionStrategyFromDestinationResolvers(conflict, FileMetadata);

			switch (conflictResolutionStrategy)
			{
				case ConflictResolutionStrategy.NoResolution:
					return await ApplyConflictOnDestinationAsync(conflict, FileMetadata, destination, FileSystemInfo.Url, log);
				case ConflictResolutionStrategy.CurrentVersion:
					await ApplyConflictOnDestinationAsync(conflict, FileMetadata, destination, FileSystemInfo.Url, log);
					await destination.Commands.Synchronization.ResolveConflictAsync(FileName, conflictResolutionStrategy);
					return new SynchronizationReport(FileName, FileETag, SynchronizationType);
				case ConflictResolutionStrategy.RemoteVersion:
					// we can push the file even though it conflicted, the conflict will be automatically resolved on the destination side
					return null;
				default:
					return new SynchronizationReport(FileName, FileETag, SynchronizationType)
					{
						Exception = new SynchronizationException(string.Format("Unknown resolution strategy: {0}", conflictResolutionStrategy)),
					};
			}
		}
Ejemplo n.º 32
0
        protected async Task <SynchronizationReport> ApplyConflictOnDestinationAsync(ConflictItem conflict, RavenJObject remoteMetadata, IAsyncFilesSynchronizationCommands destination, string localServerUrl, ILog log)
        {
            var commands = (IAsyncFilesCommandsImpl)destination.Commands;

            log.Debug("File '{0}' is in conflict with destination version from {1}. Applying conflict on destination", FileName, commands.UrlFor());

            try
            {
                var version  = conflict.RemoteHistory.Last().Version;
                var serverId = conflict.RemoteHistory.Last().ServerId;
                var history  = new List <HistoryItem>(conflict.RemoteHistory);
                history.RemoveAt(conflict.RemoteHistory.Count - 1);

                await destination.ApplyConflictAsync(FileName, version, serverId, remoteMetadata, localServerUrl);
            }
            catch (Exception ex)
            {
                log.WarnException(string.Format("Failed to apply conflict on {0} for file '{1}'", destination, FileName), ex);
            }

            return(new SynchronizationReport(FileName, FileETag, SynchronizationType)
            {
                Exception = new SynchronizationException(string.Format("File {0} is conflicted", FileName)),
            });
        }
Ejemplo n.º 33
0
 public abstract Task<SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination);
Ejemplo n.º 34
0
 public abstract Task <SynchronizationReport> PerformAsync(IAsyncFilesSynchronizationCommands destination);
Ejemplo n.º 35
0
        protected async Task<SynchronizationReport> ApplyConflictOnDestinationAsync(ConflictItem conflict, RavenJObject remoteMetadata, IAsyncFilesSynchronizationCommands destination, string localServerUrl, ILog log)
		{
            var commands = (IAsyncFilesCommandsImpl)destination.Commands;

            log.Debug("File '{0}' is in conflict with destination version from {1}. Applying conflict on destination", FileName, commands.UrlFor());

			try
			{
				var version = conflict.RemoteHistory.Last().Version;
				var serverId = conflict.RemoteHistory.Last().ServerId;
				var history = new List<HistoryItem>(conflict.RemoteHistory);
				history.RemoveAt(conflict.RemoteHistory.Count - 1);

                await destination.ApplyConflictAsync(FileName, version, serverId, remoteMetadata, localServerUrl);
			}
			catch (Exception ex)
			{
				log.WarnException(string.Format("Failed to apply conflict on {0} for file '{1}'", destination, FileName), ex);
			}

			return new SynchronizationReport(FileName, FileETag, SynchronizationType)
			{
				Exception = new SynchronizationException(string.Format("File {0} is conflicted", FileName)),
			};
		}