예제 #1
0
        public void Should_create_apropriate_config_after_indicating_file_to_delete()
        {
            var filename = FileHeader.Canonize("toDelete.bin");

            var client = NewAsyncClient();
            var rfs    = GetFileSystem();

            client.UploadAsync(filename, new MemoryStream(new byte[] { 1, 2, 3, 4, 5 })).Wait();

            rfs.Files.IndicateFileToDelete(filename, null);

            DeleteFileOperation deleteFile = null;

            rfs.Storage.Batch(accessor =>
                              deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(filename))));

            Assert.Equal(RavenFileNameHelper.DeletingFileName(filename), deleteFile.CurrentFileName);
            Assert.Equal(filename, deleteFile.OriginalFileName);
        }
예제 #2
0
        public void Should_reuse_second_page_if_only_first_one_changed()
        {
            string filename = FileHeader.Canonize("test");

            var file = SyncTestUtils.PreparePagesStream(2);

            file.Position = 0;

            var sourceContent = new MemoryStream();

            file.CopyTo(sourceContent);
            sourceContent.Position = 0;
            sourceContent.Write(new byte[] { 0, 0, 0, 0 }, 0, 4);           // change content of the 1st page

            var destinationContent = file;

            sourceContent.Position = 0;
            source.UploadAsync(filename, sourceContent).Wait();
            destinationContent.Position = 0;
            destination.UploadAsync(filename, destinationContent).Wait();

            var contentUpdate = new ContentUpdateWorkItem(filename, "http://localhost:12345", sourceRfs.Storage,
                                                          sourceRfs.SigGenerator);


            sourceContent.Position = 0;
            // force to upload entire file, we just want to check which pages will be reused
            contentUpdate.UploadToAsync(destination.Synchronization).Wait();
            destination.Synchronization.ResolveConflictAsync(filename, ConflictResolutionStrategy.RemoteVersion).Wait();
            contentUpdate.UploadToAsync(destination.Synchronization).Wait();

            FileAndPagesInformation fileAndPages = null;

            destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile(filename, 0, 256));

            Assert.Equal(2, fileAndPages.Pages.Count);
            Assert.Equal(3, fileAndPages.Pages[0].Id);             // new page -> id == 3
            Assert.Equal(2, fileAndPages.Pages[1].Id);             // reused page -> id still == 2

            sourceContent.Position = 0;
            Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync(filename).Result["Content-MD5"]);
        }
예제 #3
0
        private void StreamExportToClient(Stream stream, string[] fileNames)
        {
            using (var bufferedStream = new BufferedStream(stream))
            {
                var binaryWriter = new BinaryWriter(bufferedStream);

                var buffer     = new byte[StorageConstants.MaxPageSize];
                var pageBuffer = new byte[StorageConstants.MaxPageSize];

                foreach (var name in fileNames)
                {
                    FileAndPagesInformation fileAndPages = null;
                    var cannonizedName = FileHeader.Canonize(name);

                    Storage.Batch(accessor => fileAndPages = accessor.GetFile(cannonizedName, 0, 0));


                    // if we didn't find the document, we'll write "-1" to the sourceStream, signaling that
                    if (fileAndPages.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
                    {
                        if (log.IsDebugEnabled)
                        {
                            log.Debug("File '{0}' is not accessible to get (Raven-Delete-Marker set)", name);
                        }

                        binaryWriter.Write(-1);
                        continue;
                    }

                    var fileSize = fileAndPages.UploadedSize;
                    binaryWriter.Write(fileSize);

                    var readingStream = StorageStream.Reading(Storage, cannonizedName);
                    var bytesRead     = 0;
                    do
                    {
                        bytesRead = readingStream.ReadUsingExternalTempBuffer(buffer, 0, buffer.Length, pageBuffer);
                        bufferedStream.Write(buffer, 0, bytesRead);
                    } while (bytesRead > 0);
                }
            }
        }
예제 #4
0
        public HttpResponseMessage Post(string name)
        {
            name = FileHeader.Canonize(name);

            var metadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
            var etag     = GetEtag();

            Storage.Batch(accessor =>
            {
                Synchronizations.AssertFileIsNotBeingSynced(name);

                Historian.Update(name, metadata);
                Files.UpdateMetadata(name, metadata, etag);

                Synchronizations.StartSynchronizeDestinationsInBackground();
            });

            //Hack needed by jquery on the client side. We need to find a better solution for this
            return(GetEmptyMessage(HttpStatusCode.NoContent));
        }
예제 #5
0
        public async Task <HttpResponseMessage> Delete(string fileName)
        {
            fileName = FileHeader.Canonize(fileName);

            var sourceInfo     = GetSourceFileSystemInfo();
            var sourceFileETag = GetEtag();
            var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);

            Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceInfo);

            var report = await new SynchronizationBehavior(fileName, sourceFileETag, sourceMetadata, sourceInfo, SynchronizationType.Delete, FileSystem)
                         .Execute();

            if (report.Exception == null)
            {
                Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceInfo);
            }

            return(GetMessageWithObject(report));
        }
예제 #6
0
        public async Task <HttpResponseMessage> Confirm()
        {
            var contentStream = await Request.Content.ReadAsStreamAsync();

            var confirmingFiles = new JsonSerializer().Deserialize <IEnumerable <Tuple <string, Guid> > >(new JsonTextReader(new StreamReader(contentStream)));


            var result = confirmingFiles.Select(x =>
            {
                string canonicalFilename = FileHeader.Canonize(x.Item1);
                return(new SynchronizationConfirmation
                {
                    FileName = canonicalFilename,
                    Status = CheckSynchronizedFileStatus(canonicalFilename, x.Item2)
                });
            });

            return(this.GetMessageWithObject(result)
                   .WithNoCache());
        }
예제 #7
0
        public async Task Resolve_with_local_should_create_marker_in_conflict_item()
        {
            var server1 = NewAsyncClient(0);
            var server2 = NewAsyncClient(1);

            await server1.UploadAsync("test", new MemoryStream(new byte[] { 1, 2, 3 }));

            await server2.UploadAsync("test", new MemoryStream(new byte[] { 1, 2 }));

            var shouldBeConflict = await server1.Synchronization.StartAsync("test", server2);

            Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test")), shouldBeConflict.Exception.Message);

            await server2.Synchronization.ResolveConflictAsync("test", ConflictResolutionStrategy.RemoteVersion);

            var conflicts = await server2.Synchronization.GetConflictsAsync(0, 5);

            Assert.Equal(1, conflicts.Items.Count);
            Assert.True(conflicts.Items.First().ResolveUsingRemote);
        }
예제 #8
0
        public async void Should_reuse_pages_when_data_appended(int numberOfPages)
        {
            string filename = FileHeader.Canonize("test");

            var file = SyncTestUtils.PreparePagesStream(numberOfPages);

            var sourceContent = new CombinedStream(file, SyncTestUtils.PreparePagesStream(numberOfPages));
            // add new pages at the end
            var destinationContent = file;

            sourceContent.Position = 0;
            await source.UploadAsync(filename, sourceContent);

            destinationContent.Position = 0;
            await destination.UploadAsync(filename, destinationContent);

            var contentUpdate = new ContentUpdateWorkItem(filename, "http://localhost:12345", sourceRfs.Storage, sourceRfs.SigGenerator);

            // force to upload entire file, we just want to check which pages will be reused
            await contentUpdate.UploadToAsync(destination.Synchronization);

            await destination.Synchronization.ResolveConflictAsync(filename, ConflictResolutionStrategy.RemoteVersion);

            await contentUpdate.UploadToAsync(destination.Synchronization);


            FileAndPagesInformation fileAndPages = null;

            destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile(filename, 0, 2 * numberOfPages));

            Assert.Equal(2 * numberOfPages, fileAndPages.Pages.Count);

            for (var i = 0; i < numberOfPages; i++)
            {
                Assert.Equal(i + 1, fileAndPages.Pages[i].Id);
                // if page ids are in the original order it means that they were used the existing pages
            }

            sourceContent.Position = 0;
            Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync(filename).Result["Content-MD5"]);
        }
예제 #9
0
        public async void Should_detect_conflict_on_destination()
        {
            var destination = (IAsyncFilesCommandsImpl)NewAsyncClient(1);

            const string fileName = "test.txt";

            await destination.UploadAsync(fileName, new MemoryStream(new byte[] { 1 }));

            var request = (HttpWebRequest)WebRequest.Create(destination.ServerUrl + "/fs/" + destination.FileSystem + "/synchronization/updatemetadata/" + fileName);

            request.Method        = "POST";
            request.ContentLength = 0;

            var conflictedMetadata = new RavenJObject
            {
                { SynchronizationConstants.RavenSynchronizationVersion, new RavenJValue(1) },
                { SynchronizationConstants.RavenSynchronizationSource, new RavenJValue(Guid.Empty) },
                { SynchronizationConstants.RavenSynchronizationHistory, "[]" }
            }
            .WithETag(Guid.Empty);

            request.AddHeaders(conflictedMetadata);

            request.Headers[SyncingMultipartConstants.SourceServerInfo] = new ServerInfo {
                Id = Guid.Empty, FileSystemUrl = "http://localhost:12345"
            }.AsJson();

            var response = await request.GetResponseAsync();

            using (var stream = response.GetResponseStream())
            {
                Assert.NotNull(stream);
                if (stream == null)
                {
                    return;
                }

                var report = new JsonSerializer().Deserialize <SynchronizationReport>(new JsonTextReader(new StreamReader(stream)));
                Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test.txt")), report.Exception.Message);
            }
        }
예제 #10
0
        public async void Conflict_item_should_have_remote_server_url()
        {
            var source      = (IAsyncFilesCommandsImpl)NewAsyncClient(0);
            var destination = NewAsyncClient(1);

            await source.UploadAsync("test", new MemoryStream(new byte[] { 1, 2, 3 }));

            await destination.UploadAsync("test", new MemoryStream(new byte[] { 1, 2 }));

            var shouldBeConflict = await source.Synchronization.StartAsync("test", destination);

            Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test")), shouldBeConflict.Exception.Message);

            var pages = await destination.Synchronization.GetConflictsAsync();

            var remoteServerUrl = pages.Items[0].RemoteServerUrl;

            Assert.NotNull(remoteServerUrl);

            Assert.Equal(new Uri(source.ServerUrl).Port, new Uri(remoteServerUrl).Port);
        }
예제 #11
0
        public void Delete_conflicted_document_should_delete_conflict_items_as_well()
        {
            var source      = NewAsyncClient(0);
            var destination = NewAsyncClient(1);

            source.UploadAsync("test", new MemoryStream(new byte[] { 1, 2, 3 })).Wait();
            destination.UploadAsync("test", new MemoryStream(new byte[] { 1, 2 })).Wait();

            var shouldBeConflict = source.Synchronization.StartAsync("test", destination).Result;

            Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test")), shouldBeConflict.Exception.Message);

            var pages = destination.Synchronization.GetConflictsAsync().Result;

            Assert.Equal(1, pages.TotalCount);

            destination.DeleteAsync("test").Wait();

            pages = destination.Synchronization.GetConflictsAsync().Result;
            Assert.Equal(0, pages.TotalCount);
        }
예제 #12
0
        public async Task Source_should_save_configuration_record_after_synchronization()
        {
            var sourceClient  = NewAsyncClient(0);
            var sourceContent = new RandomStream(10000);

            var destinationClient = NewAsyncClient(1);

            await sourceClient.UploadAsync("test.bin", sourceContent);

            await sourceClient.Synchronization.SetDestinationsAsync(destinationClient.ToSynchronizationDestination());

            await sourceClient.Synchronization.SynchronizeAsync();

            var fullDstUrl = destinationClient.ToSynchronizationDestination().Url;

            var synchronizationDetails = sourceClient.Configuration.GetKeyAsync <SynchronizationDetails>(RavenFileNameHelper.SyncNameForFile("test.bin", fullDstUrl)).Result;

            Assert.Equal(FileHeader.Canonize("test.bin"), synchronizationDetails.FileName);
            Assert.Equal(fullDstUrl, synchronizationDetails.DestinationUrl);
            Assert.NotEqual(Guid.Empty, synchronizationDetails.FileETag);
            Assert.Equal(SynchronizationType.ContentUpdate, synchronizationDetails.Type);
        }
예제 #13
0
        public async Task <HttpResponseMessage> MultipartProceed()
        {
            if (!Request.Content.IsMimeMultipartContent())
            {
                throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
            }

            var fileName = FileHeader.Canonize(Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault());

            var sourceInfo     = GetSourceFileSystemInfo();
            var sourceFileETag = GetEtag();
            var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);

            if (Log.IsDebugEnabled)
            {
                Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceInfo);
            }

            var report = await new SynchronizationBehavior(fileName, sourceFileETag, sourceMetadata, sourceInfo, SynchronizationType.ContentUpdate, FileSystem)
            {
                MultipartContent = Request.Content
            }.Execute().ConfigureAwait(false);

            if (report.Exception == null)
            {
                if (Log.IsDebugEnabled)
                {
                    Log.Debug(
                        "File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
                        fileName, sourceInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
                }
            }
            else
            {
                Log.WarnException(string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceInfo), report.Exception);
            }

            return(GetMessageWithObject(report));
        }
예제 #14
0
        public void Should_resume_to_rename_file_if_appropriate_config_exists()
        {
            var client = NewAsyncClient();
            var rfs    = GetFileSystem();

            string fileName = FileHeader.Canonize("file.bin");
            string rename   = FileHeader.Canonize("renamed.bin");

            client.UploadAsync(fileName, new RandomStream(1)).Wait();

            // create config to say to the server that rename operation performed last time were not finished
            var renameOpConfig  = RavenFileNameHelper.RenameOperationConfigNameForFile(fileName);
            var renameOperation = new RenameFileOperation
            {
                Name   = fileName,
                Rename = rename,
                MetadataAfterOperation = new RavenJObject().WithETag(Guid.Empty)
            };

            rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(renameOpConfig, renameOperation));

            rfs.StorageOperationsTask.ResumeFileRenamingAsync().Wait();

            IEnumerable <string> configNames = null;

            rfs.Storage.Batch(accessor => configNames = accessor.GetConfigNames(0, 10).ToArray());

            Assert.DoesNotContain(renameOpConfig, configNames);

            var renamedMetadata = client.GetMetadataForAsync(rename).Result;

            Assert.NotNull(renamedMetadata);

            var results = client.SearchOnDirectoryAsync("/").Result;             // make sure that indexes are updated

            Assert.Equal(1, results.FileCount);
            Assert.Equal(rename, results.Files[0].FullPath);
        }
예제 #15
0
        public HttpResponseMessage Head(string name)
        {
            name = FileHeader.Canonize(name);
            FileAndPagesInformation fileAndPages = null;

            Storage.Batch(accessor => fileAndPages = accessor.GetFile(name, 0, 0));

            if (fileAndPages.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
            {
                log.Debug("Cannot get metadata of a file '{0}' because file was deleted", name);
                throw new FileNotFoundException();
            }

            var httpResponseMessage = GetEmptyMessage();

            var etag = new Etag(fileAndPages.Metadata.Value <string>(Constants.MetadataEtagField));

            fileAndPages.Metadata.Remove(Constants.MetadataEtagField);

            WriteHeaders(fileAndPages.Metadata, etag, httpResponseMessage);

            return(httpResponseMessage);
        }
예제 #16
0
        public async Task <HttpResponseMessage> UpdateMetadata(string fileName)
        {
            fileName = FileHeader.Canonize(fileName);

            var sourceInfo     = GetSourceFileSystemInfo();
            var sourceFileETag = GetEtag();
            var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);

            if (Log.IsDebugEnabled)
            {
                Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceInfo);
            }

            var report = await new SynchronizationBehavior(fileName, sourceFileETag, sourceMetadata, sourceInfo, SynchronizationType.MetadataUpdate, FileSystem)
                         .Execute().ConfigureAwait(false);

            if (Log.IsDebugEnabled && report.Exception == null)
            {
                Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceInfo);
            }

            return(GetMessageWithObject(report));
        }
예제 #17
0
        public void Should_detect_conflict_on_renaming_synchronization()
        {
            var content = new MemoryStream(new byte[] { 1, 2, 3, 4 });

            var sourceClient      = NewAsyncClient(0);
            var destinationClient = NewAsyncClient(1);

            sourceClient.UploadAsync("test.bin", content, new RavenJObject {
                { "key", "value" }
            }).Wait();
            content.Position = 0;
            destinationClient.UploadAsync("test.bin", content, new RavenJObject {
                { "key", "value" }
            }).Wait();

            sourceClient.RenameAsync("test.bin", "renamed.bin").Wait();

            // we need to indicate old file name, otherwise content update would be performed because renamed file does not exist on dest
            var report = sourceClient.Synchronization.StartAsync("test.bin", destinationClient).Result;

            Assert.Equal(SynchronizationType.Rename, report.Type);
            Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test.bin")), report.Exception.Message);
        }
예제 #18
0
        public async Task <HttpResponseMessage> Manifest(string id)
        {
            var canonicalFilename = FileHeader.Canonize(id);

            FileAndPagesInformation fileAndPages = null;

            try
            {
                Storage.Batch(accessor => fileAndPages = accessor.GetFile(canonicalFilename, 0, 0));
            }
            catch (FileNotFoundException)
            {
                Log.Debug("Signature manifest for a file '{0}' was not found", id);
                return(Request.CreateResponse(HttpStatusCode.NotFound));
            }

            long?fileLength = fileAndPages.TotalSize;

            using (var signatureRepository = new StorageSignatureRepository(Storage, canonicalFilename))
            {
                var rdcManager        = new LocalRdcManager(signatureRepository, Storage, SigGenerator);
                var signatureManifest = await rdcManager.GetSignatureManifestAsync(
                    new DataInfo
                {
                    Name         = canonicalFilename,
                    LastModified = fileAndPages.Metadata.Value <DateTime>(Constants.LastModified)
                                   .ToUniversalTime()
                });

                signatureManifest.FileLength = fileLength ?? 0;

                Log.Debug("Signature manifest for a file '{0}' was downloaded. Signatures count was {1}", id, signatureManifest.Signatures.Count);

                return(GetMessageWithObject(signatureManifest)
                       .WithNoCache());
            }
        }
예제 #19
0
        public HttpResponseMessage Post(string name)
        {
            name = FileHeader.Canonize(name);

            var headers = this.GetFilteredMetadataFromHeaders(InnerHeaders);

            Historian.UpdateLastModified(headers);
            Historian.Update(name, headers);

            try
            {
                ConcurrencyAwareExecutor.Execute(() =>
                                                 Storage.Batch(accessor =>
                {
                    AssertFileIsNotBeingSynced(name, accessor, true);
                    accessor.UpdateFileMetadata(name, headers);
                }), ConcurrencyResponseException);
            }
            catch (FileNotFoundException)
            {
                log.Debug("Cannot update metadata because file '{0}' was not found", name);
                return(GetEmptyMessage(HttpStatusCode.NotFound));
            }

            Search.Index(name, headers);

            Publisher.Publish(new FileChangeNotification {
                File = FilePathTools.Cannoicalise(name), Action = FileChangeAction.Update
            });

            StartSynchronizeDestinationsInBackground();

            log.Debug("Metadata of a file '{0}' was updated", name);

            //Hack needed by jquery on the client side. We need to find a better solution for this
            return(GetEmptyMessage(HttpStatusCode.NoContent));
        }
예제 #20
0
        public async Task Should_report_that_file_is_broken_if_last_synchronization_set_exception()
        {
            var destinationClient = NewAsyncClient(0);

            var sampleGuid = Guid.NewGuid();

            var failureSynchronization = new SynchronizationReport("test.bin", sampleGuid, SynchronizationType.Unknown)
            {
                Exception = new Exception("There was an exception in last synchronization.")
            };

            await destinationClient.Configuration.SetKeyAsync(RavenFileNameHelper.SyncResultNameForFile("test.bin"), failureSynchronization);

            var confirmations = await destinationClient.Synchronization
                                .GetConfirmationForFilesAsync(new List <Tuple <string, Guid> >
            {
                new Tuple <string, Guid>(
                    "test.bin", sampleGuid)
            });

            Assert.Equal(1, confirmations.Count());
            Assert.Equal(FileStatus.Broken, confirmations[0].Status);
            Assert.Equal(FileHeader.Canonize("test.bin"), confirmations[0].FileName);
        }
예제 #21
0
        public HttpResponseMessage Delete(string name)
        {
            name = FileHeader.Canonize(name);

            Storage.Batch(accessor =>
            {
                Synchronizations.AssertFileIsNotBeingSynced(name);

                var fileAndPages = accessor.GetFile(name, 0, 0);

                var metadata = fileAndPages.Metadata;

                if (metadata == null)
                {
                    throw new FileNotFoundException();
                }

                if (metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
                {
                    throw new FileNotFoundException();
                }

                Historian.Update(name, metadata);
                Files.IndicateFileToDelete(name, GetEtag());

                if (name.EndsWith(RavenFileNameHelper.DownloadingFileSuffix) == false) // don't create a tombstone for .downloading file
                {
                    Files.PutTombstone(name, metadata);
                    accessor.DeleteConfig(RavenFileNameHelper.ConflictConfigNameForFile(name)); // delete conflict item too
                }
            });

            SynchronizationTask.Context.NotifyAboutWork();

            return(GetEmptyMessage(HttpStatusCode.NoContent));
        }
예제 #22
0
        public async Task <HttpResponseMessage> Rename(string fileName, string rename)
        {
            fileName = FileHeader.Canonize(fileName);
            rename   = FileHeader.Canonize(rename);

            var sourceInfo     = GetSourceFileSystemInfo();
            var sourceFileEtag = GetEtag();
            var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);

            Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName,
                      rename, sourceFileEtag, sourceInfo);

            var report = await new SynchronizationBehavior(fileName, sourceFileEtag, sourceMetadata, sourceInfo, SynchronizationType.Rename, FileSystem)
            {
                Rename = rename
            }.Execute().ConfigureAwait(false);

            if (report.Exception == null)
            {
                Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceInfo);
            }

            return(GetMessageWithObject(report));
        }
예제 #23
0
        public void Can_resolve_all_conflicts()
        {
            var sourceClient      = NewAsyncClient(0);
            var destinationClient = NewAsyncClient(1);

            for (var i = 0; i < 10; i++)
            {
                sourceClient.UploadAsync("test" + i, new MemoryStream(new byte[] { 1, 2, 3 })).Wait();
                destinationClient.UploadAsync("test" + i, new MemoryStream(new byte[] { 1, 2 })).Wait();
            }

            for (var i = 0; i < 10; i++)
            {
                var shouldBeConflict = sourceClient.Synchronization.StartAsync("test" + i, destinationClient).Result;
                Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test" + i)), shouldBeConflict.Exception.Message);
            }

            destinationClient.Synchronization.ResolveConflictsAsync(ConflictResolutionStrategy.CurrentVersion).Wait();

            var conflicts = destinationClient.Synchronization.GetConflictsAsync(0, 100).Result;

            Assert.Equal(0, conflicts.TotalCount);
            Assert.Equal(0, conflicts.Items.Count);
        }
예제 #24
0
        public async Task Should_refuse_to_synchronize_if_limit_of_concurrent_synchronizations_exceeded()
        {
            var sourceContent     = new RandomStream(1);
            var sourceClient      = NewAsyncClient(0);
            var destinationClient = (IAsyncFilesCommandsImpl)NewAsyncClient(1);

            await sourceClient.Configuration.SetKeyAsync(SynchronizationConstants.RavenSynchronizationLimit, -1);

            await sourceClient.UploadAsync("test.bin", sourceContent);

            var synchronizationReport = await sourceClient.Synchronization.StartAsync("test.bin", destinationClient);

            Assert.Contains("The limit of active synchronizations to " + destinationClient.ServerUrl, synchronizationReport.Exception.Message);
            Assert.Contains(string.Format("server has been achieved. Cannot process a file '{0}'.", FileHeader.Canonize("test.bin")), synchronizationReport.Exception.Message);
        }
예제 #25
0
        public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName)
        {
            if (!Request.Content.IsMimeMultipartContent())
            {
                throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
            }

            var fileName          = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault();
            var canonicalFilename = FileHeader.Canonize(fileName);

            var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename);

            var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
            var sourceFileETag   = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));

            var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate);

            Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo);

            StorageStream localFile          = null;
            var           isNewFile          = false;
            var           isConflictResolved = false;

            try
            {
                Storage.Batch(accessor =>
                {
                    AssertFileIsNotBeingSynced(canonicalFilename, accessor);
                    FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
                });

                SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate);

                PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);

                Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));

                RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);

                var localMetadata = GetLocalMetadata(canonicalFilename);
                if (localMetadata != null)
                {
                    AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
                    localFile = StorageStream.Reading(Storage, canonicalFilename);
                }
                else
                {
                    isNewFile = true;
                }

                Historian.UpdateLastModified(sourceMetadata);

                var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata);

                var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);

                Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName);

                await Request.Content.ReadAsMultipartAsync(provider);

                Log.Debug("Multipart content of a file '{0}' was processed/read", fileName);

                report.BytesCopied     = provider.BytesCopied;
                report.BytesTransfered = provider.BytesTransfered;
                report.NeedListLength  = provider.NumberOfFileParts;

                synchronizingFile.PreventUploadComplete = false;
                synchronizingFile.Flush();
                synchronizingFile.Dispose();
                sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;

                Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata));

                Storage.Batch(accessor =>
                {
                    StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
                    accessor.RenameFile(tempFileName, canonicalFilename);

                    Search.Delete(tempFileName);
                    Search.Index(canonicalFilename, sourceMetadata);
                });

                if (isNewFile)
                {
                    Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName);
                }
                else
                {
                    Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName);
                }

                if (isConflictResolved)
                {
                    ConflictArtifactManager.Delete(canonicalFilename);
                }
            }
            catch (Exception ex)
            {
                if (ShouldAddExceptionToReport(ex))
                {
                    report.Exception = ex;
                }
            }
            finally
            {
                if (localFile != null)
                {
                    localFile.Dispose();
                }
            }

            if (report.Exception == null)
            {
                Log.Debug(
                    "File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
                    fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
            }
            else
            {
                Log.WarnException(
                    string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo),
                    report.Exception);
            }

            FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);

            PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update);
            PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);

            if (isConflictResolved)
            {
                Publisher.Publish(new ConflictNotification
                {
                    FileName = fileName,
                    Status   = ConflictStatus.Resolved
                });
            }

            return(GetMessageWithObject(report));
        }
예제 #26
0
        public HttpResponseMessage Rename(string fileSystemName, string fileName, string rename)
        {
            bool isConflictResolved = false;

            var canonicalFilename = FileHeader.Canonize(fileName);
            var canonicalRename   = FileHeader.Canonize(rename);

            var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
            var sourceFileETag   = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
            var sourceMetadata   = GetFilteredMetadataFromHeaders(ReadInnerHeaders);

            Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName,
                      rename, sourceFileETag, sourceServerInfo);

            var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Rename);

            try
            {
                Storage.Batch(accessor =>
                {
                    AssertFileIsNotBeingSynced(canonicalFilename, accessor);
                    FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
                });

                SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Rename);

                PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);

                Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));

                var localMetadata = GetLocalMetadata(canonicalFilename);

                AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);

                if (isConflictResolved)
                {
                    ConflictArtifactManager.Delete(canonicalFilename);
                }

                StorageOperationsTask.RenameFile(new RenameFileOperation
                {
                    FileSystem             = FileSystem.Name,
                    Name                   = canonicalFilename,
                    Rename                 = canonicalRename,
                    MetadataAfterOperation = sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers()
                });
            }
            catch (Exception ex)
            {
                if (ShouldAddExceptionToReport(ex))
                {
                    report.Exception = ex;
                    Log.WarnException(string.Format("Error was occurred during renaming synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
                }
            }
            finally
            {
                FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
            }

            PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Finish);

            if (isConflictResolved)
            {
                Publisher.Publish(new ConflictNotification
                {
                    FileName = fileName,
                    Status   = ConflictStatus.Resolved
                });
            }

            if (report.Exception == null)
            {
                Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo);
            }

            return(GetMessageWithObject(report));
        }
예제 #27
0
        public HttpResponseMessage Delete(string fileSystemName, string fileName)
        {
            var canonicalFilename = FileHeader.Canonize(fileName);

            var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
            var sourceFileETag   = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));

            Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceServerInfo);

            var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Delete);

            try
            {
                Storage.Batch(accessor =>
                {
                    AssertFileIsNotBeingSynced(canonicalFilename, accessor);
                    FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
                });


                SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Delete);

                PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);

                Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));

                var localMetadata = GetLocalMetadata(canonicalFilename);

                if (localMetadata != null)
                {
                    // REVIEW: Use InnerHeaders for consistency?
                    var sourceMetadata = GetFilteredMetadataFromHeaders(Request.Headers); // Request.Headers.FilterHeadersToObject();

                    bool isConflictResolved;

                    AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);

                    Storage.Batch(accessor =>
                    {
                        StorageOperationsTask.IndicateFileToDelete(canonicalFilename);

                        var tombstoneMetadata = new RavenJObject
                        {
                            {
                                SynchronizationConstants.RavenSynchronizationHistory,
                                localMetadata[SynchronizationConstants.RavenSynchronizationHistory]
                            },
                            {
                                SynchronizationConstants.RavenSynchronizationVersion,
                                localMetadata[SynchronizationConstants.RavenSynchronizationVersion]
                            },
                            {
                                SynchronizationConstants.RavenSynchronizationSource,
                                localMetadata[SynchronizationConstants.RavenSynchronizationSource]
                            }
                        }.WithDeleteMarker();

                        Historian.UpdateLastModified(tombstoneMetadata);
                        accessor.PutFile(canonicalFilename, 0, tombstoneMetadata, true);
                    });

                    PublishFileNotification(fileName, FileChangeAction.Delete);
                }
            }
            catch (Exception ex)
            {
                if (ShouldAddExceptionToReport(ex))
                {
                    report.Exception = ex;

                    Log.WarnException(string.Format("Error was occurred during deletion synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
                }
            }
            finally
            {
                FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
            }

            PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);

            if (report.Exception == null)
            {
                Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceServerInfo);
            }

            return(this.GetMessageWithObject(report, HttpStatusCode.OK));
        }
예제 #28
0
        public HttpResponseMessage UpdateMetadata(string fileSystemName, string fileName)
        {
            bool isConflictResolved = false;

            var canonicalFilename = FileHeader.Canonize(fileName);

            var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
            // REVIEW: (Oren) It works, but it seems to me it is not an scalable solution.
            var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));

            Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName,
                      sourceFileETag, sourceServerInfo);

            var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.MetadataUpdate);

            try
            {
                Storage.Batch(accessor =>
                {
                    AssertFileIsNotBeingSynced(canonicalFilename, accessor);
                    FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
                });

                SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.MetadataUpdate);

                PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);

                Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));

                var localMetadata  = GetLocalMetadata(canonicalFilename);
                var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);

                AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);

                Historian.UpdateLastModified(sourceMetadata);

                Storage.Batch(accessor => accessor.UpdateFileMetadata(canonicalFilename, sourceMetadata));

                Search.Index(canonicalFilename, sourceMetadata);

                if (isConflictResolved)
                {
                    ConflictArtifactManager.Delete(canonicalFilename);
                }

                PublishFileNotification(fileName, FileChangeAction.Update);
            }
            catch (Exception ex)
            {
                if (ShouldAddExceptionToReport(ex))
                {
                    report.Exception = ex;

                    Log.WarnException(
                        string.Format("Error was occurred during metadata synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
                }
            }
            finally
            {
                FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
            }

            PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);

            if (isConflictResolved)
            {
                Publisher.Publish(new ConflictNotification
                {
                    FileName = fileName,
                    Status   = ConflictStatus.Resolved
                });
            }

            if (report.Exception == null)
            {
                Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceServerInfo);
            }

            return(this.GetMessageWithObject(report, HttpStatusCode.OK));
        }
예제 #29
0
        public HttpResponseMessage Patch(string name, string rename)
        {
            name   = FileHeader.Canonize(name);
            rename = FileHeader.Canonize(rename);
            var etag = GetEtag();

            if (rename.Length > SystemParameters.KeyMost)
            {
                if (Log.IsDebugEnabled)
                {
                    Log.Debug("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename);
                }
                return(GetMessageWithString(string.Format("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename), HttpStatusCode.BadRequest));
            }

            Storage.Batch(accessor =>
            {
                FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);

                var existingFile = accessor.ReadFile(name);
                if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker))
                {
                    throw new FileNotFoundException();
                }

                var renamingFile = accessor.ReadFile(rename);
                if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false)
                {
                    throw new FileExistsException("Cannot rename because file " + rename + " already exists");
                }

                var metadata = existingFile.Metadata;

                if (etag != null && existingFile.Etag != etag)
                {
                    throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag")
                    {
                        ActualETag   = existingFile.Etag,
                        ExpectedETag = etag
                    }
                }
                ;

                Historian.UpdateLastModified(metadata);

                var operation = new RenameFileOperation(name, rename, existingFile.Etag, metadata);

                accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation));
                accessor.PulseTransaction(); // commit rename operation config

                Files.ExecuteRenameOperation(operation);
            });

            if (Log.IsDebugEnabled)
            {
                Log.Debug("File '{0}' was renamed to '{1}'", name, rename);
            }

            SynchronizationTask.Context.NotifyAboutWork();

            return(GetEmptyMessage(HttpStatusCode.NoContent));
        }
예제 #30
0
        public HttpResponseMessage Get([FromUri] string[] fileNames)
        {
            var list = new List <FileHeader>();

            var startsWith = GetQueryStringValue("startsWith");

            if (string.IsNullOrEmpty(startsWith) == false)
            {
                var matches = GetQueryStringValue("matches");

                var endsWithSlash = startsWith.EndsWith("/") || startsWith.EndsWith("\\");
                startsWith = FileHeader.Canonize(startsWith);
                if (endsWithSlash)
                {
                    startsWith += "/";
                }

                Storage.Batch(accessor =>
                {
                    var actualStart             = 0;
                    var filesToSkip             = Paging.Start;
                    int fileCount, matchedFiles = 0, addedFiles = 0;

                    do
                    {
                        fileCount = 0;

                        foreach (var file in accessor.GetFilesStartingWith(startsWith, actualStart, Paging.PageSize))
                        {
                            fileCount++;

                            var keyTest = file.FullPath.Substring(startsWith.Length);

                            if (WildcardMatcher.Matches(matches, keyTest) == false)
                            {
                                continue;
                            }

                            if (FileSystem.ReadTriggers.CanReadFile(file.FullPath, file.Metadata, ReadOperation.Load) == false)
                            {
                                continue;
                            }

                            matchedFiles++;

                            if (matchedFiles <= filesToSkip)
                            {
                                continue;
                            }

                            list.Add(file);
                            addedFiles++;
                        }

                        actualStart += Paging.PageSize;
                    }while (fileCount > 0 && addedFiles < Paging.PageSize && actualStart > 0 && actualStart < int.MaxValue);
                });
            }
            else
            {
                if (fileNames != null && fileNames.Length > 0)
                {
                    Storage.Batch(accessor =>
                    {
                        foreach (var path in fileNames.Where(x => x != null).Select(FileHeader.Canonize))
                        {
                            var file = accessor.ReadFile(path);

                            if (file == null || file.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
                            {
                                list.Add(null);
                                continue;
                            }

                            list.Add(file);
                        }
                    });
                }
                else
                {
                    int  results;
                    long durationInMs;
                    var  keys = Search.Query(null, null, Paging.Start, Paging.PageSize, out results, out durationInMs);

                    Storage.Batch(accessor => list.AddRange(keys.Select(accessor.ReadFile).Where(x => x != null)));
                }
            }

            return(GetMessageWithObject(list)
                   .WithNoCache());
        }