public void SynchronizingFileStream_should_write_to_storage_by_64kB_pages() { using (var stream = SynchronizingFileStream.CreatingOrOpeningAndWriting( transactionalStorage, new MockIndexStorage(), new StorageOperationsTask(transactionalStorage, new MockIndexStorage(), new EmptyNotificationsPublisher()), "file", EmptyETagMetadata)) { var buffer = new byte[StorageConstants.MaxPageSize]; new Random().NextBytes(buffer); stream.Write(buffer, 0, 32768); stream.Write(buffer, 32767, 32768); stream.Write(buffer, 0, 1); stream.PreventUploadComplete = false; } FileAndPagesInformation fileAndPages = null; transactionalStorage.Batch(accessor => fileAndPages = accessor.GetFile("file", 0, 10)); Assert.Equal(2, fileAndPages.Pages.Count); Assert.Equal(StorageConstants.MaxPageSize, fileAndPages.Pages[0].Size); Assert.Equal(1, fileAndPages.Pages[1].Size); }
public void SynchronizingFileStream_should_write_to_storage_by_64kB_pages() { using (var stream = SynchronizingFileStream.CreatingOrOpeningAndWriting(fs, "file", new RavenJObject())) { var buffer = new byte[StorageConstants.MaxPageSize]; new Random().NextBytes(buffer); stream.Write(buffer, 0, 32768); stream.Write(buffer, 32767, 32768); stream.Write(buffer, 0, 1); stream.PreventUploadComplete = false; } FileAndPagesInformation fileAndPages = null; transactionalStorage.Batch(accessor => fileAndPages = accessor.GetFile("file", 0, 10)); Assert.Equal(2, fileAndPages.Pages.Count); Assert.Equal(StorageConstants.MaxPageSize, fileAndPages.Pages[0].Size); Assert.Equal(1, fileAndPages.Pages[1].Size); }
public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName) { if (!Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault(); var canonicalFilename = FileHeader.Canonize(fileName); var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate); Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo); StorageStream localFile = null; var isNewFile = false; var isConflictResolved = false; try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders); var localMetadata = GetLocalMetadata(canonicalFilename); if (localMetadata != null) { AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); localFile = StorageStream.Reading(Storage, canonicalFilename); } else { isNewFile = true; } Historian.UpdateLastModified(sourceMetadata); var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata); var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile); Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName); await Request.Content.ReadAsMultipartAsync(provider); Log.Debug("Multipart content of a file '{0}' was processed/read", fileName); report.BytesCopied = provider.BytesCopied; report.BytesTransfered = provider.BytesTransfered; report.NeedListLength = provider.NumberOfFileParts; synchronizingFile.PreventUploadComplete = false; synchronizingFile.Flush(); synchronizingFile.Dispose(); sourceMetadata["Content-MD5"] = synchronizingFile.FileHash; Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata)); Storage.Batch(accessor => { StorageOperationsTask.IndicateFileToDelete(canonicalFilename); accessor.RenameFile(tempFileName, canonicalFilename); Search.Delete(tempFileName); Search.Index(canonicalFilename, sourceMetadata); }); if (isNewFile) { Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName); } else { Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName); } if (isConflictResolved) { ConflictArtifactManager.Delete(canonicalFilename); } } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; } } finally { if (localFile != null) { localFile.Dispose(); } } if (report.Exception == null) { Log.Debug( "File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}", fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength); } else { Log.WarnException( string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo), report.Exception); } FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update); PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (isConflictResolved) { Publisher.Publish(new ConflictNotification { FileName = fileName, Status = ConflictStatus.Resolved }); } return(GetMessageWithObject(report)); }
private async Task ExecuteContentUpdate(RavenJObject localMetadata, SynchronizationReport report) { var tempFileName = RavenFileNameHelper.DownloadingFileName(fileName); using (var localFile = localMetadata != null ? StorageStream.Reading(fs.Storage, fileName) : null) { fs.PutTriggers.Apply(trigger => trigger.OnPut(tempFileName, sourceMetadata)); fs.Historian.UpdateLastModified(sourceMetadata); var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(fs, tempFileName, sourceMetadata); fs.PutTriggers.Apply(trigger => trigger.AfterPut(tempFileName, null, sourceMetadata)); var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile); if (Log.IsDebugEnabled) { Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName); } await MultipartContent.ReadAsMultipartAsync(provider).ConfigureAwait(false); if (Log.IsDebugEnabled) { Log.Debug("Multipart content of a file '{0}' was processed/read", fileName); } report.BytesCopied = provider.BytesCopied; report.BytesTransfered = provider.BytesTransfered; report.NeedListLength = provider.NumberOfFileParts; synchronizingFile.PreventUploadComplete = false; synchronizingFile.Flush(); synchronizingFile.Dispose(); sourceMetadata["Content-MD5"] = synchronizingFile.FileHash; FileUpdateResult updateResult = null; fs.Storage.Batch(accessor => updateResult = accessor.UpdateFileMetadata(tempFileName, sourceMetadata, null)); fs.Storage.Batch(accessor => { using (fs.DisableAllTriggersForCurrentThread()) { fs.Files.IndicateFileToDelete(fileName, null); } accessor.RenameFile(tempFileName, fileName); fs.Search.Delete(tempFileName); fs.Search.Index(fileName, sourceMetadata, updateResult.Etag); }); if (Log.IsDebugEnabled) { var message = localFile == null ? string.Format("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName) : string.Format("Old file '{0}' was deleted. Indexes were updated.", fileName); Log.Debug(message); } fs.Publisher.Publish(new FileChangeNotification { File = fileName, Action = localFile == null ? FileChangeAction.Add : FileChangeAction.Update }); } }