public HttpResponseMessage Delete(string name) { name = RavenFileNameHelper.RavenPath(name); try { ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor => { AssertFileIsNotBeingSynced(name, accessor, true); var fileAndPages = accessor.GetFile(name, 0, 0); var metadata = fileAndPages.Metadata; if (metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker)) { throw new FileNotFoundException(); } StorageOperationsTask.IndicateFileToDelete(name); if (!name.EndsWith(RavenFileNameHelper.DownloadingFileSuffix) && // don't create a tombstone for .downloading file metadata != null) // and if file didn't exist { var tombstoneMetadata = new RavenJObject { { SynchronizationConstants.RavenSynchronizationHistory, metadata[SynchronizationConstants.RavenSynchronizationHistory] }, { SynchronizationConstants.RavenSynchronizationVersion, metadata[SynchronizationConstants.RavenSynchronizationVersion] }, { SynchronizationConstants.RavenSynchronizationSource, metadata[SynchronizationConstants.RavenSynchronizationSource] } }.WithDeleteMarker(); Historian.UpdateLastModified(tombstoneMetadata); accessor.PutFile(name, 0, tombstoneMetadata, true); accessor.DeleteConfig(RavenFileNameHelper.ConflictConfigNameForFile(name)); // delete conflict item too } }), ConcurrencyResponseException); } catch (FileNotFoundException) { return(new HttpResponseMessage(HttpStatusCode.NotFound)); } Publisher.Publish(new FileChangeNotification { File = FilePathTools.Cannoicalise(name), Action = FileChangeAction.Delete }); log.Debug("File '{0}' was deleted", name); StartSynchronizeDestinationsInBackground(); return(GetEmptyMessage(HttpStatusCode.NoContent)); }
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState recievedTransportState = null) { this.Name = name; this.systemConfiguration = systemConfiguration; var storageType = systemConfiguration.FileSystem.DefaultStorageTypeName; storage = CreateTransactionalStorage(storageType, systemConfiguration); search = new IndexStorage(systemConfiguration.FileSystem.IndexStoragePath, systemConfiguration.Settings); sigGenerator = new SigGenerator(); var replicationHiLo = new SynchronizationHiLo(storage); var sequenceActions = new SequenceActions(storage); transportState = recievedTransportState ?? new TransportState(); notificationPublisher = new NotificationPublisher(transportState); fileLockManager = new FileLockManager(); storage.Initialize(); search.Initialize(); var uuidGenerator = new UuidGenerator(sequenceActions); historian = new Historian(storage, replicationHiLo, uuidGenerator); BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024); conflictArtifactManager = new ConflictArtifactManager(storage, search); conflictDetector = new ConflictDetector(); conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog)); synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration); storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher); metricsCounters = new MetricsCountersManager(); AppDomain.CurrentDomain.ProcessExit += ShouldDispose; AppDomain.CurrentDomain.DomainUnload += ShouldDispose; }
private SynchronizingFileStream(ITransactionalStorage transactionalStorage, string fileName, StorageStreamAccess storageStreamAccess, RavenJObject metadata, IndexStorage indexStorage, StorageOperationsTask operations) : base(transactionalStorage, fileName, storageStreamAccess, metadata, indexStorage, operations) { md5Hasher = Encryptor.Current.CreateHash(); }
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState receivedTransportState = null) { ExtensionsState = new AtomicDictionary <object>(); Name = name; this.systemConfiguration = systemConfiguration; systemConfiguration.Container.SatisfyImportsOnce(this); transportState = receivedTransportState ?? new TransportState(); storage = CreateTransactionalStorage(systemConfiguration); sigGenerator = new SigGenerator(); fileLockManager = new FileLockManager(); BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024); conflictDetector = new ConflictDetector(); conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog)); notificationPublisher = new NotificationPublisher(transportState); synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration); metricsCounters = new MetricsCountersManager(); search = new IndexStorage(name, systemConfiguration); conflictArtifactManager = new ConflictArtifactManager(storage, search); storageOperationsTask = new StorageOperationsTask(storage, DeleteTriggers, search, notificationPublisher); AppDomain.CurrentDomain.ProcessExit += ShouldDispose; AppDomain.CurrentDomain.DomainUnload += ShouldDispose; }
protected StorageStream(ITransactionalStorage transactionalStorage, string fileName, StorageStreamAccess storageStreamAccess, RavenJObject metadata, IndexStorage indexStorage, StorageOperationsTask operations) { TransactionalStorage = transactionalStorage; StorageStreamAccess = storageStreamAccess; Name = fileName; switch (storageStreamAccess) { case StorageStreamAccess.Read: TransactionalStorage.Batch(accessor => fileHeader = accessor.ReadFile(fileName)); if (fileHeader.TotalSize == null) { throw new FileNotFoundException("File is not uploaded yet"); } Metadata = fileHeader.Metadata; Seek(0, SeekOrigin.Begin); break; case StorageStreamAccess.CreateAndWrite: TransactionalStorage.Batch(accessor => { operations.IndicateFileToDelete(fileName); accessor.PutFile(fileName, null, metadata); indexStorage.Index(fileName, metadata); }); Metadata = metadata; break; default: throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value"); } }
private void StartupProceed(string fileName, IStorageActionsAccessor accessor) { // remove previous SyncResult DeleteSynchronizationReport(fileName, accessor); // remove previous .downloading file StorageOperationsTask.IndicateFileToDelete(RavenFileNameHelper.DownloadingFileName(fileName)); }
public static SynchronizingFileStream CreatingOrOpeningAndWriting(ITransactionalStorage storage, IndexStorage search, StorageOperationsTask operationsTask, string fileName, RavenJObject metadata) { return(new SynchronizingFileStream(storage, fileName, StorageStreamAccess.CreateAndWrite, metadata, search, operationsTask) { PreventUploadComplete = true }); }
public HttpResponseMessage Patch(string name, string rename) { name = RavenFileNameHelper.RavenPath(name); rename = RavenFileNameHelper.RavenPath(rename); try { ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor => { AssertFileIsNotBeingSynced(name, accessor, true); var metadata = accessor.GetFile(name, 0, 0).Metadata; if (metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker)) { throw new FileNotFoundException(); } var existingHeader = accessor.ReadFile(rename); if (existingHeader != null && !existingHeader.Metadata.ContainsKey(SynchronizationConstants.RavenDeleteMarker)) { throw new HttpResponseException( Request.CreateResponse(HttpStatusCode.Forbidden, new InvalidOperationException("Cannot rename because file " + rename + " already exists"))); } Historian.UpdateLastModified(metadata); var operation = new RenameFileOperation { FileSystem = FileSystem.Name, Name = name, Rename = rename, MetadataAfterOperation = metadata }; accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation)); accessor.PulseTransaction(); // commit rename operation config StorageOperationsTask.RenameFile(operation); }), ConcurrencyResponseException); } catch (FileNotFoundException) { log.Debug("Cannot rename a file '{0}' to '{1}' because a file was not found", name, rename); return(GetEmptyMessage(HttpStatusCode.NotFound)); } log.Debug("File '{0}' was renamed to '{1}'", name, rename); StartSynchronizeDestinationsInBackground(); return(GetMessageWithString("", HttpStatusCode.NoContent)); }
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, TransportState transportState, string name) { this.Name = name; this.systemConfiguration = systemConfiguration; var storageType = systemConfiguration.DefaultFileSystemStorageTypeName; if (string.Equals(InMemoryRavenConfiguration.VoronTypeName, storageType, StringComparison.OrdinalIgnoreCase) == false) { if (Directory.Exists(systemConfiguration.FileSystemDataDirectory) && Directory.EnumerateFileSystemEntries(systemConfiguration.FileSystemDataDirectory).Any()) { throw new InvalidOperationException( string.Format( "We do not allow to run on a storage engine other then Voron, while we are in the early pre-release phase of RavenDB 3.0. You are currently running on {0}", storageType)); } Trace.WriteLine("Forcing filesystem to run on Voron - pre release behavior only, mind " + Path.GetFileName(Path.GetDirectoryName(systemConfiguration.FileSystemDataDirectory))); storageType = InMemoryRavenConfiguration.VoronTypeName; } storage = CreateTransactionalStorage(storageType, systemConfiguration); search = new IndexStorage(systemConfiguration.FileSystemIndexStoragePath, systemConfiguration.Settings); sigGenerator = new SigGenerator(); var replicationHiLo = new SynchronizationHiLo(storage); var sequenceActions = new SequenceActions(storage); this.transportState = transportState; notificationPublisher = new NotificationPublisher(transportState); fileLockManager = new FileLockManager(); storage.Initialize(); search.Initialize(); var uuidGenerator = new UuidGenerator(sequenceActions); historian = new Historian(storage, replicationHiLo, uuidGenerator); BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024); conflictArtifactManager = new ConflictArtifactManager(storage, search); conflictDetector = new ConflictDetector(); conflictResolver = new ConflictResolver(); synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration); storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher); metricsCounters = new MetricsCountersManager(); AppDomain.CurrentDomain.ProcessExit += ShouldDispose; AppDomain.CurrentDomain.DomainUnload += ShouldDispose; }
public HttpResponseMessage Delete(string fileSystemName, string fileName) { var canonicalFilename = FileHeader.Canonize(fileName); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceServerInfo); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Delete); try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Delete); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); var localMetadata = GetLocalMetadata(canonicalFilename); if (localMetadata != null) { // REVIEW: Use InnerHeaders for consistency? var sourceMetadata = GetFilteredMetadataFromHeaders(Request.Headers); // Request.Headers.FilterHeadersToObject(); bool isConflictResolved; AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); Storage.Batch(accessor => { StorageOperationsTask.IndicateFileToDelete(canonicalFilename); var tombstoneMetadata = new RavenJObject { { SynchronizationConstants.RavenSynchronizationHistory, localMetadata[SynchronizationConstants.RavenSynchronizationHistory] }, { SynchronizationConstants.RavenSynchronizationVersion, localMetadata[SynchronizationConstants.RavenSynchronizationVersion] }, { SynchronizationConstants.RavenSynchronizationSource, localMetadata[SynchronizationConstants.RavenSynchronizationSource] } }.WithDeleteMarker(); Historian.UpdateLastModified(tombstoneMetadata); accessor.PutFile(canonicalFilename, 0, tombstoneMetadata, true); }); PublishFileNotification(fileName, FileChangeAction.Delete); } } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; Log.WarnException(string.Format("Error was occurred during deletion synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex); } } finally { FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); } PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (report.Exception == null) { Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceServerInfo); } return(this.GetMessageWithObject(report, HttpStatusCode.OK)); }
public Task RetryRenaming() { return(StorageOperationsTask.ResumeFileRenamingAsync()); }
public Task CleanUp() { return(StorageOperationsTask.CleanupDeletedFilesAsync()); }
public async Task <HttpResponseMessage> Put(string name, string uploadId = null) { try { FileSystem.MetricsCounters.FilesPerSecond.Mark(); name = RavenFileNameHelper.RavenPath(name); var headers = this.GetFilteredMetadataFromHeaders(InnerHeaders); Historian.UpdateLastModified(headers); headers["Creation-Date"] = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture); Historian.Update(name, headers); SynchronizationTask.Cancel(name); long?size = -1; ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor => { AssertFileIsNotBeingSynced(name, accessor, true); StorageOperationsTask.IndicateFileToDelete(name); var contentLength = Request.Content.Headers.ContentLength; var sizeHeader = GetHeader("RavenFS-size"); long sizeForParse; if (contentLength == 0 || long.TryParse(sizeHeader, out sizeForParse) == false) { size = contentLength; if (Request.Headers.TransferEncodingChunked ?? false) { size = null; } } else { size = sizeForParse; } accessor.PutFile(name, size, headers); Search.Index(name, headers); })); log.Debug("Inserted a new file '{0}' with ETag {1}", name, headers.Value <Guid>(Constants.MetadataEtagField)); using (var contentStream = await Request.Content.ReadAsStreamAsync()) using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, contentStream, name)) { await readFileToDatabase.Execute(); if (readFileToDatabase.TotalSizeRead != size) { Storage.Batch(accessor => { StorageOperationsTask.IndicateFileToDelete(name); }); throw new HttpResponseException(HttpStatusCode.BadRequest); } Historian.UpdateLastModified(headers); // update with the final file size log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name); headers["Content-MD5"] = readFileToDatabase.FileHash; Storage.Batch(accessor => accessor.UpdateFileMetadata(name, headers)); int totalSizeRead = readFileToDatabase.TotalSizeRead; headers["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture); Search.Index(name, headers); Publisher.Publish(new FileChangeNotification { Action = FileChangeAction.Add, File = FilePathTools.Cannoicalise(name) }); log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, headers.Value <Guid>(Constants.MetadataEtagField)); StartSynchronizeDestinationsInBackground(); } } catch (Exception ex) { if (uploadId != null) { Guid uploadIdentifier; if (Guid.TryParse(uploadId, out uploadIdentifier)) { Publisher.Publish(new CancellationNotification { UploadId = uploadIdentifier, File = name }); } } log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex); var concurrencyException = ex as ConcurrencyException; if (concurrencyException != null) { throw ConcurrencyResponseException(concurrencyException); } throw; } return(GetEmptyMessage(HttpStatusCode.Created)); }
public HttpResponseMessage Rename(string fileName, string rename) { var sourceServerInfo = InnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(InnerHeaders.GetValues("ETag").First().Trim('\"')); var sourceMetadata = GetFilteredMetadataFromHeaders(InnerHeaders); Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName, rename, sourceFileETag, sourceServerInfo); var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.Rename); try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(fileName, accessor); FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor); }); PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(fileName, accessor)); var localMetadata = GetLocalMetadata(fileName); bool isConflictResolved; AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); if (isConflictResolved) { ConflictArtifactManager.Delete(fileName); Publisher.Publish(new ConflictResolvedNotification { FileName = fileName }); } StorageOperationsTask.RenameFile(new RenameFileOperation { Name = fileName, Rename = rename, MetadataAfterOperation = sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers() }); } catch (Exception ex) { report.Exception = ex; Log.WarnException(string.Format("Error was occurred during renaming synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex); } finally { FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag); PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); } if (report.Exception == null) { Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo); } return(this.GetMessageWithObject(report, HttpStatusCode.OK)); }
public static StorageStream CreatingNewAndWritting(ITransactionalStorage transactionalStorage, IndexStorage indexStorage, StorageOperationsTask operations, string fileName, RavenJObject metadata) { if (indexStorage == null) { throw new ArgumentNullException("indexStorage", "indexStorage == null"); } return(new StorageStream(transactionalStorage, fileName, StorageStreamAccess.CreateAndWrite, metadata, indexStorage, operations)); }
public HttpResponseMessage Rename(string fileSystemName, string fileName, string rename) { bool isConflictResolved = false; var canonicalFilename = FileHeader.Canonize(fileName); var canonicalRename = FileHeader.Canonize(rename); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders); Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName, rename, sourceFileETag, sourceServerInfo); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Rename); try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Rename); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); var localMetadata = GetLocalMetadata(canonicalFilename); AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); if (isConflictResolved) { ConflictArtifactManager.Delete(canonicalFilename); } StorageOperationsTask.RenameFile(new RenameFileOperation { FileSystem = FileSystem.Name, Name = canonicalFilename, Rename = canonicalRename, MetadataAfterOperation = sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers() }); } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; Log.WarnException(string.Format("Error was occurred during renaming synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex); } } finally { FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); } PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (isConflictResolved) { Publisher.Publish(new ConflictNotification { FileName = fileName, Status = ConflictStatus.Resolved }); } if (report.Exception == null) { Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo); } return(GetMessageWithObject(report)); }
public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName) { if (!Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault(); var canonicalFilename = FileHeader.Canonize(fileName); var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate); Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo); StorageStream localFile = null; var isNewFile = false; var isConflictResolved = false; try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders); var localMetadata = GetLocalMetadata(canonicalFilename); if (localMetadata != null) { AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); localFile = StorageStream.Reading(Storage, canonicalFilename); } else { isNewFile = true; } Historian.UpdateLastModified(sourceMetadata); var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata); var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile); Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName); await Request.Content.ReadAsMultipartAsync(provider); Log.Debug("Multipart content of a file '{0}' was processed/read", fileName); report.BytesCopied = provider.BytesCopied; report.BytesTransfered = provider.BytesTransfered; report.NeedListLength = provider.NumberOfFileParts; synchronizingFile.PreventUploadComplete = false; synchronizingFile.Flush(); synchronizingFile.Dispose(); sourceMetadata["Content-MD5"] = synchronizingFile.FileHash; Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata)); Storage.Batch(accessor => { StorageOperationsTask.IndicateFileToDelete(canonicalFilename); accessor.RenameFile(tempFileName, canonicalFilename); Search.Delete(tempFileName); Search.Index(canonicalFilename, sourceMetadata); }); if (isNewFile) { Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName); } else { Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName); } if (isConflictResolved) { ConflictArtifactManager.Delete(canonicalFilename); } } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; } } finally { if (localFile != null) { localFile.Dispose(); } } if (report.Exception == null) { Log.Debug( "File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}", fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength); } else { Log.WarnException( string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo), report.Exception); } FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update); PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (isConflictResolved) { Publisher.Publish(new ConflictNotification { FileName = fileName, Status = ConflictStatus.Resolved }); } return(GetMessageWithObject(report)); }
public async Task <HttpResponseMessage> Put(string name, string uploadId = null, bool preserveTimestamps = false) { try { FileSystem.MetricsCounters.FilesPerSecond.Mark(); name = FileHeader.Canonize(name); var headers = this.GetFilteredMetadataFromHeaders(ReadInnerHeaders); if (preserveTimestamps) { if (!headers.ContainsKey(Constants.RavenCreationDate)) { if (headers.ContainsKey(Constants.CreationDate)) { headers[Constants.RavenCreationDate] = headers[Constants.CreationDate]; } else { throw new InvalidOperationException("Preserve Timestamps requires that the client includes the Raven-Creation-Date header."); } } var lastModified = GetHeader(Constants.RavenLastModified); if (lastModified != null) { DateTimeOffset when; if (!DateTimeOffset.TryParse(lastModified, out when)) { when = DateTimeOffset.UtcNow; } Historian.UpdateLastModified(headers, when); } else { Historian.UpdateLastModified(headers); } } else { headers[Constants.RavenCreationDate] = DateTimeOffset.UtcNow; Historian.UpdateLastModified(headers); } // TODO: To keep current filesystems working. We should remove when adding a new migration. headers[Constants.CreationDate] = headers[Constants.RavenCreationDate].Value <DateTimeOffset>().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture); Historian.Update(name, headers); SynchronizationTask.Cancel(name); long?size = -1; ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor => { AssertPutOperationNotVetoed(name, headers); AssertFileIsNotBeingSynced(name, accessor, true); var contentLength = Request.Content.Headers.ContentLength; var sizeHeader = GetHeader("RavenFS-size"); long sizeForParse; if (contentLength == 0 || long.TryParse(sizeHeader, out sizeForParse) == false) { size = contentLength; if (Request.Headers.TransferEncodingChunked ?? false) { size = null; } } else { size = sizeForParse; } FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(name, headers)); using (FileSystem.DisableAllTriggersForCurrentThread()) { StorageOperationsTask.IndicateFileToDelete(name); } accessor.PutFile(name, size, headers); FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(name, size, headers)); Search.Index(name, headers); })); log.Debug("Inserted a new file '{0}' with ETag {1}", name, headers.Value <Guid>(Constants.MetadataEtagField)); using (var contentStream = await Request.Content.ReadAsStreamAsync()) using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, FileSystem.PutTriggers, contentStream, name, headers)) { await readFileToDatabase.Execute(); if (readFileToDatabase.TotalSizeRead != size) { StorageOperationsTask.IndicateFileToDelete(name); throw new HttpResponseException(HttpStatusCode.BadRequest); } if (!preserveTimestamps) { Historian.UpdateLastModified(headers); // update with the final file size. } log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name); headers["Content-MD5"] = readFileToDatabase.FileHash; Storage.Batch(accessor => accessor.UpdateFileMetadata(name, headers)); int totalSizeRead = readFileToDatabase.TotalSizeRead; headers["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture); Search.Index(name, headers); Publisher.Publish(new FileChangeNotification { Action = FileChangeAction.Add, File = FilePathTools.Cannoicalise(name) }); log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, headers.Value <Guid>(Constants.MetadataEtagField)); StartSynchronizeDestinationsInBackground(); } } catch (Exception ex) { if (uploadId != null) { Guid uploadIdentifier; if (Guid.TryParse(uploadId, out uploadIdentifier)) { Publisher.Publish(new CancellationNotification { UploadId = uploadIdentifier, File = name }); } } log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex); var concurrencyException = ex as ConcurrencyException; if (concurrencyException != null) { throw ConcurrencyResponseException(concurrencyException); } throw; } return(GetEmptyMessage(HttpStatusCode.Created)); }