public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState recievedTransportState = null) { this.Name = name; this.systemConfiguration = systemConfiguration; var storageType = systemConfiguration.FileSystem.DefaultStorageTypeName; storage = CreateTransactionalStorage(storageType, systemConfiguration); search = new IndexStorage(systemConfiguration.FileSystem.IndexStoragePath, systemConfiguration.Settings); sigGenerator = new SigGenerator(); var replicationHiLo = new SynchronizationHiLo(storage); var sequenceActions = new SequenceActions(storage); transportState = recievedTransportState ?? new TransportState(); notificationPublisher = new NotificationPublisher(transportState); fileLockManager = new FileLockManager(); storage.Initialize(); search.Initialize(); var uuidGenerator = new UuidGenerator(sequenceActions); historian = new Historian(storage, replicationHiLo, uuidGenerator); BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024); conflictArtifactManager = new ConflictArtifactManager(storage, search); conflictDetector = new ConflictDetector(); conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog)); synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration); storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher); metricsCounters = new MetricsCountersManager(); AppDomain.CurrentDomain.ProcessExit += ShouldDispose; AppDomain.CurrentDomain.DomainUnload += ShouldDispose; }
private void FinishSynchronization(string fileName, SynchronizationReport report, ServerInfo sourceServer, Guid sourceFileETag) { try { // we want to execute those operation in a single batch but we also have to ensure that // Raven/Synchronization/Sources/sourceServerId config is modified only by one finishing synchronization at the same time SynchronizationFinishLocks.GetOrAdd(sourceServer.Id, new ReaderWriterLockSlim()).EnterWriteLock(); SynchronizationTask.IncomingSynchronizationFinished(fileName, sourceServer, sourceFileETag); Storage.Batch(accessor => { SaveSynchronizationReport(fileName, accessor, report); FileLockManager.UnlockByDeletingSyncConfiguration(fileName, accessor); if (report.Exception == null) { SaveSynchronizationSourceInformation(sourceServer, sourceFileETag, accessor); } }); } catch (Exception ex) { Log.ErrorException( string.Format("Failed to finish synchronization of a file '{0}' from {1}", fileName, sourceServer), ex); } finally { SynchronizationFinishLocks.GetOrAdd(sourceServer.Id, new ReaderWriterLockSlim()).ExitWriteLock(); } }
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState receivedTransportState = null) { ExtensionsState = new AtomicDictionary <object>(); Name = name; this.systemConfiguration = systemConfiguration; systemConfiguration.Container.SatisfyImportsOnce(this); transportState = receivedTransportState ?? new TransportState(); storage = CreateTransactionalStorage(systemConfiguration); sigGenerator = new SigGenerator(); fileLockManager = new FileLockManager(); BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024); conflictDetector = new ConflictDetector(); conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog)); notificationPublisher = new NotificationPublisher(transportState); synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration); metricsCounters = new MetricsCountersManager(); search = new IndexStorage(name, systemConfiguration); conflictArtifactManager = new ConflictArtifactManager(storage, search); storageOperationsTask = new StorageOperationsTask(storage, DeleteTriggers, search, notificationPublisher); AppDomain.CurrentDomain.ProcessExit += ShouldDispose; AppDomain.CurrentDomain.DomainUnload += ShouldDispose; }
private bool IsSynchronizationInProgress(string originalFileName) { if (!FileLockManager.TimeoutExceeded(originalFileName, Storage)) { return(true); } return(false); }
public RavenFileSystem(InMemoryRavenConfiguration config, string name, TransportState receivedTransportState = null) { ExtensionsState = new AtomicDictionary <object>(); Name = name; ResourceName = string.Concat(Constants.FileSystem.UrlPrefix, "/", name); configuration = config; try { ValidateStorage(); configuration.Container.SatisfyImportsOnce(this); transportState = receivedTransportState ?? new TransportState(); storage = CreateTransactionalStorage(configuration); sigGenerator = new SigGenerator(); fileLockManager = new FileLockManager(); BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024); conflictDetector = new ConflictDetector(); conflictResolver = new ConflictResolver(storage, new CompositionContainer(configuration.Catalog)); notificationPublisher = new NotificationPublisher(transportState); synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, configuration); metricsCounters = new MetricsCountersManager(); search = new IndexStorage(name, configuration); conflictArtifactManager = new ConflictArtifactManager(storage, search); TimerManager = new ResourceTimerManager(); Tasks = new TaskActions(this, Log); Files = new FileActions(this, Log); Synchronizations = new SynchronizationActions(this, Log); AppDomain.CurrentDomain.ProcessExit += ShouldDispose; AppDomain.CurrentDomain.DomainUnload += ShouldDispose; } catch (Exception e) { Log.ErrorException(string.Format("Could not create file system '{0}'", Name ?? "unknown name"), e); try { Dispose(); } catch (Exception ex) { Log.FatalException("Failed to dispose when already getting an error in file system ctor", ex); } throw; } }
public void AssertFileIsNotBeingSynced(string fileName) { Storage.Batch(accessor => { if (FileLockManager.TimeoutExceeded(fileName, accessor)) { FileLockManager.UnlockByDeletingSyncConfiguration(fileName, accessor); } else { Log.Debug("Cannot execute operation because file '{0}' is being synced", fileName); throw new SynchronizationException(string.Format("File {0} is being synced", fileName)); } }); }
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, TransportState transportState, string name) { this.Name = name; this.systemConfiguration = systemConfiguration; var storageType = systemConfiguration.DefaultFileSystemStorageTypeName; if (string.Equals(InMemoryRavenConfiguration.VoronTypeName, storageType, StringComparison.OrdinalIgnoreCase) == false) { if (Directory.Exists(systemConfiguration.FileSystemDataDirectory) && Directory.EnumerateFileSystemEntries(systemConfiguration.FileSystemDataDirectory).Any()) { throw new InvalidOperationException( string.Format( "We do not allow to run on a storage engine other then Voron, while we are in the early pre-release phase of RavenDB 3.0. You are currently running on {0}", storageType)); } Trace.WriteLine("Forcing filesystem to run on Voron - pre release behavior only, mind " + Path.GetFileName(Path.GetDirectoryName(systemConfiguration.FileSystemDataDirectory))); storageType = InMemoryRavenConfiguration.VoronTypeName; } storage = CreateTransactionalStorage(storageType, systemConfiguration); search = new IndexStorage(systemConfiguration.FileSystemIndexStoragePath, systemConfiguration.Settings); sigGenerator = new SigGenerator(); var replicationHiLo = new SynchronizationHiLo(storage); var sequenceActions = new SequenceActions(storage); this.transportState = transportState; notificationPublisher = new NotificationPublisher(transportState); fileLockManager = new FileLockManager(); storage.Initialize(); search.Initialize(); var uuidGenerator = new UuidGenerator(sequenceActions); historian = new Historian(storage, replicationHiLo, uuidGenerator); BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024); conflictArtifactManager = new ConflictArtifactManager(storage, search); conflictDetector = new ConflictDetector(); conflictResolver = new ConflictResolver(); synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration); storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher); metricsCounters = new MetricsCountersManager(); AppDomain.CurrentDomain.ProcessExit += ShouldDispose; AppDomain.CurrentDomain.DomainUnload += ShouldDispose; }
protected void AssertFileIsNotBeingSynced(string fileName, IStorageActionsAccessor accessor, bool wrapByResponseException = false) { if (FileLockManager.TimeoutExceeded(fileName, accessor)) { FileLockManager.UnlockByDeletingSyncConfiguration(fileName, accessor); } else { Log.Debug("Cannot execute operation because file '{0}' is being synced", fileName); var beingSyncedException = new SynchronizationException(string.Format("File {0} is being synced", fileName)); if (wrapByResponseException) { throw new HttpResponseException(Request.CreateResponse((HttpStatusCode)420, beingSyncedException)); } throw beingSyncedException; } }
public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName) { if (!Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault(); var canonicalFilename = FileHeader.Canonize(fileName); var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate); Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo); StorageStream localFile = null; var isNewFile = false; var isConflictResolved = false; try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders); var localMetadata = GetLocalMetadata(canonicalFilename); if (localMetadata != null) { AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); localFile = StorageStream.Reading(Storage, canonicalFilename); } else { isNewFile = true; } Historian.UpdateLastModified(sourceMetadata); var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata); var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile); Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName); await Request.Content.ReadAsMultipartAsync(provider); Log.Debug("Multipart content of a file '{0}' was processed/read", fileName); report.BytesCopied = provider.BytesCopied; report.BytesTransfered = provider.BytesTransfered; report.NeedListLength = provider.NumberOfFileParts; synchronizingFile.PreventUploadComplete = false; synchronizingFile.Flush(); synchronizingFile.Dispose(); sourceMetadata["Content-MD5"] = synchronizingFile.FileHash; Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata)); Storage.Batch(accessor => { StorageOperationsTask.IndicateFileToDelete(canonicalFilename); accessor.RenameFile(tempFileName, canonicalFilename); Search.Delete(tempFileName); Search.Index(canonicalFilename, sourceMetadata); }); if (isNewFile) { Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName); } else { Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName); } if (isConflictResolved) { ConflictArtifactManager.Delete(canonicalFilename); } } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; } } finally { if (localFile != null) { localFile.Dispose(); } } if (report.Exception == null) { Log.Debug( "File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}", fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength); } else { Log.WarnException( string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo), report.Exception); } FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update); PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (isConflictResolved) { Publisher.Publish(new ConflictNotification { FileName = fileName, Status = ConflictStatus.Resolved }); } return(GetMessageWithObject(report)); }
public HttpResponseMessage Rename(string fileSystemName, string fileName, string rename) { bool isConflictResolved = false; var canonicalFilename = FileHeader.Canonize(fileName); var canonicalRename = FileHeader.Canonize(rename); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders); Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName, rename, sourceFileETag, sourceServerInfo); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Rename); try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Rename); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); var localMetadata = GetLocalMetadata(canonicalFilename); AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); if (isConflictResolved) { ConflictArtifactManager.Delete(canonicalFilename); } StorageOperationsTask.RenameFile(new RenameFileOperation { FileSystem = FileSystem.Name, Name = canonicalFilename, Rename = canonicalRename, MetadataAfterOperation = sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers() }); } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; Log.WarnException(string.Format("Error was occurred during renaming synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex); } } finally { FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); } PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (isConflictResolved) { Publisher.Publish(new ConflictNotification { FileName = fileName, Status = ConflictStatus.Resolved }); } if (report.Exception == null) { Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo); } return(GetMessageWithObject(report)); }
public HttpResponseMessage Delete(string fileSystemName, string fileName) { var canonicalFilename = FileHeader.Canonize(fileName); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceServerInfo); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Delete); try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Delete); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); var localMetadata = GetLocalMetadata(canonicalFilename); if (localMetadata != null) { // REVIEW: Use InnerHeaders for consistency? var sourceMetadata = GetFilteredMetadataFromHeaders(Request.Headers); // Request.Headers.FilterHeadersToObject(); bool isConflictResolved; AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); Storage.Batch(accessor => { StorageOperationsTask.IndicateFileToDelete(canonicalFilename); var tombstoneMetadata = new RavenJObject { { SynchronizationConstants.RavenSynchronizationHistory, localMetadata[SynchronizationConstants.RavenSynchronizationHistory] }, { SynchronizationConstants.RavenSynchronizationVersion, localMetadata[SynchronizationConstants.RavenSynchronizationVersion] }, { SynchronizationConstants.RavenSynchronizationSource, localMetadata[SynchronizationConstants.RavenSynchronizationSource] } }.WithDeleteMarker(); Historian.UpdateLastModified(tombstoneMetadata); accessor.PutFile(canonicalFilename, 0, tombstoneMetadata, true); }); PublishFileNotification(fileName, FileChangeAction.Delete); } } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; Log.WarnException(string.Format("Error was occurred during deletion synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex); } } finally { FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); } PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (report.Exception == null) { Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceServerInfo); } return(this.GetMessageWithObject(report, HttpStatusCode.OK)); }
public HttpResponseMessage UpdateMetadata(string fileSystemName, string fileName) { bool isConflictResolved = false; var canonicalFilename = FileHeader.Canonize(fileName); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); // REVIEW: (Oren) It works, but it seems to me it is not an scalable solution. var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceServerInfo); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.MetadataUpdate); try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.MetadataUpdate); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); var localMetadata = GetLocalMetadata(canonicalFilename); var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders); AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); Historian.UpdateLastModified(sourceMetadata); Storage.Batch(accessor => accessor.UpdateFileMetadata(canonicalFilename, sourceMetadata)); Search.Index(canonicalFilename, sourceMetadata); if (isConflictResolved) { ConflictArtifactManager.Delete(canonicalFilename); } PublishFileNotification(fileName, FileChangeAction.Update); } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; Log.WarnException( string.Format("Error was occurred during metadata synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex); } } finally { FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); } PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (isConflictResolved) { Publisher.Publish(new ConflictNotification { FileName = fileName, Status = ConflictStatus.Resolved }); } if (report.Exception == null) { Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceServerInfo); } return(this.GetMessageWithObject(report, HttpStatusCode.OK)); }
public HttpResponseMessage Rename(string fileName, string rename) { var sourceServerInfo = InnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(InnerHeaders.GetValues("ETag").First().Trim('\"')); var sourceMetadata = GetFilteredMetadataFromHeaders(InnerHeaders); Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName, rename, sourceFileETag, sourceServerInfo); var report = new SynchronizationReport(fileName, sourceFileETag, SynchronizationType.Rename); try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(fileName, accessor); FileLockManager.LockByCreatingSyncConfiguration(fileName, sourceServerInfo, accessor); }); PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(fileName, accessor)); var localMetadata = GetLocalMetadata(fileName); bool isConflictResolved; AssertConflictDetection(fileName, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); if (isConflictResolved) { ConflictArtifactManager.Delete(fileName); Publisher.Publish(new ConflictResolvedNotification { FileName = fileName }); } StorageOperationsTask.RenameFile(new RenameFileOperation { Name = fileName, Rename = rename, MetadataAfterOperation = sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers() }); } catch (Exception ex) { report.Exception = ex; Log.WarnException(string.Format("Error was occurred during renaming synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex); } finally { FinishSynchronization(fileName, report, sourceServerInfo, sourceFileETag); PublishSynchronizationNotification(fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); } if (report.Exception == null) { Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo); } return(this.GetMessageWithObject(report, HttpStatusCode.OK)); }