private List <FileSystemData> GetFileSystemsData(IEnumerable <RavenJToken> fileSystems) { return(fileSystems .Select(fileSystem => { var bundles = new string[] { }; var settings = fileSystem.Value <RavenJObject>("Settings"); if (settings != null) { var activeBundles = settings.Value <string>(RavenConfiguration.GetKey(x => x.Core._ActiveBundlesString)); if (activeBundles != null) { bundles = activeBundles.Split(';'); } } var fsName = fileSystem.Value <RavenJObject>("@metadata").Value <string>("@id").Replace(Constants.FileSystem.Prefix, string.Empty); return new FileSystemData { Name = fsName, Disabled = fileSystem.Value <bool>("Disabled"), Bundles = bundles, IsAdminCurrentTenant = true, IsLoaded = FileSystemsLandlord.IsFileSystemLoaded(fsName) }; }).ToList()); }
public HttpResponseMessage Delete(string id) { var docKey = "Raven/FileSystems/" + id; var configuration = FileSystemsLandlord.CreateTenantConfiguration(id); if (configuration == null) { return(GetEmptyMessage()); } Database.Documents.Delete(docKey, null, null); bool result; if (bool.TryParse(InnerRequest.RequestUri.ParseQueryString()["hard-delete"], out result) && result) { IOExtensions.DeleteDirectory(configuration.FileSystemDataDirectory); if (configuration.IndexStoragePath != null) { IOExtensions.DeleteDirectory(configuration.IndexStoragePath); } if (configuration.JournalsStoragePath != null) { IOExtensions.DeleteDirectory(configuration.JournalsStoragePath); } } return(GetEmptyMessage()); }
public RavenDBOptions(InMemoryRavenConfiguration configuration, DocumentDatabase db = null) { if (configuration == null) throw new ArgumentNullException("configuration"); try { HttpEndpointRegistration.RegisterHttpEndpointTarget(); HttpEndpointRegistration.RegisterAdminLogsTarget(); if (db == null) { configuration.UpdateDataDirForLegacySystemDb(); systemDatabase = new DocumentDatabase(configuration); systemDatabase.SpinBackgroundWorkers(); } else { systemDatabase = db; } fileSystemLandlord = new FileSystemsLandlord(systemDatabase); databasesLandlord = new DatabasesLandlord(systemDatabase); countersLandlord = new CountersLandlord(systemDatabase); requestManager = new RequestManager(databasesLandlord); mixedModeRequestAuthorizer = new MixedModeRequestAuthorizer(); mixedModeRequestAuthorizer.Initialize(systemDatabase, new RavenServer(databasesLandlord.SystemDatabase, configuration)); } catch { if (systemDatabase != null) systemDatabase.Dispose(); throw; } }
public HttpResponseMessage Stats() { if (Database != DatabasesLandlord.SystemDatabase) { return(GetMessageWithString("Admin stats can only be had from the root database", HttpStatusCode.NotFound)); } var allDbs = new List <DocumentDatabase>(); var allFs = new List <RavenFileSystem>(); DatabasesLandlord.ForAllDatabases(allDbs.Add); FileSystemsLandlord.ForAllFileSystems(allFs.Add); var currentConfiguration = DatabasesLandlord.SystemConfiguration; var stats = new AdminStatistics { ServerName = currentConfiguration.ServerName, TotalNumberOfRequests = RequestManager.NumberOfRequests, Uptime = SystemTime.UtcNow - RequestManager.StartUpTime, Memory = new AdminMemoryStatistics { DatabaseCacheSizeInMB = ConvertBytesToMBs(DatabasesLandlord.SystemDatabase.TransactionalStorage.GetDatabaseCacheSizeInBytes()), ManagedMemorySizeInMB = ConvertBytesToMBs(GetCurrentManagedMemorySize()), TotalProcessMemorySizeInMB = ConvertBytesToMBs(GetCurrentProcessPrivateMemorySize64()), }, LoadedDatabases = from documentDatabase in allDbs let indexStorageSize = documentDatabase.GetIndexStorageSizeOnDisk() let transactionalStorageSize = documentDatabase.GetTransactionalStorageSizeOnDisk() let totalDatabaseSize = indexStorageSize + transactionalStorageSize.AllocatedSizeInBytes let lastUsed = DatabasesLandlord.LastRecentlyUsed.GetOrDefault(documentDatabase.Name ?? Constants.SystemDatabase) select new LoadedDatabaseStatistics { Name = documentDatabase.Name, LastActivity = new[] { lastUsed, documentDatabase.WorkContext.LastWorkTime }.Max(), TransactionalStorageAllocatedSize = transactionalStorageSize.AllocatedSizeInBytes, TransactionalStorageAllocatedSizeHumaneSize = SizeHelper.Humane(transactionalStorageSize.AllocatedSizeInBytes), TransactionalStorageUsedSize = transactionalStorageSize.UsedSizeInBytes, TransactionalStorageUsedSizeHumaneSize = SizeHelper.Humane(transactionalStorageSize.UsedSizeInBytes), IndexStorageSize = indexStorageSize, IndexStorageHumaneSize = SizeHelper.Humane(indexStorageSize), TotalDatabaseSize = totalDatabaseSize, TotalDatabaseHumaneSize = SizeHelper.Humane(totalDatabaseSize), CountOfDocuments = documentDatabase.Statistics.CountOfDocuments, CountOfAttachments = documentDatabase.Statistics.CountOfAttachments, DatabaseTransactionVersionSizeInMB = ConvertBytesToMBs(documentDatabase.TransactionalStorage.GetDatabaseTransactionVersionSizeInBytes()), Metrics = documentDatabase.CreateMetrics() }, LoadedFileSystems = from fileSystem in allFs select fileSystem.GetFileSystemStats() }; return(GetMessageWithObject(stats)); }
public RavenDBOptions(InMemoryRavenConfiguration configuration, DocumentDatabase db = null) { if (configuration == null) { throw new ArgumentNullException("configuration"); } try { HttpEndpointRegistration.RegisterHttpEndpointTarget(); if (db == null) { systemDatabase = new DocumentDatabase(configuration); systemDatabase.SpinBackgroundWorkers(); } else { systemDatabase = db; } fileSystemLandlord = new FileSystemsLandlord(systemDatabase); databasesLandlord = new DatabasesLandlord(systemDatabase); countersLandlord = new CountersLandlord(systemDatabase); requestManager = new RequestManager(databasesLandlord); mixedModeRequestAuthorizer = new MixedModeRequestAuthorizer(); mixedModeRequestAuthorizer.Initialize(systemDatabase, new RavenServer(databasesLandlord.SystemDatabase, configuration)); } catch { if (systemDatabase != null) { systemDatabase.Dispose(); } throw; } }
public async Task <HttpResponseMessage> FileSystemPut(string id, bool update = false) { MessageWithStatusCode fileSystemNameFormat = CheckNameFormat(id, Database.Configuration.FileSystem.DataDirectory); if (fileSystemNameFormat.Message != null) { return(GetMessageWithString(fileSystemNameFormat.Message, fileSystemNameFormat.ErrorCode)); } var docKey = "Raven/FileSystems/" + id; // There are 2 possible ways to call this put. We either want to update a filesystem configuration or we want to create a new one. if (!update) { // As we are not updating, we should fail when the filesystem already exists. var existingFilesystem = Database.Documents.Get(docKey, null); if (existingFilesystem != null) { return(GetEmptyMessage(HttpStatusCode.Conflict)); } } var fsDoc = await ReadJsonObjectAsync <DatabaseDocument>(); FileSystemsLandlord.Protect(fsDoc); var json = RavenJObject.FromObject(fsDoc); json.Remove("Id"); Database.Documents.Put(docKey, null, json, new RavenJObject(), null); return(GetEmptyMessage(HttpStatusCode.Created)); }
private AdminStatistics CreateAdminStats() { var allDbs = new List <DocumentDatabase>(); var allFs = new List <RavenFileSystem>(); DatabasesLandlord.ForAllDatabases(allDbs.Add); FileSystemsLandlord.ForAllFileSystems(allFs.Add); var currentConfiguration = DatabasesLandlord.SystemConfiguration; var stats = new AdminStatistics { ServerName = currentConfiguration.ServerName, TotalNumberOfRequests = RequestManager.NumberOfRequests, Uptime = SystemTime.UtcNow - RequestManager.StartUpTime, Memory = new AdminMemoryStatistics { DatabaseCacheSizeInMB = ConvertBytesToMBs(DatabasesLandlord.SystemDatabase.TransactionalStorage.GetDatabaseCacheSizeInBytes()), ManagedMemorySizeInMB = ConvertBytesToMBs(GetCurrentManagedMemorySize()), TotalProcessMemorySizeInMB = ConvertBytesToMBs(GetCurrentProcessPrivateMemorySize64()), }, LoadedDatabases = LoadedDatabasesStats(allDbs), LoadedFileSystems = from fileSystem in allFs select fileSystem.GetFileSystemStats() }; return(stats); }
private MessageWithStatusCode DeleteFileSystem(string id, bool isHardDeleteNeeded) { //get configuration even if the file system is disabled var configuration = FileSystemsLandlord.CreateTenantConfiguration(id, true); if (configuration == null) { return new MessageWithStatusCode { ErrorCode = HttpStatusCode.NotFound, Message = "File system wasn't found" } } ; var docKey = Constants.FileSystem.Prefix + id; SystemDatabase.Documents.Delete(docKey, null, null); if (isHardDeleteNeeded && configuration.Core.RunInMemory == false) { IOExtensions.DeleteDirectory(configuration.FileSystem.DataDirectory); if (configuration.FileSystem.IndexStoragePath != null) { IOExtensions.DeleteDirectory(configuration.FileSystem.IndexStoragePath); } } return(new MessageWithStatusCode()); }
private async Task <HttpResponseMessage> ExecuteActualRequest(HttpControllerContext controllerContext, CancellationToken cancellationToken, MixedModeRequestAuthorizer authorizer) { HttpResponseMessage authMsg; if (authorizer.TryAuthorize(this, out authMsg) == false) { return(authMsg); } if (IsInternalRequest == false) { RequestManager.IncrementRequestCount(); } var fileSystemInternal = await FileSystemsLandlord.GetFileSystemInternal(FileSystemName); if (fileSystemInternal == null) { var msg = "Could not find a file system named: " + FileSystemName; return(GetMessageWithObject(new { Error = msg }, HttpStatusCode.ServiceUnavailable)); } var sp = Stopwatch.StartNew(); var result = await base.ExecuteAsync(controllerContext, cancellationToken); sp.Stop(); AddRavenHeader(result, sp); return(result); }
private MessageWithStatusCode ToggleFileSystemDisabled(string fileSystemId, bool isSettingDisabled) { var docKey = "Raven/FileSystems/" + fileSystemId; var document = Database.Documents.Get(docKey, null); if (document == null) { return new MessageWithStatusCode { ErrorCode = HttpStatusCode.NotFound, Message = "File system " + fileSystemId + " wasn't found" } } ; var fsDoc = document.DataAsJson.JsonDeserialization <DatabaseDocument>(); if (fsDoc.Disabled == isSettingDisabled) { string state = isSettingDisabled ? "disabled" : "enabled"; return(new MessageWithStatusCode { ErrorCode = HttpStatusCode.BadRequest, Message = "File system " + fileSystemId + " is already " + state }); } FileSystemsLandlord.Unprotect(fsDoc); fsDoc.Disabled = !fsDoc.Disabled; var json = RavenJObject.FromObject(fsDoc); json.Remove("Id"); Database.Documents.Put(docKey, document.Etag, json, new RavenJObject(), null); return(new MessageWithStatusCode()); } }
protected override void InnerInitialization(HttpControllerContext controllerContext) { base.InnerInitialization(controllerContext); landlord = (FileSystemsLandlord)controllerContext.Configuration.Properties[typeof(FileSystemsLandlord)]; requestManager = (RequestManager)controllerContext.Configuration.Properties[typeof(RequestManager)]; var values = controllerContext.Request.GetRouteData().Values; if (values.ContainsKey("MS_SubRoutes")) { var routeDatas = (IHttpRouteData[])controllerContext.Request.GetRouteData().Values["MS_SubRoutes"]; var selectedData = routeDatas.FirstOrDefault(data => data.Values.ContainsKey("fileSystemName")); if (selectedData != null) { FileSystemName = selectedData.Values["fileSystemName"] as string; } } else { if (values.ContainsKey("fil")) { FileSystemName = values["fileSystemName"] as string; } } if (FileSystemName == null) { throw new InvalidOperationException("Could not find file system name for this request"); } }
public async Task <HttpResponseMessage> Stats() { var fileSystemsDocument = GetFileSystemsDocuments(); var fileSystemsData = GetFileSystemsData(fileSystemsDocument); var fileSystemsNames = fileSystemsData.Select(fileSystemObject => fileSystemObject.Name).ToArray(); var stats = new List <FileSystemStats>(); foreach (var fileSystemName in fileSystemsNames) { Task <RavenFileSystem> fsTask; if (!FileSystemsLandlord.TryGetFileSystem(fileSystemName, out fsTask)) // we only care about active file systems { continue; } if (fsTask.IsCompleted == false) { continue; // we don't care about in process of starting file systems } var ravenFileSystem = await fsTask; var fsStats = ravenFileSystem.GetFileSystemStats(); stats.Add(fsStats); } return(GetMessageWithObject(stats).WithNoCache()); }
private MessageWithStatusCode DeleteFileSystem(string fileSystemId, bool isHardDeleteNeeded) { //get configuration even if the file system is disabled var configuration = FileSystemsLandlord.CreateTenantConfiguration(fileSystemId, true); if (configuration == null) { return new MessageWithStatusCode { ErrorCode = HttpStatusCode.NotFound, Message = "File system wasn't found" } } ; var docKey = Constants.FileSystem.Prefix + fileSystemId; Database.Documents.Delete(docKey, null, null); if (isHardDeleteNeeded) { IOExtensions.DeleteDirectory(configuration.FileSystem.DataDirectory); //TODO: find out which path should be deleted /*if (configuration.IndexStoragePath != null) * IOExtensions.DeleteDirectory(configuration.IndexStoragePath);*/ /*if (configuration.JournalsStoragePath != null) * IOExtensions.DeleteDirectory(configuration.JournalsStoragePath);*/ } return(new MessageWithStatusCode()); }
public RavenDBOptions(InMemoryRavenConfiguration configuration, DocumentDatabase db = null) { if (configuration == null) { throw new ArgumentNullException("configuration"); } try { ThreadPool.SetMinThreads(configuration.MinThreadPoolWorkerThreads, configuration.MinThreadPoolCompletionThreads); HttpEndpointRegistration.RegisterHttpEndpointTarget(); HttpEndpointRegistration.RegisterAdminLogsTarget(); if (db == null) { configuration.UpdateDataDirForLegacySystemDb(); systemDatabase = new DocumentDatabase(configuration, null); systemDatabase.SpinBackgroundWorkers(false); } else { systemDatabase = db; } WebSocketBufferPool.Initialize(configuration.WebSockets.InitialBufferPoolSize); fileSystemLandlord = new FileSystemsLandlord(systemDatabase); databasesLandlord = new DatabasesLandlord(systemDatabase); countersLandlord = new CountersLandlord(systemDatabase); timeSeriesLandlord = new TimeSeriesLandlord(systemDatabase); requestManager = new RequestManager(databasesLandlord); systemDatabase.RequestManager = requestManager; ClusterManager = new Reference <ClusterManager>(); systemDatabase.ClusterManager = ClusterManager; mixedModeRequestAuthorizer = new MixedModeRequestAuthorizer(); mixedModeRequestAuthorizer.Initialize(systemDatabase, new RavenServer(databasesLandlord.SystemDatabase, configuration)); serverStartupTasks = configuration.Container.GetExportedValues <IServerStartupTask>(); foreach (var task in serverStartupTasks) { toDispose.Add(task); try { task.Execute(this); } catch (Exception e) { systemDatabase.LogErrorAndAddAlertOnStartupTaskException(task.GetType().FullName, e); } } } catch (Exception) { if (systemDatabase != null) { systemDatabase.Dispose(); } throw; } }
public WebSocketsRequestParser(DatabasesLandlord databasesLandlord, CountersLandlord countersLandlord, FileSystemsLandlord fileSystemsLandlord, MixedModeRequestAuthorizer authorizer, string expectedRequestSuffix) { DatabasesLandlord = databasesLandlord; this.countersLandlord = countersLandlord; this.fileSystemsLandlord = fileSystemsLandlord; this.authorizer = authorizer; this.expectedRequestSuffix = expectedRequestSuffix; }
public async Task <HttpResponseMessage> FileSystemPut(string id, bool update = false) { MessageWithStatusCode fileSystemNameFormat = CheckNameFormat(id, Database.Configuration.FileSystem.DataDirectory); if (fileSystemNameFormat.Message != null) { return(GetMessageWithObject(new { Error = fileSystemNameFormat.Message }, fileSystemNameFormat.ErrorCode)); } if (Authentication.IsLicensedForRavenFs == false) { return(GetMessageWithObject(new { Error = "Your license does not allow the use of RavenFS!" }, HttpStatusCode.BadRequest)); } var docKey = Constants.FileSystem.Prefix + id; // There are 2 possible ways to call this put. We either want to update a filesystem configuration or we want to create a new one. if (!update) { // As we are not updating, we should fail when the filesystem already exists. var existingFilesystem = Database.Documents.Get(docKey, null); if (existingFilesystem != null) { return(GetEmptyMessage(HttpStatusCode.Conflict)); } } var fsDoc = await ReadJsonObjectAsync <FileSystemDocument>(); EnsureFileSystemHasRequiredSettings(id, fsDoc); string bundles; if (fsDoc.Settings.TryGetValue(Constants.ActiveBundles, out bundles) && bundles.IndexOf("Encryption", StringComparison.OrdinalIgnoreCase) != -1) { if (fsDoc.SecuredSettings == null || !fsDoc.SecuredSettings.ContainsKey(Constants.EncryptionKeySetting) || !fsDoc.SecuredSettings.ContainsKey(Constants.AlgorithmTypeSetting)) { return(GetMessageWithString(string.Format("Failed to create '{0}' file system, because of invalid encryption configuration.", id), HttpStatusCode.BadRequest)); } } FileSystemsLandlord.Protect(fsDoc); var json = RavenJObject.FromObject(fsDoc); json.Remove("Id"); Database.Documents.Put(docKey, null, json, new RavenJObject(), null); return(GetEmptyMessage(HttpStatusCode.Created)); }
private void CleanupFileSystems(DatabasesLandlord databaseLandlord, FileSystemsLandlord fileSystemLandlord) { var systemDatabase = databaseLandlord.SystemDatabase; int nextStart = 0; var fileSystemDocuments = systemDatabase .Documents .GetDocumentsWithIdStartingWith(Constants.FileSystem.Prefix, null, null, 0, int.MaxValue, CancellationToken.None, ref nextStart); var fileSystemIds = fileSystemDocuments .Select(x => ((RavenJObject)x)["@metadata"]) .Where(x => x != null) .Select(x => x.Value<string>("@id")) .Where(x => x != null && x != Constants.SystemDatabase) .ToList(); foreach (var fileSystemId in fileSystemIds) { try { var key = fileSystemId; if (key.StartsWith(Constants.FileSystem.Prefix)) key = key.Substring(Constants.FileSystem.Prefix.Length); var shouldCleanup = false; DateTime value; if (fileSystemLandlord.IsFileSystemLoaded(key) == false) shouldCleanup = true; else if (fileSystemLandlord.LastRecentlyUsed.TryGetValue(key, out value) == false || (SystemTime.UtcNow - value) > maxTimeResourceCanBeIdle) shouldCleanup = true; if (shouldCleanup == false) continue; var configuration = fileSystemLandlord.CreateTenantConfiguration(key, true); fileSystemLandlord.Cleanup(key, maxTimeResourceCanBeIdle, database => false); var docKey = Constants.FileSystem.Prefix + key; systemDatabase.Documents.Delete(docKey, null, null); if (configuration == null) continue; IOExtensions.DeleteDirectory(configuration.FileSystem.DataDirectory); } catch (Exception e) { log.WarnException(string.Format("Failed to cleanup '{0}' filesystem.", fileSystemId), e); } } }
public RavenDBOptions(RavenConfiguration configuration, DocumentDatabase db = null) { if (configuration == null) { throw new ArgumentNullException("configuration"); } try { HttpEndpointRegistration.RegisterHttpEndpointTarget(); HttpEndpointRegistration.RegisterAdminLogsTarget(); if (db == null) { systemDatabase = new DocumentDatabase(configuration, null); systemDatabase.SpinBackgroundWorkers(false); } else { systemDatabase = db; } WebSocketBufferPool.Initialize((int)configuration.WebSockets.InitialBufferPoolSize.GetValue(SizeUnit.Bytes)); fileSystemLandlord = new FileSystemsLandlord(systemDatabase); databasesLandlord = new DatabasesLandlord(systemDatabase); countersLandlord = new CountersLandlord(systemDatabase); timeSeriesLandlord = new TimeSeriesLandlord(systemDatabase); requestManager = new RequestManager(databasesLandlord); systemDatabase.RequestManager = requestManager; ClusterManager = new Reference <ClusterManager>(); mixedModeRequestAuthorizer = new MixedModeRequestAuthorizer(); mixedModeRequestAuthorizer.Initialize(systemDatabase, new RavenServer(databasesLandlord.SystemDatabase, configuration)); serverStartupTasks = configuration.Container.GetExportedValues <IServerStartupTask>(); foreach (var task in serverStartupTasks) { toDispose.Add(task); task.Execute(this); } } catch (Exception) { if (systemDatabase != null) { systemDatabase.Dispose(); } throw; } }
private List <FileSystemData> GetFileSystemsData(IEnumerable <RavenJToken> fileSystems) { return(fileSystems .Select(fileSystem => { var bundles = new string[] { }; var settings = fileSystem.Value <RavenJObject>("Settings"); if (settings != null) { var activeBundles = settings.Value <string>("Raven/ActiveBundles"); if (activeBundles != null) { bundles = activeBundles.Split(';'); } } var fsName = fileSystem.Value <RavenJObject>("@metadata").Value <string>("@id").Replace(Constants.FileSystem.Prefix, string.Empty); var isFileSystemLoaded = FileSystemsLandlord.IsFileSystemLoaded(fsName); FileSystemStats stats = null; if (isFileSystemLoaded) { try { var fs = FileSystemsLandlord.GetResourceInternal(fsName).Result; if (fs != null) { stats = fs.GetFileSystemStats(); } } catch (Exception) { //the file system is shutting down or locked //we can ignore this } } return new FileSystemData { Name = fsName, Disabled = fileSystem.Value <bool>("Disabled"), Bundles = bundles, IsAdminCurrentTenant = true, IsLoaded = isFileSystemLoaded, Stats = stats }; }).ToList()); }
public RavenDBOptions(InMemoryRavenConfiguration configuration, DocumentDatabase db = null) { if (configuration == null) throw new ArgumentNullException("configuration"); try { HttpEndpointRegistration.RegisterHttpEndpointTarget(); HttpEndpointRegistration.RegisterAdminLogsTarget(); ThreadPool.SetMinThreads(configuration.MinThreadPoolWorkerThreads, configuration.MinThreadPoolCompletionThreads); if (db == null) { configuration.UpdateDataDirForLegacySystemDb(); systemDatabase = new DocumentDatabase(configuration); systemDatabase.SpinBackgroundWorkers(false); } else { systemDatabase = db; } WebSocketBufferPool.Initialize(configuration.WebSockets.InitialBufferPoolSize); fileSystemLandlord = new FileSystemsLandlord(systemDatabase); databasesLandlord = new DatabasesLandlord(systemDatabase); countersLandlord = new CountersLandlord(systemDatabase); requestManager = new RequestManager(databasesLandlord); mixedModeRequestAuthorizer = new MixedModeRequestAuthorizer(); mixedModeRequestAuthorizer.Initialize(systemDatabase, new RavenServer(databasesLandlord.SystemDatabase, configuration)); serverStartupTasks = configuration.Container.GetExportedValues<IServerStartupTask>(); foreach (var task in serverStartupTasks) { toDispose.Add(task); task.Execute(this); } } catch { if (systemDatabase != null) systemDatabase.Dispose(); throw; } }
public HttpResponseMessage Compact() { var fs = InnerRequest.RequestUri.ParseQueryString()["filesystem"]; if (string.IsNullOrWhiteSpace(fs)) { return(GetMessageWithString("Compact request requires a valid filesystem parameter", HttpStatusCode.BadRequest)); } var configuration = FileSystemsLandlord.CreateTenantConfiguration(fs); if (configuration == null) { return(GetMessageWithString("No filesystem named: " + fs, HttpStatusCode.NotFound)); } var task = Task.Factory.StartNew(() => { // as we perform compact async we don't catch exceptions here - they will be propaged to operation var targetFs = FileSystemsLandlord.GetFileSystemInternal(fs).ResultUnwrap(); FileSystemsLandlord.Lock(fs, () => targetFs.Storage.Compact(configuration)); return(GetEmptyMessage()); }); long id; Database.Tasks.AddTask(task, new TaskBasedOperationState(task), new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.CompactFilesystem, Payload = "Compact filesystem " + fs, }, out id); return(GetMessageWithObject(new { OperationId = id })); }
private static HttpConfiguration CreateHttpCfg( DatabasesLandlord databasesLandlord, FileSystemsLandlord fileSystemsLandlord, MixedModeRequestAuthorizer mixedModeRequestAuthorizer, RequestManager requestManager) { var cfg = new HttpConfiguration(); cfg.Properties[typeof(DatabasesLandlord)] = databasesLandlord; cfg.Properties[typeof(FileSystemsLandlord)] = fileSystemsLandlord; cfg.Properties[typeof(MixedModeRequestAuthorizer)] = mixedModeRequestAuthorizer; cfg.Properties[typeof(RequestManager)] = requestManager; cfg.Formatters.Remove(cfg.Formatters.XmlFormatter); cfg.Formatters.JsonFormatter.SerializerSettings.Converters.Add(new NaveValueCollectionJsonConverterOnlyForConfigFormatters()); cfg.Services.Replace(typeof(IAssembliesResolver), new MyAssemblyResolver()); cfg.Filters.Add(new RavenExceptionFilterAttribute()); cfg.MapHttpAttributeRoutes(); cfg.Routes.MapHttpRoute( "RavenFs", "ravenfs/{controller}/{action}", new {id = RouteParameter.Optional}); cfg.Routes.MapHttpRoute( "API Default", "{controller}/{action}", new { id = RouteParameter.Optional }); cfg.Routes.MapHttpRoute( "Database Route", "databases/{databaseName}/{controller}/{action}", new { id = RouteParameter.Optional }); cfg.MessageHandlers.Add(new GZipToJsonAndCompressHandler()); cfg.Services.Replace(typeof(IHostBufferPolicySelector), new SelectiveBufferPolicySelector()); cfg.EnsureInitialized(); return cfg; }
private void CleanupFileSystems(DatabasesLandlord databaseLandlord, FileSystemsLandlord fileSystemLandlord) { var systemDatabase = databaseLandlord.SystemDatabase; int nextStart = 0; var fileSystemDocuments = systemDatabase .Documents .GetDocumentsWithIdStartingWith(Constants.FileSystem.Prefix, null, null, 0, int.MaxValue, CancellationToken.None, ref nextStart); var fileSystemIds = fileSystemDocuments .Select(x => ((RavenJObject)x)["@metadata"]) .Where(x => x != null) .Select(x => x.Value <string>("@id")) .Where(x => x != null && x != Constants.SystemDatabase) .ToList(); foreach (var fileSystemId in fileSystemIds) { try { var key = fileSystemId; if (key.StartsWith(Constants.FileSystem.Prefix)) { key = key.Substring(Constants.FileSystem.Prefix.Length); } var shouldCleanup = false; DateTime value; if (fileSystemLandlord.IsFileSystemLoaded(key) == false) { shouldCleanup = true; } else if (fileSystemLandlord.LastRecentlyUsed.TryGetValue(key, out value) == false || (SystemTime.UtcNow - value) > maxTimeResourceCanBeIdle) { shouldCleanup = true; } if (shouldCleanup == false) { continue; } var configuration = fileSystemLandlord.CreateTenantConfiguration(key, true); fileSystemLandlord.Cleanup(key, maxTimeResourceCanBeIdle, database => false); var docKey = Constants.FileSystem.Prefix + key; systemDatabase.Documents.Delete(docKey, null, null); if (configuration == null) { continue; } IOExtensions.DeleteDirectory(configuration.FileSystem.DataDirectory); } catch (Exception e) { log.WarnException(string.Format("Failed to cleanup '{0}' filesystem.", fileSystemId), e); } } }
public AdminLogsWebSocketsRequestParser(DatabasesLandlord databasesLandlord, CountersLandlord countersLandlord, FileSystemsLandlord fileSystemsLandlord, MixedModeRequestAuthorizer authorizer, string expectedRequestSuffix) : base(databasesLandlord, countersLandlord, fileSystemsLandlord, authorizer, expectedRequestSuffix) { }
public async Task <HttpResponseMessage> Restore() { if (EnsureSystemDatabase() == false) { return(GetMessageWithString("Restore is only possible from the system database", HttpStatusCode.BadRequest)); } var restoreStatus = new RestoreStatus { State = RestoreStatusState.Running, Messages = new List <string>() }; var restoreRequest = await ReadJsonObjectAsync <FilesystemRestoreRequest>(); var fileSystemDocumentPath = FindFilesystemDocument(restoreRequest.BackupLocation); if (!File.Exists(fileSystemDocumentPath)) { throw new InvalidOperationException("Cannot restore when the Filesystem.Document file is missing in the backup folder: " + restoreRequest.BackupLocation); } var filesystemDocumentText = File.ReadAllText(fileSystemDocumentPath); var filesystemDocument = RavenJObject.Parse(filesystemDocumentText).JsonDeserialization <FileSystemDocument>(); var filesystemName = !string.IsNullOrWhiteSpace(restoreRequest.FilesystemName) ? restoreRequest.FilesystemName : filesystemDocument == null ? null : filesystemDocument.Id; if (string.IsNullOrWhiteSpace(filesystemName)) { var errorMessage = (filesystemDocument == null || String.IsNullOrWhiteSpace(filesystemDocument.Id)) ? Constants.FilesystemDocumentFilename + " file is invalid - filesystem name was not found and not supplied in the request (Id property is missing or null). This is probably a bug - should never happen." : "A filesystem name must be supplied if the restore location does not contain a valid " + Constants.FilesystemDocumentFilename + " file"; restoreStatus.Messages.Add(errorMessage); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(new { restoreStatus }), new RavenJObject(), null); return(GetMessageWithString(errorMessage, HttpStatusCode.BadRequest)); } var ravenConfiguration = new RavenConfiguration { FileSystemName = filesystemName, }; if (filesystemDocument != null) { foreach (var setting in filesystemDocument.Settings) { ravenConfiguration.Settings[setting.Key] = setting.Value; } } if (Directory.Exists(Path.Combine(restoreRequest.BackupLocation, "new"))) { ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.EsentTypeName; } else { ravenConfiguration.FileSystem.DefaultStorageTypeName = InMemoryRavenConfiguration.VoronTypeName; } ravenConfiguration.CustomizeValuesForFileSystemTenant(filesystemName); ravenConfiguration.Initialize(); string documentDataDir; ravenConfiguration.FileSystem.DataDirectory = ResolveTenantDataDirectory(restoreRequest.FilesystemLocation, filesystemName, out documentDataDir); restoreRequest.FilesystemLocation = ravenConfiguration.FileSystem.DataDirectory; string anotherRestoreResourceName; if (IsAnotherRestoreInProgress(out anotherRestoreResourceName)) { if (restoreRequest.RestoreStartTimeout.HasValue) { try { using (var cts = new CancellationTokenSource()) { cts.CancelAfter(TimeSpan.FromSeconds(restoreRequest.RestoreStartTimeout.Value)); var token = cts.Token; do { await Task.Delay(500, token); }while (IsAnotherRestoreInProgress(out anotherRestoreResourceName)); } } catch (OperationCanceledException) { return(GetMessageWithString(string.Format("Another restore is still in progress (resource name = {0}). Waited {1} seconds for other restore to complete.", anotherRestoreResourceName, restoreRequest.RestoreStartTimeout.Value), HttpStatusCode.ServiceUnavailable)); } } else { return(GetMessageWithString(string.Format("Another restore is in progress (resource name = {0})", anotherRestoreResourceName), HttpStatusCode.ServiceUnavailable)); } } Database.Documents.Put(RestoreInProgress.RavenRestoreInProgressDocumentKey, null, RavenJObject.FromObject(new RestoreInProgress { Resource = filesystemName }), new RavenJObject(), null); DatabasesLandlord.SystemDatabase.Documents.Delete(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, null); bool defrag; if (bool.TryParse(GetQueryStringValue("defrag"), out defrag)) { restoreRequest.Defrag = defrag; } var task = Task.Factory.StartNew(() => { try { if (!string.IsNullOrWhiteSpace(restoreRequest.FilesystemLocation)) { ravenConfiguration.FileSystem.DataDirectory = restoreRequest.FilesystemLocation; } using (var transactionalStorage = RavenFileSystem.CreateTransactionalStorage(ravenConfiguration)) { transactionalStorage.Restore(restoreRequest, msg => { restoreStatus.Messages.Add(msg); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); }); } if (filesystemDocument == null) { return; } filesystemDocument.Settings[Constants.FileSystem.DataDirectory] = documentDataDir; if (restoreRequest.IndexesLocation != null) { filesystemDocument.Settings[Constants.RavenIndexPath] = restoreRequest.IndexesLocation; } if (restoreRequest.JournalsLocation != null) { filesystemDocument.Settings[Constants.RavenTxJournalPath] = restoreRequest.JournalsLocation; } filesystemDocument.Id = filesystemName; FileSystemsLandlord.Protect(filesystemDocument); DatabasesLandlord.SystemDatabase.Documents.Put(Constants.FileSystem.Prefix + filesystemName, null, RavenJObject.FromObject(filesystemDocument), new RavenJObject(), null); restoreStatus.State = RestoreStatusState.Completed; restoreStatus.Messages.Add("The new filesystem was created"); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); } catch (Exception e) { restoreStatus.State = RestoreStatusState.Faulted; restoreStatus.Messages.Add("Unable to restore filesystem " + e.Message); DatabasesLandlord.SystemDatabase.Documents.Put(RestoreStatus.RavenFilesystemRestoreStatusDocumentKey(filesystemName), null, RavenJObject.FromObject(restoreStatus), new RavenJObject(), null); throw; } finally { Database.Documents.Delete(RestoreInProgress.RavenRestoreInProgressDocumentKey, null, null); } }, TaskCreationOptions.LongRunning); long id; Database.Tasks.AddTask(task, new TaskBasedOperationState(task), new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.RestoreFilesystem, Payload = "Restoring filesystem " + filesystemName + " from " + restoreRequest.BackupLocation }, out id); return(GetMessageWithObject(new { OperationId = id })); }
public HttpResponseMessage Compact() { var fs = InnerRequest.RequestUri.ParseQueryString()["filesystem"]; if (string.IsNullOrWhiteSpace(fs)) { return(GetMessageWithString("Compact request requires a valid filesystem parameter", HttpStatusCode.BadRequest)); } var configuration = FileSystemsLandlord.CreateTenantConfiguration(fs); if (configuration == null) { return(GetMessageWithString("No filesystem named: " + fs, HttpStatusCode.NotFound)); } var task = Task.Factory.StartNew(() => { var compactStatus = new CompactStatus { State = CompactStatusState.Running, Messages = new List <string>() }; DatabasesLandlord.SystemDatabase.Documents.Delete(CompactStatus.RavenFilesystemCompactStatusDocumentKey(fs), null, null); try { // as we perform compact async we don't catch exceptions here - they will be propagated to operation var targetFs = FileSystemsLandlord.GetFileSystemInternal(fs).ResultUnwrap(); FileSystemsLandlord.Lock(fs, () => targetFs.Storage.Compact(configuration, msg => { bool skipProgressReport = false; bool isProgressReport = false; if (IsUpdateMessage(msg)) { isProgressReport = true; var now = SystemTime.UtcNow; compactStatus.LastProgressMessageTime = compactStatus.LastProgressMessageTime ?? DateTime.MinValue; var timeFromLastUpdate = (now - compactStatus.LastProgressMessageTime.Value); if (timeFromLastUpdate >= ReportProgressInterval) { compactStatus.LastProgressMessageTime = now; compactStatus.LastProgressMessage = msg; } else { skipProgressReport = true; } } if (!skipProgressReport) { if (!isProgressReport) { compactStatus.Messages.Add(msg); } DatabasesLandlord.SystemDatabase.Documents.Put(CompactStatus.RavenFilesystemCompactStatusDocumentKey(fs), null, RavenJObject.FromObject(compactStatus), new RavenJObject(), null); } })); compactStatus.State = CompactStatusState.Completed; compactStatus.Messages.Add("File system compaction completed."); DatabasesLandlord.SystemDatabase.Documents.Put(CompactStatus.RavenFilesystemCompactStatusDocumentKey(fs), null, RavenJObject.FromObject(compactStatus), new RavenJObject(), null); } catch (Exception e) { compactStatus.Messages.Add("Unable to compact file system " + e.Message); compactStatus.State = CompactStatusState.Faulted; DatabasesLandlord.SystemDatabase.Documents.Put(CompactStatus.RavenFilesystemCompactStatusDocumentKey(fs), null, RavenJObject.FromObject(compactStatus), new RavenJObject(), null); throw; } return(GetEmptyMessage()); }); long id; Database.Tasks.AddTask(task, new TaskBasedOperationState(task), new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.CompactFilesystem, Payload = "Compact filesystem " + fs, }, out id); return(GetMessageWithObject(new { OperationId = id })); }
public async Task <HttpResponseMessage> Backup() { var backupRequest = await ReadJsonObjectAsync <FilesystemBackupRequest>(); var incrementalString = InnerRequest.RequestUri.ParseQueryString()["incremental"]; bool incrementalBackup; if (bool.TryParse(incrementalString, out incrementalBackup) == false) { incrementalBackup = false; } if (backupRequest.FileSystemDocument == null && FileSystem.Name != null) { var jsonDocument = DatabasesLandlord.SystemDatabase.Documents.Get("Raven/FileSystems/" + FileSystem.Name, null); if (jsonDocument != null) { backupRequest.FileSystemDocument = jsonDocument.DataAsJson.JsonDeserialization <FileSystemDocument>(); FileSystemsLandlord.Unprotect(backupRequest.FileSystemDocument); backupRequest.FileSystemDocument.Id = FileSystem.Name; } } var transactionalStorage = FileSystem.Storage; var filesystemDocument = backupRequest.FileSystemDocument; var backupDestinationDirectory = backupRequest.BackupLocation; RavenJObject document = null; try { FileSystem.Storage.Batch(accessor => document = accessor.GetConfig(BackupStatus.RavenBackupStatusDocumentKey)); } catch (FileNotFoundException) { // ok, there isn't another backup in progress } if (document != null) { var backupStatus = document.JsonDeserialization <BackupStatus>(); if (backupStatus.IsRunning) { throw new InvalidOperationException("Backup is already running"); } } bool enableIncrementalBackup; if (incrementalBackup && transactionalStorage is Storage.Esent.TransactionalStorage && (bool.TryParse(Database.Configuration.Settings["Raven/Esent/CircularLog"], out enableIncrementalBackup) == false || enableIncrementalBackup)) { throw new InvalidOperationException("In order to run incremental backups using Esent you must have circular logging disabled"); } if (incrementalBackup && transactionalStorage is Storage.Voron.TransactionalStorage && Database.Configuration.Storage.Voron.AllowIncrementalBackups == false) { throw new InvalidOperationException("In order to run incremental backups using Voron you must have the appropriate setting key (Raven/Voron/AllowIncrementalBackups) set to true"); } FileSystem.Storage.Batch(accessor => accessor.SetConfig(BackupStatus.RavenBackupStatusDocumentKey, RavenJObject.FromObject(new BackupStatus { Started = SystemTime.UtcNow, IsRunning = true, }))); if (filesystemDocument.Settings.ContainsKey(Constants.FileSystem.Storage) == false) { filesystemDocument.Settings[Constants.FileSystem.Storage] = transactionalStorage.FriendlyName.ToLower() ?? transactionalStorage.GetType().AssemblyQualifiedName; } transactionalStorage.StartBackupOperation(DatabasesLandlord.SystemDatabase, FileSystem, backupDestinationDirectory, incrementalBackup, filesystemDocument); return(GetEmptyMessage(HttpStatusCode.Created)); }
public async Task <HttpResponseMessage> Backup() { var backupRequest = await ReadJsonObjectAsync <FilesystemBackupRequest>().ConfigureAwait(false); var incrementalString = InnerRequest.RequestUri.ParseQueryString()["incremental"]; bool incrementalBackup; if (bool.TryParse(incrementalString, out incrementalBackup) == false) { incrementalBackup = false; } if (backupRequest.FileSystemDocument == null && FileSystem.Name != null) { var jsonDocument = DatabasesLandlord.SystemDatabase.Documents.Get(Constants.FileSystem.Prefix + FileSystem.Name, null); if (jsonDocument != null) { backupRequest.FileSystemDocument = jsonDocument.DataAsJson.JsonDeserialization <FileSystemDocument>(); FileSystemsLandlord.Unprotect(backupRequest.FileSystemDocument); backupRequest.FileSystemDocument.Id = FileSystem.Name; } } var transactionalStorage = FileSystem.Storage; var filesystemDocument = backupRequest.FileSystemDocument; var backupDestinationDirectory = backupRequest.BackupLocation; RavenJObject document = null; try { FileSystem.Storage.Batch(accessor => document = accessor.GetConfig(BackupStatus.RavenBackupStatusDocumentKey)); } catch (FileNotFoundException) { // ok, there isn't another backup in progress } if (document != null) { var backupStatus = document.JsonDeserialization <BackupStatus>(); if (backupStatus.IsRunning) { throw new InvalidOperationException("Backup is already running"); } } HttpResponseMessage message; if (!HasPermissions(backupDestinationDirectory, out message)) { return(message); } bool enableIncrementalBackup; if (incrementalBackup && transactionalStorage is Storage.Esent.TransactionalStorage && (bool.TryParse(FileSystem.Configuration.Settings["Raven/Esent/CircularLog"], out enableIncrementalBackup) == false || enableIncrementalBackup)) { throw new InvalidOperationException("In order to run incremental backups using Esent you must have circular logging disabled"); } if (incrementalBackup && transactionalStorage is Storage.Voron.TransactionalStorage && FileSystem.Configuration.Storage.Voron.AllowIncrementalBackups == false) { throw new InvalidOperationException("In order to run incremental backups using Voron you must have the appropriate setting key (Raven/Voron/AllowIncrementalBackups) set to true"); } FileSystem.Storage.Batch(accessor => accessor.SetConfig(BackupStatus.RavenBackupStatusDocumentKey, RavenJObject.FromObject(new BackupStatus { Started = SystemTime.UtcNow, IsRunning = true, }))); if (filesystemDocument.Settings.ContainsKey(Constants.FileSystem.Storage) == false) { filesystemDocument.Settings[Constants.FileSystem.Storage] = transactionalStorage.FriendlyName.ToLower() ?? transactionalStorage.GetType().AssemblyQualifiedName; } var cts = new CancellationTokenSource(); var state = new ResourceBackupState(); var task = transactionalStorage.StartBackupOperation(DatabasesLandlord.SystemDatabase, FileSystem, backupDestinationDirectory, incrementalBackup, filesystemDocument, state, cts.Token); task.ContinueWith(_ => cts.Dispose()); long id; SystemDatabase.Tasks.AddTask(task, state, new TaskActions.PendingTaskDescription { StartTime = SystemTime.UtcNow, TaskType = TaskActions.PendingTaskType.BackupFilesystem, Description = "Backup to: " + backupRequest.BackupLocation }, out id, cts); return(GetMessageWithObject(new { OperationId = id }, HttpStatusCode.Accepted)); }
public HttpResponseMessage ResourceDrives(string name, string type) { ResourceType resourceType; if (Enum.TryParse(type, out resourceType) == false) { return(GetMessageWithString("Unknown resourceType:" + type, HttpStatusCode.BadRequest)); } string[] drives = null; InMemoryRavenConfiguration config; switch (resourceType) { case ResourceType.Database: config = DatabasesLandlord.CreateTenantConfiguration(name); if (config == null) { return(GetMessageWithString("Unable to find database named: " + name, HttpStatusCode.NotFound)); } drives = FindUniqueDrives(new[] { config.IndexStoragePath, config.Storage.Esent.JournalsStoragePath, config.Storage.Voron.JournalsStoragePath, config.DataDirectory }); break; case ResourceType.FileSystem: config = FileSystemsLandlord.CreateTenantConfiguration(name); if (config == null) { return(GetMessageWithString("Unable to find filesystem named: " + name, HttpStatusCode.NotFound)); } drives = FindUniqueDrives(new[] { config.FileSystem.DataDirectory, config.FileSystem.IndexStoragePath, config.Storage.Esent.JournalsStoragePath, config.Storage.Voron.JournalsStoragePath }); break; case ResourceType.Counter: config = CountersLandlord.CreateTenantConfiguration(name); if (config == null) { return(GetMessageWithString("Unable to find counter named: " + name, HttpStatusCode.NotFound)); } drives = FindUniqueDrives(new[] { config.Counter.DataDirectory, config.Storage.Esent.JournalsStoragePath, config.Storage.Voron.JournalsStoragePath, config.DataDirectory }); break; case ResourceType.TimeSeries: config = TimeSeriesLandlord.CreateTenantConfiguration(name); if (config == null) { return(GetMessageWithString("Unable to find time series named: " + name, HttpStatusCode.NotFound)); } drives = FindUniqueDrives(new[] { config.TimeSeries.DataDirectory, config.Storage.Esent.JournalsStoragePath, config.Storage.Voron.JournalsStoragePath, config.DataDirectory }); break; } return(GetMessageWithObject(drives)); }
public WatchTrafficWebSocketsRequestParser(DatabasesLandlord databasesLandlord, TimeSeriesLandlord timeSeriesLandlord, CountersLandlord countersLandlord, FileSystemsLandlord fileSystemsLandlord, MixedModeRequestAuthorizer authorizer, string expectedRequestSuffix) : base(databasesLandlord, timeSeriesLandlord, countersLandlord, fileSystemsLandlord, authorizer, expectedRequestSuffix) { }