public override Task <DocumentDatabase> TryGetOrCreateResourceStore(StringSegment databaseName)
{
if (HasLocks != 0)
{
AssertLocks(databaseName);
}
Task <DocumentDatabase> database;
if (ResourcesStoresCache.TryGetValue(databaseName, out database))
{
if (database.IsFaulted || database.IsCanceled)
{
ResourcesStoresCache.TryRemove(databaseName, out database);
DateTime time;
LastRecentlyUsed.TryRemove(databaseName, out time);
// and now we will try creating it again
}
else
{
return(database);
}
}
return(CreateDatabase(databaseName));
}
private DocumentDatabase CreateDocumentsStorage(StringSegment databaseName, RavenConfiguration config)
{
try
{
var sp = Stopwatch.StartNew();
var documentDatabase = new DocumentDatabase(config.DatabaseName, config, ServerStore);
documentDatabase.Initialize();
DeleteDatabaseCachedInfo(documentDatabase, ServerStore);
if (Logger.IsInfoEnabled)
{
Logger.Info($"Started database {config.DatabaseName} in {sp.ElapsedMilliseconds:#,#;;0}ms");
}
OnDatabaseLoaded(config.DatabaseName);
// if we have a very long init process, make sure that we reset the last idle time for this db.
LastRecentlyUsed.AddOrUpdate(databaseName, SystemTime.UtcNow, (_, time) => SystemTime.UtcNow);
return(documentDatabase);
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to start database {config.DatabaseName}", e);
}
throw;
}
}
public bool TryGetOrCreateResourceStore(string tenantId, out Task <DocumentDatabase> database)
{
if (Locks.Contains(DisposingLock))
{
throw new ObjectDisposedException("DatabaseLandlord", "Server is shutting down, can't access any databases");
}
if (ResourcesStoresCache.TryGetValue(tenantId, out database))
{
if (database.IsFaulted || database.IsCanceled)
{
ResourcesStoresCache.TryRemove(tenantId, out database);
DateTime time;
LastRecentlyUsed.TryRemove(tenantId, out time);
// and now we will try creating it again
}
else
{
return(true);
}
}
if (Locks.Contains(tenantId))
{
throw new InvalidOperationException("Database '" + tenantId + "' is currently locked and cannot be accessed.");
}
var config = CreateTenantConfiguration(tenantId);
if (config == null)
{
return(false);
}
database = ResourcesStoresCache.GetOrAdd(tenantId, __ => Task.Factory.StartNew(() =>
{
var transportState = ResourseTransportStates.GetOrAdd(tenantId, s => new TransportState());
AssertLicenseParameters(config);
var documentDatabase = new DocumentDatabase(config, transportState);
documentDatabase.SpinBackgroundWorkers();
documentDatabase.Disposing += DocumentDatabaseDisposingStarted;
documentDatabase.DisposingEnded += DocumentDatabaseDisposingEnded;
documentDatabase.StorageInaccessible += UnloadDatabaseOnStorageInaccessible;
// if we have a very long init process, make sure that we reset the last idle time for this db.
LastRecentlyUsed.AddOrUpdate(tenantId, SystemTime.UtcNow, (_, time) => SystemTime.UtcNow);
return(documentDatabase);
}).ContinueWith(task =>
{
if (task.Status == TaskStatus.Faulted) // this observes the task exception
{
Logger.WarnException("Failed to create database " + tenantId, task.Exception);
}
return(task);
}).Unwrap());
return(true);
}
public bool TryGetOrCreateResourceStore(string tenantId, out Task <CounterStorage> counter)
{
if (Locks.Contains(DisposingLock))
{
throw new ObjectDisposedException("CountersLandlord", "Server is shutting down, can't access any counters");
}
if (Enabled == false)
{
throw new InvalidOperationException("Counters are an experimental feature that is not enabled");
}
if (ResourcesStoresCache.TryGetValue(tenantId, out counter))
{
if (counter.IsFaulted || counter.IsCanceled)
{
ResourcesStoresCache.TryRemove(tenantId, out counter);
DateTime time;
LastRecentlyUsed.TryRemove(tenantId, out time);
// and now we will try creating it again
}
else
{
return(true);
}
}
if (Locks.Contains(tenantId))
{
throw new InvalidOperationException("Counters '" + tenantId + "' is currently locked and cannot be accessed");
}
var config = CreateTenantConfiguration(tenantId);
if (config == null)
{
return(false);
}
counter = ResourcesStoresCache.GetOrAdd(tenantId, __ => Task.Factory.StartNew(() =>
{
var transportState = ResourseTransportStates.GetOrAdd(tenantId, s => new TransportState());
var cs = new CounterStorage(systemDatabase.ServerUrl, tenantId, config, transportState);
AssertLicenseParameters();
// if we have a very long init process, make sure that we reset the last idle time for this db.
LastRecentlyUsed.AddOrUpdate(tenantId, SystemTime.UtcNow, (_, time) => SystemTime.UtcNow);
return(cs);
}).ContinueWith(task =>
{
if (task.Status == TaskStatus.Faulted) // this observes the task exception
{
Logger.WarnException("Failed to create counters " + tenantId, task.Exception);
}
return(task);
}).Unwrap());
return(true);
}
public bool TryGetOrCreateResourceStore(string tenantId, out Task <RavenFileSystem> fileSystem)
{
if (Locks.Contains(DisposingLock))
{
throw new ObjectDisposedException("FileSystem", "Server is shutting down, can't access any file systems");
}
if (ResourcesStoresCache.TryGetValue(tenantId, out fileSystem))
{
if (fileSystem.IsFaulted || fileSystem.IsCanceled)
{
ResourcesStoresCache.TryRemove(tenantId, out fileSystem);
DateTime time;
LastRecentlyUsed.TryRemove(tenantId, out time);
// and now we will try creating it again
}
else
{
return(true);
}
}
if (Locks.Contains(tenantId))
{
throw new InvalidOperationException("FileSystem '" + tenantId + "' is currently locked and cannot be accessed");
}
var config = CreateTenantConfiguration(tenantId);
if (config == null)
{
return(false);
}
fileSystem = ResourcesStoresCache.GetOrAdd(tenantId, __ => Task.Factory.StartNew(() =>
{
var transportState = ResourseTransportStates.GetOrAdd(tenantId, s => new TransportState());
AssertLicenseParameters(config);
var fs = new RavenFileSystem(config, tenantId, transportState);
fs.Initialize();
// if we have a very long init process, make sure that we reset the last idle time for this db.
LastRecentlyUsed.AddOrUpdate(tenantId, SystemTime.UtcNow, (_, time) => SystemTime.UtcNow);
return(fs);
}).ContinueWith(task =>
{
if (task.Status == TaskStatus.Faulted) // this observes the task exception
{
Logger.WarnException("Failed to create filesystem " + tenantId, task.Exception);
}
return(task);
}).Unwrap());
return(true);
}
public bool TryGetDatabase(string tenantId, out Task <DocumentDatabase> database)
{
if (Locks.Contains(DisposingLock))
{
throw new ObjectDisposedException("DatabaseLandlord", "Server is shutting down, can't access any databases");
}
if (!ResourcesStoresCache.TryGetValue(tenantId, out database))
{
return(false);
}
if (!database.IsFaulted && !database.IsCanceled)
{
return(true);
}
ResourcesStoresCache.TryRemove(tenantId, out database);
DateTime time;
LastRecentlyUsed.TryRemove(tenantId, out time);
return(false);
}
public override bool TryGetOrCreateResourceStore(string resourceName, out Task <DocumentDatabase> resourceTask)
{
if (Locks.Contains(DisposingLock))
{
throw new ObjectDisposedException("DatabaseLandlord", "Server is shutting down, can't access any databases");
}
if (Locks.Contains(resourceName))
{
throw new InvalidOperationException("Database '" + resourceName + "' is currently locked and cannot be accessed.");
}
ManualResetEvent cleanupLock;
if (Cleanups.TryGetValue(resourceName, out cleanupLock) && cleanupLock.WaitOne(MaxSecondsForTaskToWaitForDatabaseToLoad * 1000) == false)
{
throw new InvalidOperationException(string.Format("Database '{0}' is currently being restarted and cannot be accessed. We already waited {1} seconds.", resourceName, MaxSecondsForTaskToWaitForDatabaseToLoad));
}
if (ResourcesStoresCache.TryGetValue(resourceName, out resourceTask))
{
if (resourceTask.IsFaulted || resourceTask.IsCanceled)
{
ResourcesStoresCache.TryRemove(resourceName, out resourceTask);
DateTime time;
LastRecentlyUsed.TryRemove(resourceName, out time);
// and now we will try creating it again
}
else
{
return(true);
}
}
var config = CreateTenantConfiguration(resourceName);
if (config == null)
{
return(false);
}
if (!ResourceSemaphore.Wait(ConcurrentResourceLoadTimeout))
{
throw new ConcurrentLoadTimeoutException("Too much databases loading concurrently, timed out waiting for them to load.");
}
var didReleaseDuringWait = 0;
var createdNewTask = false;
resourceTask = ResourcesStoresCache.GetOrAdd(resourceName, __ =>
{
createdNewTask = true;
return(Task.Factory.StartNew(() =>
{
var transportState = ResourseTransportStates.GetOrAdd(resourceName, s => new TransportState());
AssertLicenseParameters(config);
var documentDatabase = new DocumentDatabase(config, systemDatabase, transportState);
documentDatabase.SpinBackgroundWorkers(false);
documentDatabase.Disposing += DocumentDatabaseDisposingStarted;
documentDatabase.DisposingEnded += DocumentDatabaseDisposingEnded;
documentDatabase.StorageInaccessible += UnloadDatabaseOnStorageInaccessible;
// register only DB that has incremental backup set.
documentDatabase.OnBackupComplete += OnDatabaseBackupCompleted;
// if we have a very long init process, make sure that we reset the last idle time for this db.
LastRecentlyUsed.AddOrUpdate(resourceName, SystemTime.UtcNow, (_, time) => SystemTime.UtcNow);
documentDatabase.RequestManager = SystemDatabase.RequestManager;
documentDatabase.ClusterManager = SystemDatabase.ClusterManager;
return documentDatabase;
}).ContinueWith(task =>
{
if (task.Status == TaskStatus.RanToCompletion)
{
OnDatabaseLoaded(resourceName);
}
if (task.Status == TaskStatus.Faulted) // this observes the task exception
{
Logger.WarnException("Failed to create database " + resourceName, task.Exception);
}
ResourceSemaphore.Release();
return task;
}).Unwrap());
});
if (createdNewTask == false)
{
ResourceSemaphore.Release();
}
if (resourceTask.IsFaulted && resourceTask.Exception != null)
{
// if we are here, there is an error, and if there is an error, we need to clear it from the
// resource store cache so we can try to reload it.
// Note that we return the faulted task anyway, because we need the user to look at the error
if (resourceTask.Exception.Data.Contains("Raven/KeepInResourceStore") == false)
{
Task <DocumentDatabase> val;
ResourcesStoresCache.TryRemove(resourceName, out val);
}
}
return(true);
}
public bool TryGetOrCreateResourceStore(string tenantId, out Task <RavenFileSystem> fileSystem)
{
if (Locks.Contains(DisposingLock))
{
throw new ObjectDisposedException("FileSystem", "Server is shutting down, can't access any file systems");
}
if (Locks.Contains(tenantId))
{
throw new InvalidOperationException("FileSystem '" + tenantId + "' is currently locked and cannot be accessed");
}
ManualResetEvent cleanupLock;
if (Cleanups.TryGetValue(tenantId, out cleanupLock) && cleanupLock.WaitOne(MaxSecondsForTaskToWaitForDatabaseToLoad * 1000) == false)
{
throw new InvalidOperationException(string.Format("File system '{0}' is currently being restarted and cannot be accessed. We already waited {1} seconds.", tenantId, MaxSecondsForTaskToWaitForDatabaseToLoad));
}
if (ResourcesStoresCache.TryGetValue(tenantId, out fileSystem))
{
if (fileSystem.IsFaulted || fileSystem.IsCanceled)
{
ResourcesStoresCache.TryRemove(tenantId, out fileSystem);
DateTime time;
LastRecentlyUsed.TryRemove(tenantId, out time);
// and now we will try creating it again
}
else
{
return(true);
}
}
var config = CreateTenantConfiguration(tenantId);
if (config == null)
{
return(false);
}
var hasAcquired = false;
try
{
if (!ResourceSemaphore.Wait(ConcurrentResourceLoadTimeout))
{
throw new ConcurrentLoadTimeoutException("Too much filesystems loading concurrently, timed out waiting for them to load.");
}
hasAcquired = true;
fileSystem = ResourcesStoresCache.GetOrAdd(tenantId, __ => Task.Factory.StartNew(() =>
{
var transportState = ResourseTransportStates.GetOrAdd(tenantId, s => new TransportState());
AssertLicenseParameters(config);
var fs = new RavenFileSystem(config, tenantId, transportState);
fs.Initialize();
// if we have a very long init process, make sure that we reset the last idle time for this db.
LastRecentlyUsed.AddOrUpdate(tenantId, SystemTime.UtcNow, (_, time) => SystemTime.UtcNow);
return(fs);
}).ContinueWith(task =>
{
if (task.Status == TaskStatus.Faulted) // this observes the task exception
{
Logger.WarnException("Failed to create filesystem " + tenantId, task.Exception);
}
return(task);
}).Unwrap());
return(true);
}
finally
{
if (hasAcquired)
{
ResourceSemaphore.Release();
}
}
}
public void Cleanup(string resource,
TimeSpan?skipIfActiveInDuration,
Func <TResource, bool> shouldSkip = null,
DocumentChangeTypes notificationType = DocumentChangeTypes.None)
{
if (Cleanups.TryAdd(resource, new ManualResetEvent(false)) == false)
{
return;
}
try
{
using (ResourcesStoresCache.WithAllLocks())
{
DateTime time;
Task <TResource> resourceTask;
if (ResourcesStoresCache.TryGetValue(resource, out resourceTask) == false)
{
LastRecentlyUsed.TryRemove(resource, out time);
return;
}
if (resourceTask.Status == TaskStatus.Faulted || resourceTask.Status == TaskStatus.Canceled)
{
LastRecentlyUsed.TryRemove(resource, out time);
ResourcesStoresCache.TryRemove(resource, out resourceTask);
return;
}
if (resourceTask.Status != TaskStatus.RanToCompletion)
{
return; // still starting up
}
var database = resourceTask.Result;
if ((skipIfActiveInDuration != null && (SystemTime.UtcNow - LastWork(database)) < skipIfActiveInDuration) ||
(shouldSkip != null && shouldSkip(database)))
{
// this document might not be actively working with user, but it is actively doing indexes, we will
// wait with unloading this database until it hasn't done indexing for a while.
// This prevent us from shutting down big databases that have been left alone to do indexing work.
return;
}
try
{
database.Dispose();
}
catch (Exception e)
{
Logger.ErrorException("Could not cleanup tenant database: " + resource, e);
return;
}
LastRecentlyUsed.TryRemove(resource, out time);
ResourcesStoresCache.TryRemove(resource, out resourceTask);
if (notificationType == DocumentChangeTypes.Delete)
{
TransportState transportState;
ResourseTransportStates.TryRemove(resource, out transportState);
if (transportState != null)
{
transportState.Dispose();
}
}
var onResourceCleanupOccured = CleanupOccured;
if (onResourceCleanupOccured != null)
{
onResourceCleanupOccured(resource);
}
}
}
finally
{
ManualResetEvent cleanupLock;
if (Cleanups.TryRemove(resource, out cleanupLock))
{
cleanupLock.Set();
}
}
}
public bool TryGetOrCreateResourceStore(string tenantId, out Task <DocumentDatabase> database)
{
if (Locks.Contains(DisposingLock))
{
throw new ObjectDisposedException("DatabaseLandlord", "Server is shutting down, can't access any databases");
}
if (ResourcesStoresCache.TryGetValue(tenantId, out database))
{
if (database.IsFaulted || database.IsCanceled)
{
ResourcesStoresCache.TryRemove(tenantId, out database);
DateTime time;
LastRecentlyUsed.TryRemove(tenantId, out time);
// and now we will try creating it again
}
else
{
return(true);
}
}
if (Locks.Contains(tenantId))
{
throw new InvalidOperationException("Database '" + tenantId + "' is currently locked and cannot be accessed.");
}
var config = CreateTenantConfiguration(tenantId);
if (config == null)
{
return(false);
}
database = ResourcesStoresCache.GetOrAdd(tenantId, __ => Task.Factory.StartNew(() =>
{
var transportState = ResourseTransportStates.GetOrAdd(tenantId, s => new TransportState());
AssertLicenseParameters(config);
var documentDatabase = new DocumentDatabase(config, transportState);
documentDatabase.SpinBackgroundWorkers();
documentDatabase.Disposing += DocumentDatabaseDisposingStarted;
documentDatabase.DisposingEnded += DocumentDatabaseDisposingEnded;
documentDatabase.StorageInaccessible += UnloadDatabaseOnStorageInaccessible;
// register only DB that has incremental backup set.
documentDatabase.OnBackupComplete += OnDatabaseBackupCompleted;
// if we have a very long init process, make sure that we reset the last idle time for this db.
LastRecentlyUsed.AddOrUpdate(tenantId, SystemTime.UtcNow, (_, time) => SystemTime.UtcNow);
return(documentDatabase);
}).ContinueWith(task =>
{
if (task.Status == TaskStatus.Faulted) // this observes the task exception
{
Logger.WarnException("Failed to create database " + tenantId, task.Exception);
}
return(task);
}).Unwrap());
if (database.IsFaulted && database.Exception != null)
{
// if we are here, there is an error, and if there is an error, we need to clear it from the
// resource store cache so we can try to reload it.
// Note that we return the faulted task anyway, because we need the user to look at the error
if (database.Exception.Data.Contains("Raven/KeepInResourceStore") == false)
{
Task <DocumentDatabase> val;
ResourcesStoresCache.TryRemove(tenantId, out val);
}
}
return(true);
}
