public HttpListenerContextAdpater(HttpListenerContext ctx, RavenConfiguration configuration)
{
this.ctx = ctx;
this.configuration = configuration;
Request = new HttpListenerRequestAdapter(ctx.Request);
ResponseInternal = new HttpListenerResponseAdapter(ctx.Response);
}
public static void Init()
{
if (database != null)
return;
lock (locker)
{
if (database != null)
return;
try
{
var ravenConfiguration = new RavenConfiguration();
HttpEndpointRegistration.RegisterHttpEndpointTarget();
database = new DocumentDatabase(ravenConfiguration);
database.SpinBackgroundWorkers();
server = new HttpServer(ravenConfiguration, database);
server.Init();
}
catch
{
if (database != null)
{
database.Dispose();
database = null;
}
if (server != null)
{
server.Dispose();
server = null;
}
throw;
}
}
}
public TcpHttpContext(TcpClient client, RavenConfiguration configuration)
{
_client = client;
_configuration = configuration;
_request = new TcpHttpRequest(_client, configuration.Port);
_response = new TcpHttpResponse(_client);
}
public HttpContextAdapter(HttpContext context, RavenConfiguration configuration)
{
this.context = context;
this.configuration = configuration;
request = new HttpRequestAdapter(context.Request);
response = new HttpResponseAdapter(context.Response);
}
private DocumentDatabase CreateDocumentDatabase()
{
var configuration = new RavenConfiguration();
configuration.DataDirectory = Path.Combine(NewDataPath(), "System");
configuration.RunInMemory = configuration.DefaultStorageTypeName == InMemoryRavenConfiguration.VoronTypeName;
return new DocumentDatabase(configuration);
}
protected override void ConfigureServer(RavenConfiguration ravenConfiguration)
{
ravenConfiguration.AnonymousUserAccessMode = AnonymousUserAccessMode.None;
ravenConfiguration.AuthenticationMode = "OAuth";
ravenConfiguration.OAuthTokenCertificate = CertGenerator.GenerateNewCertificate("RavenDB.Test");
ravenConfiguration.Catalog.Catalogs.Add(new TypeCatalog(typeof(FakeAuthenticateClient)));
}
public static void Init()
{
if (database != null)
return;
lock (locker)
{
if (database != null)
return;
try
{
var ravenConfiguration = new RavenConfiguration();
if (RoleEnvironment.IsAvailable)
{
ravenConfiguration.RunInMemory = true;
// Mount Cloud drive and set it as Data Directory
//var currentConfiguredRavenDataDir = ConfigurationManager.AppSettings["Raven/DataDir"] ?? string.Empty;
//string azureDrive = @"D:\"; // Environment.GetEnvironmentVariable(RavenDriveConfiguration.AzureDriveEnvironmentVariableName, EnvironmentVariableTarget.Machine);
//if (string.IsNullOrWhiteSpace(azureDrive))
//{
// throw new ArgumentException("RavenDb drive environment variable is not yet set by worker role. Please, retry in a couple of seconds");
//}
//string azurePath = Path.Combine(azureDrive,
// currentConfiguredRavenDataDir.StartsWith(@"~\")
// ? currentConfiguredRavenDataDir.Substring(2)
// : "Data");
//ravenConfiguration.DataDirectory = azurePath;
// Read port number specified for this Raven instance and set it in configuration
var endpoint = RoleEnvironment.CurrentRoleInstance.InstanceEndpoints["Raven"];
ravenConfiguration.Port = endpoint.IPEndpoint.Port;
// When mounting drives in emulator only Munin storage is supported, since drive is not actually present and low level access to it failes (Esent mode)
}
HttpEndpointRegistration.RegisterHttpEndpointTarget();
database = new DocumentDatabase(ravenConfiguration);
database.SpinBackgroundWorkers();
server = new HttpServer(ravenConfiguration, database);
server.Init();
}
catch
{
if (database != null)
{
database.Dispose();
database = null;
}
if (server != null)
{
server.Dispose();
server = null;
}
throw;
}
HostingEnvironment.RegisterObject(new ReleaseRavenDBWhenAppDomainIsTornDown());
}
}
public override bool OnStart()
{
// Set the maximum number of concurrent connections
ServicePointManager.DefaultConnectionLimit = 12;
// For information on handling configuration changes
// see the MSDN topic at http://go.microsoft.com/fwlink/?LinkId=166357.
var drive = MountCloudDrive();
var config = new RavenConfiguration
{
DataDirectory = drive,
AnonymousUserAccessMode = AnonymousUserAccessMode.All,
HttpCompression = true,
DefaultStorageTypeName = "munin",
Port = MyInstanceEndpoint.IPEndpoint.Port,
PluginsDirectory = "plugins"
};
StartRaven(config);
SetupReplication();
RoleEnvironment.Changed += RoleEnvironmentChanged;
RoleEnvironment.StatusCheck += RoleEnvironmentStatusCheck;
return base.OnStart();
}
public EmbeddableDocumentStore()
{
Conventions = new DocumentConvention();
Listeners = new DocumentSessionListeners();
Configuration = new RavenConfiguration();
EnlistInDistributedTransactions = true;
}
protected RavenDbServer CreateRavenDbServer(int port,
string dataDirectory = null,
bool runInMemory = true,
string requestedStorage = null,
bool enableAuthentication = false,
string fileSystemName = null,
Action<RavenConfiguration> customConfig = null)
{
NonAdminHttp.EnsureCanListenToWhenInNonAdminContext(port);
var storageType = GetDefaultStorageType(requestedStorage);
var directory = dataDirectory ?? NewDataPath(fileSystemName + "_" + port);
var ravenConfiguration = new RavenConfiguration()
{
Port = port,
DataDirectory = directory,
RunInMemory = storageType.Equals("esent", StringComparison.OrdinalIgnoreCase) == false && runInMemory,
#if DEBUG
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = runInMemory,
#endif
DefaultStorageTypeName = storageType,
AnonymousUserAccessMode = enableAuthentication ? AnonymousUserAccessMode.None : AnonymousUserAccessMode.Admin,
};
ravenConfiguration.Encryption.UseFips = SettingsHelper.UseFipsEncryptionAlgorithms;
ravenConfiguration.FileSystem.MaximumSynchronizationInterval = this.SynchronizationInterval;
ravenConfiguration.FileSystem.DataDirectory = Path.Combine(directory, "FileSystem");
ravenConfiguration.FileSystem.DefaultStorageTypeName = storageType;
if (customConfig != null)
{
customConfig(ravenConfiguration);
}
if (enableAuthentication)
{
Authentication.EnableOnce();
}
var ravenDbServer = new RavenDbServer(ravenConfiguration)
{
UseEmbeddedHttpServer = true
};
ravenDbServer.Initialize();
servers.Add(ravenDbServer);
if (enableAuthentication)
{
EnableAuthentication(ravenDbServer.SystemDatabase);
}
ConfigureServer(ravenDbServer, fileSystemName);
return ravenDbServer;
}
public HttpServer(RavenConfiguration configuration, DocumentDatabase database)
{
defaultDatabase = database;
defaultConfiguration = configuration;
configuration.Container.SatisfyImportsOnce(this);
foreach (var requestResponder in RequestResponders)
{
requestResponder.Initialize(() => currentDatabase.Value, () => currentConfiguration.Value);
}
}
public HttpServer(RavenConfiguration configuration, DocumentDatabase database)
{
Configuration = configuration;
configuration.Container.SatisfyImportsOnce(this);
foreach (var requestResponder in RequestResponders)
{
requestResponder.Database = database;
requestResponder.Settings = configuration;
}
}
public TransactionalStorage(RavenConfiguration configuration, Action onCommit)
{
database = configuration.DataDirectory;
this.configuration = configuration;
this.onCommit = onCommit;
path = database;
if (Path.IsPathRooted(database) == false)
path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, database);
database = Path.Combine(path, "Data");
LimitSystemCache();
Api.JetCreateInstance(out instance, database + Guid.NewGuid());
}
public static void Init()
{
if (database != null)
return;
lock (locker)
{
if (database != null)
return;
var ravenConfiguration = new RavenConfiguration();
HttpServer.RegisterHttpEndpointTarget();
database = new DocumentDatabase(ravenConfiguration);
database.SpinBackgroundWorkers();
server = new HttpServer(ravenConfiguration, database);
}
}
protected RavenDbServer CreateRavenDbServer(int port,
string dataDirectory = null,
bool runInMemory = true,
string requestedStorage = null,
bool enableAuthentication = false,
string fileSystemName = null)
{
var storageType = GetDefaultStorageType(requestedStorage);
var directory = dataDirectory ?? NewDataPath(fileSystemName + "_" + port);
var ravenConfiguration = new RavenConfiguration()
{
Port = port,
DataDirectory = directory,
FileSystemDataDirectory = Path.Combine(directory, "FileSystem"),
RunInMemory = storageType.Equals("esent", StringComparison.OrdinalIgnoreCase) == false && runInMemory,
#if DEBUG
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = runInMemory,
#endif
DefaultStorageTypeName = storageType,
AnonymousUserAccessMode = enableAuthentication ? AnonymousUserAccessMode.None : AnonymousUserAccessMode.Admin,
};
if (enableAuthentication)
{
Authentication.EnableOnce();
}
var ravenDbServer = new RavenDbServer(ravenConfiguration)
{
UseEmbeddedHttpServer = true
};
ravenDbServer.Initialize();
servers.Add(ravenDbServer);
if (enableAuthentication)
{
EnableAuthentication(ravenDbServer.SystemDatabase);
}
ConfigureServer(ravenDbServer, fileSystemName);
return ravenDbServer;
}
public RavenDbServer(RavenConfiguration settings)
{
database = new DocumentDatabase(settings);
try
{
database.SpinBackgroundWorkers();
server = new HttpServer(settings, database);
server.StartListening();
}
catch (Exception)
{
database.Dispose();
database = null;
throw;
}
}
public IndexStorage(IndexDefinitionStorage indexDefinitionStorage, RavenConfiguration configuration)
{
this.configuration = configuration;
path = Path.Combine(configuration.DataDirectory, "Indexes");
if (Directory.Exists(path) == false)
Directory.CreateDirectory(path);
foreach (var indexDirectory in indexDefinitionStorage.IndexNames)
{
log.DebugFormat("Loading saved index {0}", indexDirectory);
var indexDefinition = indexDefinitionStorage.GetIndexDefinition(indexDirectory);
if (indexDefinition == null)
continue;
indexes.TryAdd(indexDirectory,
CreateIndexImplementation(indexDirectory, indexDefinition,
OpenOrCreateLuceneDirectory(indexDirectory)));
}
}
public static void Init()
{
if (database != null)
return;
lock (locker)
{
if (database != null)
return;
log.Info("Setting up RavenDB Http Integration to the ASP.Net Pipeline");
try
{
var ravenConfiguration = new RavenConfiguration();
HttpEndpointRegistration.RegisterHttpEndpointTarget();
database = new DocumentDatabase(ravenConfiguration);
database.SpinBackgroundWorkers();
server = new HttpServer(ravenConfiguration, database);
server.Init();
}
catch
{
if (database != null)
{
database.Dispose();
database = null;
}
if (server != null)
{
server.Dispose();
server = null;
}
throw;
}
shutdownDetector = new ShutdownDetector(log);
shutdownDetector.Initialize();
shutdownDetector.Token.Register(OnShutdown);
}
}
public static void Init()
{
if (database != null)
return;
lock (locker)
{
if (database != null)
return;
log.Info("Setting up RavenDB Http Integration to the ASP.Net Pipeline");
try
{
var ravenConfiguration = new RavenConfiguration();
HttpEndpointRegistration.RegisterHttpEndpointTarget();
database = new DocumentDatabase(ravenConfiguration);
database.SpinBackgroundWorkers();
server = new HttpServer(ravenConfiguration, database);
server.Init();
}
catch
{
if (database != null)
{
database.Dispose();
database = null;
}
if (server != null)
{
server.Dispose();
server = null;
}
throw;
}
HostingEnvironment.RegisterObject(new ReleaseRavenDBWhenAppDomainIsTornDown());
}
}
public static void Restore(RavenConfiguration configuration, string backupLocation, string databaseLocation, Action<string> output, bool defrag)
{
using (var transactionalStorage = configuration.CreateTransactionalStorage(() => { }))
{
if (!string.IsNullOrWhiteSpace(databaseLocation))
{
configuration.DataDirectory = databaseLocation;
}
transactionalStorage.Restore(backupLocation, databaseLocation, output, defrag);
}
}
public ITransactionalStorage NewTransactionalStorage(string requestedStorage = null, string dataDir = null, string tempDir = null, bool? runInMemory = null, OrderedPartCollection<AbstractDocumentCodec> documentCodecs = null, Action onCommit = null)
{
ITransactionalStorage newTransactionalStorage;
string storageType = GetDefaultStorageType(requestedStorage);
var dataDirectory = dataDir ?? NewDataPath();
var ravenConfiguration = new RavenConfiguration
{
DataDirectory = dataDirectory,
RunInMemory = storageType.Equals("esent", StringComparison.OrdinalIgnoreCase) == false && (runInMemory ?? true),
};
ravenConfiguration.FileSystem.DataDirectory = Path.Combine(dataDirectory, "FileSystem");
ravenConfiguration.Storage.Voron.TempPath = tempDir;
Action onCommitNotification = () =>
{
if (onCommit != null)
onCommit();
};
if (storageType == "voron")
newTransactionalStorage = new Raven.Storage.Voron.TransactionalStorage(ravenConfiguration, onCommitNotification, () => { }, () => { }, () => { });
else
newTransactionalStorage = new Raven.Storage.Esent.TransactionalStorage(ravenConfiguration, onCommitNotification, () => { }, () => { }, () => { });
newTransactionalStorage.Initialize(new SequentialUuidGenerator { EtagBase = 0 }, documentCodecs ?? new OrderedPartCollection<AbstractDocumentCodec>());
return newTransactionalStorage;
}
public async Task OnDirectoryInitializeInMemoryTest()
{
string script;
IDictionary <string, string> customSettings = new ConcurrentDictionary <string, string>();
var scriptFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var outputFile = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".txt"));
if (PlatformDetails.RunningOnPosix)
{
customSettings[RavenConfiguration.GetKey(x => x.Storage.OnDirectoryInitializeExec)] = "bash";
customSettings[RavenConfiguration.GetKey(x => x.Storage.OnDirectoryInitializeExecArguments)] = $"{scriptFile} {outputFile}";
script = "#!/bin/bash\r\necho \"$2 $3 $4 $5 $6\" >> $1";
File.WriteAllText(scriptFile, script);
Process.Start("chmod", $"700 {scriptFile}");
}
else
{
customSettings[RavenConfiguration.GetKey(x => x.Storage.OnDirectoryInitializeExec)] = "powershell";
customSettings[RavenConfiguration.GetKey(x => x.Storage.OnDirectoryInitializeExecArguments)] = $"{scriptFile} {outputFile}";
script = @"
param([string]$userArg ,[string]$type, [string]$name, [string]$dataPath, [string]$tempPath, [string]$journalPath)
Add-Content $userArg ""$type $name $dataPath $tempPath $journalPath\r\n""
exit 0";
File.WriteAllText(scriptFile, script);
}
UseNewLocalServer(customSettings: customSettings);
// Creating dummy storage env options, so we can tell all the different paths
using (var options = StorageEnvironmentOptions.CreateMemoryOnly())
{
using (var store = GetDocumentStore())
{
store.Maintenance.Send(new CreateSampleDataOperation(DatabaseItemType.Indexes));
// the database loads after all indexes are loaded
var documentDatabase = await Server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(store.Database);
var lines = File.ReadAllLines(outputFile);
Assert.Equal(10, lines.Length);
Assert.True(lines[0].Contains($"{DirectoryExecUtils.EnvironmentType.System} {SystemDbName} {options.BasePath} {options.TempPath} {options.JournalPath}"));
Assert.True(lines[1].Contains($"{DirectoryExecUtils.EnvironmentType.Configuration} {store.Database} {options.BasePath} {options.TempPath} {options.JournalPath}"));
Assert.True(lines[2].Contains($"{DirectoryExecUtils.EnvironmentType.Database} {store.Database} {options.BasePath} {options.TempPath} {options.JournalPath}"));
var indexes = documentDatabase.IndexStore.GetIndexes().ToArray();
Assert.Equal(7, indexes.Length);
// The indexes order in the IndexStore don't match the order of storage env creation and we need a one-to-one match.
var matches = lines.ToList().GetRange(3, 7);
foreach (var index in indexes)
{
var expected = $"{DirectoryExecUtils.EnvironmentType.Index} {store.Database} {index._environment.Options.BasePath} {index._environment.Options.TempPath} {index._environment.Options.JournalPath}";
var indexToRemove = matches.FindIndex(str => str.Contains(expected));
if (indexToRemove != -1)
{
matches.RemoveAt(indexToRemove);
}
}
Assert.Equal(0, matches.Count);
}
}
}
private static (HashSet <SnmpVersion> Versions, string HandlerVersion) GetVersions(RavenServer server)
{
var length = server.Configuration.Monitoring.Snmp.SupportedVersions.Length;
if (length <= 0)
{
throw new InvalidOperationException($"There are no SNMP versions configured. Please set at least one in via '{RavenConfiguration.GetKey(x => x.Monitoring.Snmp.SupportedVersions)}' configuration option.");
}
var protocols = new HashSet <string>();
var versions = new HashSet <SnmpVersion>();
foreach (string version in server.Configuration.Monitoring.Snmp.SupportedVersions)
{
if (Enum.TryParse(version, ignoreCase: true, out SnmpVersion v) == false)
{
throw new InvalidOperationException($"Could not recognize '{version}' as a valid SNMP version.");
}
versions.Add(v);
switch (v)
{
case SnmpVersion.V2C:
protocols.Add("V2");
break;
case SnmpVersion.V3:
protocols.Add("V3");
break;
default:
throw new ArgumentOutOfRangeException();
}
}
return(versions, string.Join(",", protocols));
}
public void Compact(RavenConfiguration ravenConfiguration, Action <string> output)
{
if (configuration.Core.RunInMemory)
{
throw new InvalidOperationException("Cannot compact in-memory running Voron storage");
}
tableStorage.Dispose();
var sourcePath = path;
var compactPath = Path.Combine(path, "Voron.Compaction");
if (Directory.Exists(compactPath))
{
Directory.Delete(compactPath, true);
}
RecoverFromFailedCompact(sourcePath);
var sourceOptions = CreateStorageOptionsFromConfiguration(path, configuration);
var compactOptions = (StorageEnvironmentOptions.DirectoryStorageEnvironmentOptions)StorageEnvironmentOptions.ForPath(compactPath);
output("Executing storage compaction");
StorageCompaction.Execute(sourceOptions, compactOptions,
x => output(string.Format("Copied {0} of {1} records in '{2}' tree. Copied {3} of {4} trees.", x.CopiedTreeRecords, x.TotalTreeRecordsCount, x.TreeName, x.CopiedTrees, x.TotalTreeCount)));
var sourceDir = new DirectoryInfo(sourcePath);
var sourceFiles = new List <FileInfo>();
foreach (var pattern in new[] { "*.journal", "headers.one", "headers.two", VoronConstants.DatabaseFilename })
{
sourceFiles.AddRange(sourceDir.GetFiles(pattern));
}
var compactionBackup = Path.Combine(sourcePath, "Voron.Compaction.Backup");
if (Directory.Exists(compactionBackup))
{
Directory.Delete(compactionBackup, true);
output("Removing existing compaction backup directory");
}
Directory.CreateDirectory(compactionBackup);
output("Backing up original data files");
foreach (var file in sourceFiles)
{
File.Move(file.FullName, Path.Combine(compactionBackup, file.Name));
}
var compactedFiles = new DirectoryInfo(compactPath).GetFiles();
output("Moving compacted files into target location");
foreach (var file in compactedFiles)
{
File.Move(file.FullName, Path.Combine(sourcePath, file.Name));
}
output("Deleting original data backup");
Directory.Delete(compactionBackup, true);
Directory.Delete(compactPath, true);
}
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
{
try
{
if (onProgress == null)
{
onProgress = _ => { }
}
;
var result = new RestoreResult
{
DataDirectory = _restoreConfiguration.DataDirectory
};
Stopwatch sw = null;
RestoreSettings restoreSettings = null;
var firstFile = _filesToRestore[0];
var extension = Path.GetExtension(firstFile);
var snapshotRestore = false;
if (extension == Constants.Documents.PeriodicBackup.SnapshotExtension)
{
onProgress.Invoke(result.Progress);
snapshotRestore = true;
sw = Stopwatch.StartNew();
// restore the snapshot
restoreSettings = SnapshotRestore(firstFile,
_restoreConfiguration.DataDirectory,
onProgress,
result);
// removing the snapshot from the list of files
_filesToRestore.RemoveAt(0);
}
else
{
result.SnapshotRestore.Skipped = true;
result.SnapshotRestore.Processed = true;
onProgress.Invoke(result.Progress);
}
var databaseName = _restoreConfiguration.DatabaseName;
if (restoreSettings == null)
{
restoreSettings = new RestoreSettings
{
DatabaseRecord = new DatabaseRecord(databaseName)
{
// we only have a smuggler restore
// use the encryption key to encrypt the database
Encrypted = _hasEncryptionKey
}
};
DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord);
}
var databaseRecord = restoreSettings.DatabaseRecord;
if (databaseRecord.Settings == null)
{
databaseRecord.Settings = new Dictionary <string, string>();
}
databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false";
databaseRecord.Settings[RavenConfiguration.GetKey(x => x.Core.DataDirectory)] = _restoreConfiguration.DataDirectory;
if (_hasEncryptionKey)
{
// save the encryption key so we'll be able to access the database
_serverStore.PutSecretKey(_restoreConfiguration.EncryptionKey,
databaseName, overwrite: false);
}
using (var database = new DocumentDatabase(databaseName,
new RavenConfiguration(databaseName, ResourceType.Database)
{
Core =
{
DataDirectory = new PathSetting(_restoreConfiguration.DataDirectory),
RunInMemory = false
}
}, _serverStore))
{
// smuggler needs an existing document database to operate
var options = InitializeOptions.SkipLoadingDatabaseRecord;
if (snapshotRestore)
{
options |= InitializeOptions.GenerateNewDatabaseId;
}
database.Initialize(options);
if (snapshotRestore)
{
result.SnapshotRestore.Processed = true;
var summary = database.GetDatabaseSummary();
result.Documents.ReadCount += summary.DocumentsCount;
result.Documents.Attachments.ReadCount += summary.AttachmentsCount;
result.RevisionDocuments.ReadCount += summary.RevisionsCount;
result.Indexes.ReadCount += databaseRecord.GetIndexesCount();
result.Identities.ReadCount += restoreSettings.Identities.Count;
result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} " +
$"files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms");
onProgress.Invoke(result.Progress);
}
SmugglerRestore(_restoreConfiguration.BackupLocation, database, databaseRecord, restoreSettings, onProgress, result);
}
result.Documents.Processed = true;
result.RevisionDocuments.Processed = true;
result.Indexes.Processed = true;
result.Identities.Processed = true;
onProgress.Invoke(result.Progress);
databaseRecord.Topology = new DatabaseTopology();
// restoring to the current node only
databaseRecord.Topology.Members.Add(_nodeTag);
await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, restoreSettings.DatabaseValues, isRestore : true);
var index = await WriteIdentitiesAsync(databaseName, restoreSettings.Identities);
await _serverStore.Cluster.WaitForIndexNotification(index);
return(result);
}
catch (OperationCanceledException)
{
// database shutdown
throw;
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations("Failed to restore database", e);
}
var alert = AlertRaised.Create(
"Failed to restore database",
$"Could not restore database named {_restoreConfiguration.DatabaseName}",
AlertType.RestoreError,
NotificationSeverity.Error,
details: new ExceptionDetails(e));
_serverStore.NotificationCenter.Add(alert);
// delete any files that we already created during the restore
IOExtensions.DeleteDirectory(_restoreConfiguration.DataDirectory);
throw;
}
finally
{
_operationCancelToken.Dispose();
}
}
protected virtual void ModifyConfiguration(RavenConfiguration configuration)
{
}
private void ProcessMaps(RavenConfiguration configuration, ObjectInstance definitions, JintPreventResolvingTasksReferenceResolver resolver, List <string> mapList,
List <HashSet <CollectionName> > mapReferencedCollections, out Dictionary <string, List <JavaScriptMapOperation> > collectionFunctions)
{
var mapsArray = definitions.GetProperty(MapsProperty).Value;
if (mapsArray.IsNull() || mapsArray.IsUndefined() || mapsArray.IsArray() == false)
{
ThrowIndexCreationException($"doesn't contain any map function or '{GlobalDefinitions}.{Maps}' was modified in the script");
}
var maps = mapsArray.AsArray();
collectionFunctions = new Dictionary <string, List <JavaScriptMapOperation> >();
for (int i = 0; i < maps.GetLength(); i++)
{
var mapObj = maps.Get(i.ToString());
if (mapObj.IsNull() || mapObj.IsUndefined() || mapObj.IsObject() == false)
{
ThrowIndexCreationException($"map function #{i} is not a valid object");
}
var map = mapObj.AsObject();
if (map.HasProperty(CollectionProperty) == false)
{
ThrowIndexCreationException($"map function #{i} is missing a collection name");
}
var mapCollectionStr = map.Get(CollectionProperty);
if (mapCollectionStr.IsString() == false)
{
ThrowIndexCreationException($"map function #{i} collection name isn't a string");
}
var mapCollection = mapCollectionStr.AsString();
if (collectionFunctions.TryGetValue(mapCollection, out var list) == false)
{
list = new List <JavaScriptMapOperation>();
collectionFunctions.Add(mapCollection, list);
}
if (map.HasProperty(MethodProperty) == false)
{
ThrowIndexCreationException($"map function #{i} is missing its {MethodProperty} property");
}
var funcInstance = map.Get(MethodProperty).As <FunctionInstance>();
if (funcInstance == null)
{
ThrowIndexCreationException($"map function #{i} {MethodProperty} property isn't a 'FunctionInstance'");
}
var operation = new JavaScriptMapOperation(_engine, resolver)
{
MapFunc = funcInstance,
IndexName = _definitions.Name,
Configuration = configuration,
MapString = mapList[i]
};
if (map.HasOwnProperty(MoreArgsProperty))
{
var moreArgsObj = map.Get(MoreArgsProperty);
if (moreArgsObj.IsArray())
{
var array = moreArgsObj.AsArray();
if (array.GetLength() > 0)
{
operation.MoreArguments = array;
}
}
}
operation.Analyze(_engine);
if (ReferencedCollections.TryGetValue(mapCollection, out var collectionNames) == false)
{
collectionNames = new HashSet <CollectionName>();
ReferencedCollections.Add(mapCollection, collectionNames);
}
collectionNames.UnionWith(mapReferencedCollections[i]);
list.Add(operation);
}
}
public async Task ShouldKeepPullingDocsAfterServerRestart()
{
var dataPath = NewDataPath();
IDocumentStore store = null;
RavenServer server = null;
SubscriptionWorker <dynamic> subscriptionWorker = null;
try
{
server = GetNewServer(runInMemory: false, customSettings: new Dictionary <string, string>()
{
[RavenConfiguration.GetKey(x => x.Core.DataDirectory)] = dataPath
});
store = new DocumentStore()
{
Urls = new[] { server.ServerStore.GetNodeHttpServerUrl() },
Database = "RavenDB_2627",
}.Initialize();
var doc = new DatabaseRecord(store.Database);
var result = store.Maintenance.Server.Send(new CreateDatabaseOperationWithoutNameValidation(doc));
await WaitForRaftIndexToBeAppliedInCluster(result.RaftCommandIndex, _reasonableWaitTime);
using (var session = store.OpenSession())
{
session.Store(new User());
session.Store(new User());
session.Store(new User());
session.Store(new User());
session.SaveChanges();
}
var id = store.Subscriptions.Create(new SubscriptionCreationOptions <User>());
subscriptionWorker = store.Subscriptions.GetSubscriptionWorker(new SubscriptionWorkerOptions(id)
{
TimeToWaitBeforeConnectionRetry = TimeSpan.FromSeconds(1),
MaxDocsPerBatch = 1
});
var gotBatch = new ManualResetEventSlim();
var gotArek = new ManualResetEventSlim();
var t = subscriptionWorker.Run(x =>
{
gotBatch.Set();
foreach (var item in x.Items)
{
if (item.Id == "users/arek")
{
gotArek.Set();
}
}
});
Assert.True(gotBatch.Wait(_reasonableWaitTime));
Server.ServerStore.DatabasesLandlord.UnloadDirectly(store.Database);
for (int i = 0; i < 150; i++)
{
try
{
using (var session = store.OpenSession())
{
session.Store(new User(), "users/arek");
session.SaveChanges();
}
break;
}
catch
{
Thread.Sleep(25);
if (i > 100)
{
throw;
}
}
}
Assert.True(gotArek.Wait(_reasonableWaitTime));
}
finally
{
subscriptionWorker?.Dispose();
store?.Dispose();
server.Dispose();
}
}
protected override void ModifyConfiguration(RavenConfiguration ravenConfiguration)
{
ravenConfiguration.RunInMemory = false;
ravenConfiguration.DefaultStorageTypeName = "esent";
}
public async Task CanIdleDatabaseInCluster()
{
const int clusterSize = 3;
var databaseName = GetDatabaseName();
var cluster = await CreateRaftCluster(numberOfNodes : clusterSize, shouldRunInMemory : false, customSettings : new Dictionary <string, string>()
{
[RavenConfiguration.GetKey(x => x.Cluster.MoveToRehabGraceTime)] = "10",
[RavenConfiguration.GetKey(x => x.Cluster.AddReplicaTimeout)] = "1",
[RavenConfiguration.GetKey(x => x.Cluster.ElectionTimeout)] = "300",
[RavenConfiguration.GetKey(x => x.Cluster.StabilizationTime)] = "1",
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3"
});
_nodes = cluster.Nodes;
var wakeUpReasons = new Dictionary <string, List <string> >();
try
{
foreach (var server in _nodes)
{
wakeUpReasons.Add(server.ServerStore.NodeTag, new List <string>());
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = true;
server.ServerStore.DatabasesLandlord.ForTestingPurposesOnly().AfterDatabaseCreation = tuple =>
{
var list = wakeUpReasons[tuple.Database.ServerStore.NodeTag];
list.Add(tuple.caller);
wakeUpReasons[tuple.Database.ServerStore.NodeTag] = list;
};
}
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseName = s => databaseName,
ReplicationFactor = clusterSize,
Server = cluster.Leader,
RunInMemory = false
}))
{
var count = RavenDB_13987.WaitForCount(TimeSpan.FromSeconds(300), clusterSize, GetIdleCount);
Assert.True(clusterSize == count, string.Join(Environment.NewLine, wakeUpReasons.Select(x => string.Join(": ", x.Key, string.Join(", ", x.Value)))));
foreach (var server in _nodes)
{
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
Assert.True(server.ServerStore.IdleDatabases.TryGetValue(databaseName, out var dictionary));
// new incoming replications not saved in IdleDatabases
Assert.Equal(0, dictionary.Count);
}
var rnd = new Random();
var index = rnd.Next(0, clusterSize);
using (var store2 = new DocumentStore {
Urls = new[] { _nodes[index].WebUrl }, Conventions = { DisableTopologyUpdates = true }, Database = databaseName
}.Initialize())
{
await store2.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.True(2 == GetIdleCount(), string.Join(Environment.NewLine, wakeUpReasons.Select(x => string.Join(": ", x.Key, string.Join(", ", x.Value)))));
using (var s = store2.OpenAsyncSession())
{
await s.StoreAsync(new User()
{
Name = "Egor"
}, "foo/bar");
await s.SaveChangesAsync();
}
}
count = RavenDB_13987.WaitForCount(TimeSpan.FromSeconds(300), 0, GetIdleCount);
Assert.True(0 == count, string.Join(Environment.NewLine, wakeUpReasons.Select(x => string.Join(": ", x.Key, string.Join(", ", x.Value)))));
var timeout = 5000;
foreach (var server in _nodes)
{
using (var store2 = new DocumentStore {
Urls = new[] { server.WebUrl }, Conventions = { DisableTopologyUpdates = true }, Database = databaseName
}.Initialize())
{
Assert.True(WaitForDocument(store2, "foo/bar", timeout, databaseName), $"WaitForDocument for {server.ServerStore.NodeTag} returned false, after {timeout}, leader: {cluster.Leader}");
}
}
index = rnd.Next(0, clusterSize);
var nextNow = DateTime.Now + TimeSpan.FromSeconds(300);
var now = DateTime.Now;
using (var store2 = new DocumentStore {
Urls = new[] { _nodes[index].WebUrl }, Conventions = { DisableTopologyUpdates = true }, Database = databaseName
}.Initialize())
{
while (now < nextNow && GetIdleCount() < 2)
{
await Task.Delay(2000);
await store2.Maintenance.SendAsync(new GetStatisticsOperation());
now = DateTime.Now;
}
}
Assert.True(2 == GetIdleCount(), string.Join(Environment.NewLine, wakeUpReasons.Select(x => string.Join(": ", x.Key, string.Join(", ", x.Value)))));
}
}
finally
{
foreach (var server in _nodes)
{
server.ServerStore.DatabasesLandlord.SkipShouldContinueDisposeCheck = false;
server.ServerStore.DatabasesLandlord.ForTestingPurposes = null;
}
}
}
public async Task TwoWayExternalReplicationShouldNotLoadIdleDatabase()
{
using (var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
}))
using (var store1 = GetDocumentStore(new Options
{
Server = server,
RunInMemory = false
}))
using (var store2 = GetDocumentStore(new Options
{
Server = server,
RunInMemory = false
}))
{
var externalTask1 = new ExternalReplication(store2.Database, "MyConnectionString1")
{
Name = "MyExternalReplication1"
};
var externalTask2 = new ExternalReplication(store1.Database, "MyConnectionString2")
{
Name = "MyExternalReplication2"
};
await AddWatcherToReplicationTopology(store1, externalTask1);
await AddWatcherToReplicationTopology(store2, externalTask2);
Assert.True(server.ServerStore.DatabasesLandlord.LastRecentlyUsed.TryGetValue(store1.Database, out _));
Assert.True(server.ServerStore.DatabasesLandlord.LastRecentlyUsed.TryGetValue(store2.Database, out _));
var now = DateTime.Now;
var nextNow = now + TimeSpan.FromSeconds(60);
while (now < nextNow && server.ServerStore.IdleDatabases.Count < 2)
{
await Task.Delay(3000);
now = DateTime.Now;
}
Assert.Equal(2, server.ServerStore.IdleDatabases.Count);
await store1.Maintenance.SendAsync(new CreateSampleDataOperation());
WaitForIndexing(store1);
var count = 0;
var docs = store1.Maintenance.Send(new GetStatisticsOperation()).CountOfDocuments;
var replicatedDocs = store2.Maintenance.Send(new GetStatisticsOperation()).CountOfDocuments;
while (docs != replicatedDocs && count < 20)
{
await Task.Delay(3000);
replicatedDocs = store2.Maintenance.Send(new GetStatisticsOperation()).CountOfDocuments;
count++;
}
Assert.Equal(docs, replicatedDocs);
count = 0;
nextNow = DateTime.Now + TimeSpan.FromMinutes(5);
while (server.ServerStore.IdleDatabases.Count == 0 && now < nextNow)
{
await Task.Delay(500);
if (count % 10 == 0)
{
store1.Maintenance.Send(new GetStatisticsOperation());
}
now = DateTime.Now;
count++;
}
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
nextNow = DateTime.Now + TimeSpan.FromSeconds(15);
while (now < nextNow)
{
await Task.Delay(2000);
store1.Maintenance.Send(new GetStatisticsOperation());
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
now = DateTime.Now;
}
nextNow = DateTime.Now + TimeSpan.FromMinutes(10);
while (now < nextNow && server.ServerStore.IdleDatabases.Count < 2)
{
await Task.Delay(3000);
now = DateTime.Now;
}
Assert.Equal(2, server.ServerStore.IdleDatabases.Count);
using (var s = store2.OpenSession())
{
s.Advanced.RawQuery <dynamic>("from @all_docs")
.ToList();
}
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
var operation = await store2
.Operations
.ForDatabase(store2.Database)
.SendAsync(new PatchByQueryOperation("from Companies update { this.Name = this.Name + '_patched'; }"));
await operation.WaitForCompletionAsync();
nextNow = DateTime.Now + TimeSpan.FromMinutes(2);
while (now < nextNow && server.ServerStore.IdleDatabases.Count > 0)
{
await Task.Delay(5000);
now = DateTime.Now;
}
Assert.Equal(0, server.ServerStore.IdleDatabases.Count);
nextNow = DateTime.Now + TimeSpan.FromMinutes(10);
while (server.ServerStore.IdleDatabases.Count == 0 && now < nextNow)
{
await Task.Delay(500);
if (count % 10 == 0)
{
store2.Maintenance.Send(new GetStatisticsOperation());
}
now = DateTime.Now;
count++;
}
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
nextNow = DateTime.Now + TimeSpan.FromSeconds(15);
while (now < nextNow)
{
await Task.Delay(2000);
store2.Maintenance.Send(new GetStatisticsOperation());
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
now = DateTime.Now;
}
}
}
public async Task ExternalReplicationShouldNotLoadIdleDatabase()
{
using (var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Databases.MaxIdleTime)] = "10",
[RavenConfiguration.GetKey(x => x.Databases.FrequencyToCheckForIdle)] = "3",
[RavenConfiguration.GetKey(x => x.Replication.RetryMaxTimeout)] = "1",
[RavenConfiguration.GetKey(x => x.Core.RunInMemory)] = "false"
}
}))
using (var store1 = GetDocumentStore(new Options
{
Server = server,
RunInMemory = false
}))
using (var store2 = GetDocumentStore(new Options
{
Server = server,
RunInMemory = false
}))
{
// lowercase for test
var externalTask = new ExternalReplication(store2.Database.ToLowerInvariant(), "MyConnectionString")
{
Name = "MyExternalReplication"
};
await AddWatcherToReplicationTopology(store1, externalTask);
Assert.True(server.ServerStore.DatabasesLandlord.LastRecentlyUsed.TryGetValue(store1.Database, out _));
Assert.True(server.ServerStore.DatabasesLandlord.LastRecentlyUsed.TryGetValue(store2.Database, out _));
var now = DateTime.Now;
var nextNow = now + TimeSpan.FromSeconds(60);
while (now < nextNow && server.ServerStore.IdleDatabases.Count < 2)
{
await Task.Delay(3000);
now = DateTime.Now;
}
Assert.Equal(2, server.ServerStore.IdleDatabases.Count);
await store1.Maintenance.SendAsync(new CreateSampleDataOperation());
WaitForIndexing(store1);
var count = 0;
var docs = store1.Maintenance.Send(new GetStatisticsOperation()).CountOfDocuments;
var replicatedDocs = store2.Maintenance.Send(new GetStatisticsOperation()).CountOfDocuments;
while (docs != replicatedDocs && count < 20)
{
await Task.Delay(3000);
replicatedDocs = store2.Maintenance.Send(new GetStatisticsOperation()).CountOfDocuments;
count++;
}
Assert.Equal(docs, replicatedDocs);
count = 0;
nextNow = DateTime.Now + TimeSpan.FromMinutes(5);
while (server.ServerStore.IdleDatabases.Count == 0 && now < nextNow)
{
await Task.Delay(500);
if (count % 10 == 0)
{
store1.Maintenance.Send(new GetStatisticsOperation());
}
now = DateTime.Now;
count++;
}
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
nextNow = DateTime.Now + TimeSpan.FromSeconds(15);
while (now < nextNow)
{
await Task.Delay(2000);
store1.Maintenance.Send(new GetStatisticsOperation());
Assert.Equal(1, server.ServerStore.IdleDatabases.Count);
now = DateTime.Now;
}
}
}
private static void RunInDebugMode(AnonymousUserAccessMode? anonymousUserAccessMode, RavenConfiguration ravenConfiguration, bool lauchBrowser)
{
ConfigureDebugLogging();
NonAdminHttp.EnsureCanListenToWhenInNonAdminContext(ravenConfiguration.Port);
if (anonymousUserAccessMode.HasValue)
ravenConfiguration.AnonymousUserAccessMode = anonymousUserAccessMode.Value;
while (RunServerInDebugMode(ravenConfiguration, lauchBrowser))
{
lauchBrowser = false;
}
}
protected virtual void ConfigureServer(RavenConfiguration ravenConfiguration)
{
}
protected override void ModifyConfiguration(RavenConfiguration ravenConfiguration)
{
ravenConfiguration.AnonymousUserAccessMode = AnonymousUserAccessMode.None;
ravenConfiguration.Catalog.Catalogs.Add(new TypeCatalog(typeof(FakeAuthenticateClient)));
}
private void WriteDatabaseInfo(string databaseName, BlittableJsonReaderObject dbRecordBlittable,
TransactionOperationContext context, AbstractBlittableJsonTextWriter writer)
{
var nodesTopology = new NodesTopology();
try
{
var online = ServerStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out Task <DocumentDatabase> dbTask) &&
dbTask != null &&
dbTask.IsCompleted;
var dbRecord = JsonDeserializationCluster.DatabaseRecord(dbRecordBlittable);
var topology = dbRecord.Topology;
var statuses = ServerStore.GetNodesStatuses();
if (topology != null)
{
nodesTopology.PriorityOrder = topology.PriorityOrder;
var clusterTopology = ServerStore.GetClusterTopology(context);
clusterTopology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext);
foreach (var member in topology.Members)
{
if (dbRecord.DeletionInProgress != null && dbRecord.DeletionInProgress.ContainsKey(member))
{
continue;
}
var url = clusterTopology.GetUrlFromTag(member);
var node = new InternalReplication
{
Database = databaseName,
NodeTag = member,
Url = url
};
nodesTopology.Members.Add(GetNodeId(node));
SetNodeStatus(topology, member, nodesTopology, statuses);
}
foreach (var promotable in topology.Promotables)
{
if (dbRecord.DeletionInProgress != null && dbRecord.DeletionInProgress.ContainsKey(promotable))
{
continue;
}
topology.PredefinedMentors.TryGetValue(promotable, out var mentorCandidate);
var node = GetNode(databaseName, clusterTopology, promotable, mentorCandidate, out var promotableTask);
var mentor = topology.WhoseTaskIsIt(ServerStore.Engine.CurrentState, promotableTask, null);
nodesTopology.Promotables.Add(GetNodeId(node, mentor));
SetNodeStatus(topology, promotable, nodesTopology, statuses);
}
foreach (var rehab in topology.Rehabs)
{
if (dbRecord.DeletionInProgress != null && dbRecord.DeletionInProgress.ContainsKey(rehab))
{
continue;
}
var node = GetNode(databaseName, clusterTopology, rehab, null, out var promotableTask);
var mentor = topology.WhoseTaskIsIt(ServerStore.Engine.CurrentState, promotableTask, null);
nodesTopology.Rehabs.Add(GetNodeId(node, mentor));
SetNodeStatus(topology, rehab, nodesTopology, statuses);
}
}
// Check for exceptions
if (dbTask != null && dbTask.IsFaulted)
{
var exception = dbTask.Exception.ExtractSingleInnerException();
WriteFaultedDatabaseInfo(databaseName, nodesTopology, exception, context, writer);
return;
}
var db = online ? dbTask.Result : null;
var indexingStatus = db?.IndexStore?.Status;
if (indexingStatus == null)
{
// Looking for disabled indexing flag inside the database settings for offline database status
if (dbRecord.Settings.TryGetValue(RavenConfiguration.GetKey(x => x.Indexing.Disabled), out var val) &&
bool.TryParse(val, out var indexingDisabled) && indexingDisabled)
{
indexingStatus = IndexRunningStatus.Disabled;
}
}
var disabled = dbRecord.Disabled;
var lockMode = dbRecord.LockMode;
var studioEnvironment = StudioConfiguration.StudioEnvironment.None;
if (dbRecord.Studio != null && !dbRecord.Studio.Disabled)
{
studioEnvironment = dbRecord.Studio.Environment;
}
if (online == false)
{
// if state of database is found in the cache we can continue
if (ServerStore.DatabaseInfoCache.TryGet(databaseName, databaseInfoJson =>
{
databaseInfoJson.Modifications = new DynamicJsonValue(databaseInfoJson)
{
[nameof(DatabaseInfo.Disabled)] = disabled,
[nameof(DatabaseInfo.LockMode)] = lockMode,
[nameof(DatabaseInfo.IndexingStatus)] = indexingStatus,
[nameof(DatabaseInfo.NodesTopology)] = nodesTopology.ToJson(),
[nameof(DatabaseInfo.DeletionInProgress)] = DynamicJsonValue.Convert(dbRecord.DeletionInProgress),
[nameof(DatabaseInfo.Environment)] = studioEnvironment
};
context.Write(writer, databaseInfoJson);
}))
{
return;
}
// we won't find it if it is a new database or after a dirty shutdown,
// so just report empty values then
}
var size = db?.GetSizeOnDisk() ?? (new Size(0), new Size(0));
var databaseInfo = new DatabaseInfo
{
Name = databaseName,
Disabled = disabled,
LockMode = lockMode,
TotalSize = size.Data,
TempBuffersSize = size.TempBuffers,
IsAdmin = true,
IsEncrypted = dbRecord.Encrypted,
UpTime = online ? (TimeSpan?)GetUptime(db) : null,
BackupInfo = GetBackupInfo(db, context),
Alerts = db?.NotificationCenter.GetAlertCount() ?? 0,
PerformanceHints = db?.NotificationCenter.GetPerformanceHintCount() ?? 0,
RejectClients = false,
LoadError = null,
IndexingErrors = db?.IndexStore?.GetIndexes()?.Sum(index => index.GetErrorCount()) ?? 0,
DocumentsCount = db?.DocumentsStorage.GetNumberOfDocuments() ?? 0,
HasRevisionsConfiguration = db?.DocumentsStorage.RevisionsStorage.Configuration != null,
HasExpirationConfiguration = (db?.ExpiredDocumentsCleaner?.ExpirationConfiguration?.Disabled ?? true) == false,
HasRefreshConfiguration = (db?.ExpiredDocumentsCleaner?.RefreshConfiguration?.Disabled ?? true) == false,
IndexesCount = db?.IndexStore?.GetIndexes()?.Count() ?? 0,
IndexingStatus = indexingStatus ?? IndexRunningStatus.Running,
Environment = studioEnvironment,
NodesTopology = nodesTopology,
ReplicationFactor = topology?.ReplicationFactor ?? -1,
DynamicNodesDistribution = topology?.DynamicNodesDistribution ?? false,
DeletionInProgress = dbRecord.DeletionInProgress
};
var doc = databaseInfo.ToJson();
context.Write(writer, doc);
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to get database info for: {databaseName}", e);
}
WriteFaultedDatabaseInfo(databaseName, nodesTopology, e, context, writer);
}
}
private static StorageEnvironmentOptions CreateStorageOptionsFromConfiguration(string path, RavenConfiguration configuration)
{
if (configuration.Core.RunInMemory)
{
return(StorageEnvironmentOptions.CreateMemoryOnly(configuration.Storage.TempPath));
}
var directoryPath = path ?? AppDomain.CurrentDomain.BaseDirectory;
var filePathFolder = new DirectoryInfo(directoryPath);
if (filePathFolder.Exists == false)
{
filePathFolder.Create();
}
var options = StorageEnvironmentOptions.ForPath(directoryPath, configuration.Storage.TempPath, configuration.Storage.JournalsStoragePath);
options.IncrementalBackupEnabled = configuration.Storage.AllowIncrementalBackups;
return(options);
}
private void ValidateLocalRootPath()
{
if (LocalRootPath == null)
{
return;
}
var directoryInfo = new DirectoryInfo(LocalRootPath.FullPath);
if (directoryInfo.Exists == false)
{
throw new ArgumentException($"The backup path '{LocalRootPath.FullPath}' defined in the configuration under '{RavenConfiguration.GetKey(x => x.Backup.LocalRootPath)}' doesn't exist.");
}
}
public async Task RestoreAndReplicateCounters()
{
var backupPath = NewDataPath(suffix: "BackupFolder");
using (var server = GetNewServer(
new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string>
{
[RavenConfiguration.GetKey(x => x.Replication.MaxItemsCount)] = 1.ToString()
},
RegisterForDisposal = false
}))
{
using (var store1 = GetDocumentStore(new Options
{
Server = server
}))
using (var store2 = GetDocumentStore(new Options
{
Server = server,
CreateDatabase = false
}))
using (var store3 = GetDocumentStore(new Options
{
Server = server
}))
{
using (var session = store1.OpenAsyncSession())
{
await session.StoreAsync(new User { Name = "Name1" }, "users/1");
await session.StoreAsync(new User { Name = "Name2" }, "users/2");
session.CountersFor("users/1").Increment("likes", 100);
session.CountersFor("users/2").Increment("downloads", 500);
await session.SaveChangesAsync();
}
using (var session = store1.OpenAsyncSession())
{
// need to be in a different transaction in order to split the replication into batches
session.CountersFor("users/1").Increment("dislikes", 200);
await session.SaveChangesAsync();
}
var config = Backup.CreateBackupConfiguration(backupPath);
await Backup.UpdateConfigAndRunBackupAsync(server, config, store1);
await Restore(backupPath, store2);
var stats = await store2.Maintenance.SendAsync(new GetStatisticsOperation());
Assert.Equal(2, stats.CountOfDocuments);
Assert.Equal(2, stats.CountOfCounterEntries);
using (var session = store2.OpenAsyncSession())
{
await AssertCounters(session);
}
await SetupReplicationAsync(store2, store3);
using (var session = store2.OpenAsyncSession())
{
await session.StoreAsync(new User(), "marker");
await session.SaveChangesAsync();
}
Assert.NotNull(WaitForDocumentToReplicate <User>(store3, "marker", 10_000));
using (var session = store3.OpenAsyncSession())
{
await AssertCounters(session);
}
}
}
}
internal static AbstractStaticIndexBase GenerateIndex(IndexDefinition definition, RavenConfiguration configuration, IndexType type, long indexVersion)
{
switch (type)
{
case IndexType.None:
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
case IndexType.Map:
case IndexType.MapReduce:
case IndexType.Faulty:
return(IndexCompiler.Compile(definition));
case IndexType.JavaScriptMap:
case IndexType.JavaScriptMapReduce:
return(AbstractJavaScriptIndex.Create(definition, configuration, indexVersion));
default:
throw new ArgumentOutOfRangeException($"Can't generate index of unknown type {definition.DetectStaticIndexType()}");
}
}
public async Task CertificateAndMasterKeyExecTest()
{
string script;
IDictionary <string, string> customSettings = new ConcurrentDictionary <string, string>();
var keyPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
var buffer = new byte[256 / 8];
using (var cryptoRandom = RandomNumberGenerator.Create())
{
cryptoRandom.GetBytes(buffer);
}
File.WriteAllBytes(keyPath, buffer);
var certificates = Certificates.GenerateAndSaveSelfSignedCertificate();
if (PlatformDetails.RunningOnPosix)
{
var scriptPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".sh"));
var keyArgs = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
scriptPath, keyPath
});
var certArgs = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
scriptPath, certificates.ServerCertificatePath
});
customSettings[RavenConfiguration.GetKey(x => x.Security.MasterKeyExec)] = "bash";
customSettings[RavenConfiguration.GetKey(x => x.Security.MasterKeyExecArguments)] = $"{keyArgs}";
customSettings[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "bash";
customSettings[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{certArgs}";
customSettings[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0";
script = "#!/bin/bash\ncat \"$1\"";
File.WriteAllText(scriptPath, script);
Process.Start("chmod", $"700 {scriptPath}");
}
else
{
var scriptPath = Path.Combine(Path.GetTempPath(), Path.ChangeExtension(Guid.NewGuid().ToString(), ".ps1"));
var keyArgs = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", scriptPath, keyPath
});
var certArgs = CommandLineArgumentEscaper.EscapeAndConcatenate(new List <string> {
"-NoProfile", scriptPath, certificates.ServerCertificatePath
});
customSettings[RavenConfiguration.GetKey(x => x.Security.MasterKeyExec)] = "powershell";
customSettings[RavenConfiguration.GetKey(x => x.Security.MasterKeyExecArguments)] = $"{keyArgs}";
customSettings[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExec)] = "powershell";
customSettings[RavenConfiguration.GetKey(x => x.Security.CertificateRenewExec)] = "powershell";
customSettings[RavenConfiguration.GetKey(x => x.Security.CertificateChangeExec)] = "powershell";
customSettings[RavenConfiguration.GetKey(x => x.Security.CertificateLoadExecArguments)] = $"{certArgs}";
customSettings[RavenConfiguration.GetKey(x => x.Core.ServerUrls)] = "https://" + Environment.MachineName + ":0";
script = @"param([string]$userArg)
try {
$bytes = Get-Content -path $userArg -encoding Byte
$stdout = [System.Console]::OpenStandardOutput()
$stdout.Write($bytes, 0, $bytes.Length)
}
catch {
Write-Error $_.Exception
exit 1
}
exit 0";
File.WriteAllText(scriptPath, script);
}
UseNewLocalServer(customSettings: customSettings, runInMemory: false);
// The master key loading is lazy, let's put a database secret key to invoke it.
var dbName = GetDatabaseName();
var databaseKey = new byte[32];
using (var rand = RandomNumberGenerator.Create())
{
rand.GetBytes(databaseKey);
}
var base64Key = Convert.ToBase64String(databaseKey);
// sometimes when using `dotnet xunit` we get platform not supported from ProtectedData
try
{
#pragma warning disable CA1416 // Validate platform compatibility
ProtectedData.Protect(Encoding.UTF8.GetBytes("Is supported?"), null, DataProtectionScope.CurrentUser);
#pragma warning restore CA1416 // Validate platform compatibility
}
catch (PlatformNotSupportedException)
{
return;
}
await Server.ServerStore.EnsureNotPassiveAsync();
Server.ServerStore.PutSecretKey(base64Key, dbName, true);
X509Certificate2 serverCertificate;
try
{
serverCertificate = new X509Certificate2(certificates.ServerCertificatePath, (string)null, X509KeyStorageFlags.Exportable | X509KeyStorageFlags.MachineKeySet);
}
catch (CryptographicException e)
{
throw new CryptographicException($"Failed to load the test certificate from {certificates}.", e);
}
using (var store = GetDocumentStore(new Options
{
AdminCertificate = serverCertificate,
ClientCertificate = serverCertificate,
ModifyDatabaseName = s => dbName,
ModifyDatabaseRecord = record => record.Encrypted = true,
Path = NewDataPath()
}))
{
}
var secrets = Server.ServerStore.Secrets;
var serverMasterKey = (Lazy <byte[]>) typeof(SecretProtection).GetField("_serverMasterKey", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(secrets);
Assert.True(serverMasterKey.Value.SequenceEqual(buffer));
Assert.True(Server.Certificate.Certificate.Equals(serverCertificate));
}
private static void InteractiveRun(string[] args)
{
switch (GetArgument(args))
{
case "install":
AdminRequired(InstallAndStart, "/install");
break;
case "uninstall":
AdminRequired(EnsureStoppedAndUninstall, "/uninstall");
break;
case "start":
AdminRequired(StartService, "/start");
break;
case "restart":
AdminRequired(RestartService, "/restart");
break;
case "stop":
AdminRequired(StopService, "/stop");
break;
case "restore":
if (args.Length != 3)
{
PrintUsage();
break;
}
RunRestoreOperation(args[0], args[1]);
break;
case "debug":
RunInDebugMode(anonymousUserAccessMode: null, ravenConfiguration: new RavenConfiguration());
break;
case "ram":
RunInDebugMode(anonymousUserAccessMode: AnonymousUserAccessMode.All, ravenConfiguration: new RavenConfiguration
{
RunInMemory = true,
StorageTypeName = typeof(Storage.Managed.TransactionalStorage).AssemblyQualifiedName
});
break;
#if DEBUG
case "test":
var dataDirectory = new RavenConfiguration().DataDirectory;
IOExtensions.DeleteDirectory(dataDirectory);
RunInDebugMode(anonymousUserAccessMode: AnonymousUserAccessMode.All, ravenConfiguration: new RavenConfiguration());
break;
#endif
default:
PrintUsage();
break;
}
}
public static AbstractStaticIndexBase GetIndexInstance(IndexDefinition definition, RavenConfiguration configuration, long indexVersion)
{
var type = definition.DetectStaticIndexType();
if (type.IsJavaScript())
{
return(GenerateIndex(definition, configuration, type, indexVersion));
}
switch (definition.SourceType)
{
case IndexSourceType.Documents:
return(GetDocumentsIndexInstance(definition, configuration, type, indexVersion));
case IndexSourceType.TimeSeries:
return(GetIndexInstance <StaticTimeSeriesIndexBase>(definition, configuration, type, indexVersion));
case IndexSourceType.Counters:
return(GetIndexInstance <StaticCountersIndexBase>(definition, configuration, type, indexVersion));
default:
throw new NotSupportedException($"Not supported source type '{definition.SourceType}'.");
}
}
public void GivenNoCertificateSpecifiedAndServerBoundOutsideOfUnsecureAccessAllowedShouldError()
{
try
{
GetConfiguration(
unsecuredAccessAddressRange: UnsecuredAccessAddressRange.Local,
serverUrl: "http://192.168.1.24");
}
catch (InvalidOperationException exception)
{
Assert.Equal($"Configured { RavenConfiguration.GetKey(x => x.Core.ServerUrls) } \"http://192.168.1.24\" is not within unsecured access address range. Use a server url within unsecure access address range ({ RavenConfiguration.GetKey(x => x.Security.UnsecuredAccessAllowed) }) or fill in certificate information.", exception.Message);
}
}
private static bool RunServer(RavenConfiguration ravenConfiguration)
{
using (new RavenDbServer(ravenConfiguration))
{
var path = Path.Combine(Environment.CurrentDirectory, "default.raven");
if (File.Exists(path))
{
Console.WriteLine("Loading data from: {0}", path);
Smuggler.Smuggler.ImportData(ravenConfiguration.ServerUrl, path);
}
Console.WriteLine("Raven is ready to process requests.");
Console.WriteLine("Data directory: {0}, HostName: {1} Port: {2}", ravenConfiguration.DataDirectory, ravenConfiguration.HostName ?? "<any>", ravenConfiguration.Port);
Console.WriteLine("Press the enter key to stop the server or enter 'cls' and then enter to clear the log");
while (true)
{
var readLine = Console.ReadLine() ?? "";
switch (readLine.ToLowerInvariant())
{
case "cls":
Console.Clear();
break;
case "reset":
Console.Clear();
return true;
default:
return false;
}
}
}
}
protected RavenDbServer GetNewServer(int port = 8079,
string dataDirectory = null,
bool runInMemory = true,
string requestedStorage = null,
bool enableAuthentication = false,
string activeBundles = null,
Action <RavenDBOptions> configureServer = null,
Action <InMemoryRavenConfiguration> configureConfig = null,
[CallerMemberName] string databaseName = null)
{
databaseName = NormalizeDatabaseName(databaseName != Constants.SystemDatabase ? databaseName : null);
checkPorts = true;
if (dataDirectory != null)
{
pathsToDelete.Add(dataDirectory);
}
var storageType = GetDefaultStorageType(requestedStorage);
var directory = dataDirectory ?? NewDataPath(databaseName == Constants.SystemDatabase ? null : databaseName);
var ravenConfiguration = new RavenConfiguration
{
Port = port,
DataDirectory = Path.Combine(directory, "System"),
RunInMemory = runInMemory,
#if DEBUG
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = runInMemory,
#endif
DefaultStorageTypeName = storageType,
AnonymousUserAccessMode = enableAuthentication ? AnonymousUserAccessMode.None : AnonymousUserAccessMode.Admin,
};
ravenConfiguration.FileSystem.DataDirectory = Path.Combine(directory, "FileSystem");
ravenConfiguration.Encryption.UseFips = SettingsHelper.UseFipsEncryptionAlgorithms;
ravenConfiguration.Settings["Raven/StorageTypeName"] = ravenConfiguration.DefaultStorageTypeName;
if (activeBundles != null)
{
ravenConfiguration.Settings["Raven/ActiveBundles"] = activeBundles;
}
if (configureConfig != null)
{
configureConfig(ravenConfiguration);
}
ModifyConfiguration(ravenConfiguration);
ravenConfiguration.PostInit();
NonAdminHttp.EnsureCanListenToWhenInNonAdminContext(ravenConfiguration.Port);
var ravenDbServer = new RavenDbServer(ravenConfiguration)
{
UseEmbeddedHttpServer = true,
};
ravenDbServer.Initialize(configureServer);
servers.Add(ravenDbServer);
try
{
using (var documentStore = new DocumentStore
{
Url = "http://localhost:" + port,
Conventions =
{
FailoverBehavior = FailoverBehavior.FailImmediately
},
DefaultDatabase = databaseName
}.Initialize())
{
CreateDefaultIndexes(documentStore);
}
}
catch
{
ravenDbServer.Dispose();
throw;
}
if (enableAuthentication)
{
EnableAuthentication(ravenDbServer.SystemDatabase);
ModifyConfiguration(ravenConfiguration);
ravenConfiguration.PostInit();
}
return(ravenDbServer);
}
public static void Restore(RavenConfiguration configuration, string backupLocation, string databaseLocation)
{
using (var transactionalStorage = configuration.CreateTransactionalStorage(() => { }))
{
transactionalStorage.Restore(backupLocation, databaseLocation);
}
}
private static void RunInDebugMode(AnonymousUserAccessMode?anonymousUserAccessMode, RavenConfiguration ravenConfiguration)
{
ConfigureDebugLogging();
NonAdminHttp.EnsureCanListenToWhenInNonAdminContext(ravenConfiguration.Port);
if (anonymousUserAccessMode.HasValue)
{
ravenConfiguration.AnonymousUserAccessMode = anonymousUserAccessMode.Value;
}
while (RunServer(ravenConfiguration))
{
}
}
protected RavenDbServer GetNewServer(int port = 8079,
string dataDirectory = null,
bool runInMemory = true,
string requestedStorage = null,
bool enableAuthentication = false,
string activeBundles = null,
Action<RavenDBOptions> configureServer = null,
Action<InMemoryRavenConfiguration> configureConfig = null,
[CallerMemberName] string databaseName = null)
{
databaseName = NormalizeDatabaseName(databaseName != Constants.SystemDatabase ? databaseName : null);
checkPorts = true;
if (dataDirectory != null)
pathsToDelete.Add(dataDirectory);
var storageType = GetDefaultStorageType(requestedStorage);
var directory = dataDirectory ?? NewDataPath(databaseName == Constants.SystemDatabase ? null : databaseName);
var ravenConfiguration = new RavenConfiguration();
ConfigurationHelper.ApplySettingsToConfiguration(ravenConfiguration);
ravenConfiguration.Port = port;
ravenConfiguration.DataDirectory = Path.Combine(directory, "System");
ravenConfiguration.RunInMemory = runInMemory;
#if DEBUG
ravenConfiguration.RunInUnreliableYetFastModeThatIsNotSuitableForProduction = runInMemory;
#endif
ravenConfiguration.DefaultStorageTypeName = storageType;
ravenConfiguration.AnonymousUserAccessMode = enableAuthentication ? AnonymousUserAccessMode.None : AnonymousUserAccessMode.Admin;
ravenConfiguration.FileSystem.DataDirectory = Path.Combine(directory, "FileSystem");
ravenConfiguration.Encryption.UseFips = ConfigurationHelper.UseFipsEncryptionAlgorithms;
ravenConfiguration.Settings["Raven/StorageTypeName"] = ravenConfiguration.DefaultStorageTypeName;
if (activeBundles != null)
{
ravenConfiguration.Settings["Raven/ActiveBundles"] = activeBundles;
}
if (configureConfig != null)
configureConfig(ravenConfiguration);
ModifyConfiguration(ravenConfiguration);
ravenConfiguration.PostInit();
NonAdminHttp.EnsureCanListenToWhenInNonAdminContext(ravenConfiguration.Port);
var ravenDbServer = new RavenDbServer(ravenConfiguration)
{
UseEmbeddedHttpServer = true,
};
ravenDbServer.Initialize(configureServer);
servers.Add(ravenDbServer);
try
{
using (var documentStore = new DocumentStore
{
Url = "http://localhost:" + port,
Conventions =
{
FailoverBehavior = FailoverBehavior.FailImmediately
},
DefaultDatabase = databaseName
}.Initialize())
{
CreateDefaultIndexes(documentStore);
}
}
catch
{
ravenDbServer.Dispose();
throw;
}
if (enableAuthentication)
{
EnableAuthentication(ravenDbServer.SystemDatabase);
ModifyConfiguration(ravenConfiguration);
ravenConfiguration.PostInit();
}
return ravenDbServer;
}
private static TIndexBase GetIndexInstance <TIndexBase>(IndexDefinition definition, RavenConfiguration configuration, IndexType type, long indexVersion)
where TIndexBase : AbstractStaticIndexBase
{
var key = GetCacheKey(definition);
Lazy <AbstractStaticIndexBase> result = _indexCache.GetOrAdd(key, _ => new Lazy <AbstractStaticIndexBase>(() => GenerateIndex(definition, configuration, type, indexVersion)));
try
{
return((TIndexBase)result.Value);
}
catch (Exception)
{
_indexCache.TryRemove(key, out _);
throw;
}
}
private static void RunRestoreOperation(string backupLocation, string databaseLocation)
{
try
{
var ravenConfiguration = new RavenConfiguration();
if(File.Exists(Path.Combine(backupLocation, "Raven.ravendb")))
{
ravenConfiguration.DefaultStorageTypeName =
"Raven.Storage.Managed.TransactionalStorage, Raven.Storage.Managed";
}
else if(Directory.Exists(Path.Combine(backupLocation, "new")))
{
ravenConfiguration.DefaultStorageTypeName = "Raven.Storage.Esent.TransactionalStorage, Raven.Storage.Esent";
}
DocumentDatabase.Restore(ravenConfiguration, backupLocation, databaseLocation);
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public async Task ShouldStoreTotalDocumentsSizeInPerformanceHint_ForRevisions()
{
using (var store = GetDocumentStore(new Options
{
ModifyDatabaseRecord = record => record.Settings[RavenConfiguration.GetKey(x => x.PerformanceHints.MaxNumberOfResults)] = "1"
}))
{
using (var session = store.OpenSession())
{
session.Store(new Company {
Name = "HR", Address = new Address {
Country = "Japan"
}
}, "companies/1");
var myRevisionsConfiguration = new RevisionsConfiguration
{
Default = new RevisionsCollectionConfiguration
{
Disabled = false,
PurgeOnDelete = false,
MinimumRevisionsToKeep = 5,
MinimumRevisionAgeToKeep = TimeSpan.FromDays(14),
},
Collections = new Dictionary <string, RevisionsCollectionConfiguration>
{
{ "companies", new RevisionsCollectionConfiguration {
} }
}
};
var revisionsConfigurationOperation = new ConfigureRevisionsOperation(myRevisionsConfiguration);
store.Maintenance.Send(revisionsConfigurationOperation);
session.SaveChanges();
}
using (var session = store.OpenSession())
{
var comp = session.Load <Company>("companies/1");
Assert.NotNull(comp);
comp.Phone = "123456789";
comp.Fax = "0987654321";
session.SaveChanges();
comp.Fax = "123456789";
comp.Phone = "0987654321";
session.SaveChanges();
var revisions = session
.Advanced
.Revisions
.GetFor <Company>("companies/1", pageSize: 4);
}
var database = await GetDatabase(store.Database);
database.NotificationCenter.Paging.UpdatePaging(null);
using (database.NotificationCenter.GetStored(out var actions))
{
var action = actions.First();
Assert.True(action.Json.TryGet("Details", out BlittableJsonReaderObject details));
Assert.True(details.TryGet("Actions", out BlittableJsonReaderObject detailsActions));
Assert.True(detailsActions.TryGet("GetRevisions", out BlittableJsonReaderArray detailsArray));
var index = detailsArray.GetByIndex <BlittableJsonReaderObject>(0);
Assert.NotNull(index);
Assert.True(index.TryGet("TotalDocumentsSizeInBytes", out int size));
Assert.True(size > 0);
}
}
}
private static bool RunServerInDebugMode(RavenConfiguration ravenConfiguration, bool lauchBrowser)
{
var sp = Stopwatch.StartNew();
using (var server = new RavenDbServer(ravenConfiguration))
{
sp.Stop();
var path = Path.Combine(Environment.CurrentDirectory, "default.raven");
if (File.Exists(path))
{
Console.WriteLine("Loading data from: {0}", path);
Smuggler.Smuggler.ImportData(ravenConfiguration.ServerUrl, path);
}
Console.WriteLine("Raven is ready to process requests. Build {0}, Version {1}", DocumentDatabase.BuildVersion, DocumentDatabase.ProductVersion);
Console.WriteLine("Server started in {0:#,#} ms", sp.ElapsedMilliseconds);
Console.WriteLine("Data directory: {0}", ravenConfiguration.DataDirectory);
Console.WriteLine("HostName: {0} Port: {1}, Storage: {2}", ravenConfiguration.HostName ?? "<any>",
ravenConfiguration.Port,
server.Database.TransactionalStorage.FriendlyName);
Console.WriteLine("Server Url: {0}", ravenConfiguration.ServerUrl);
Console.WriteLine("Press <enter> to stop or 'cls' and <enter> to clear the log");
if(lauchBrowser)
{
try
{
Process.Start(ravenConfiguration.ServerUrl);
}
catch (Exception e)
{
Console.WriteLine("Could not start browser: " + e.Message);
}
}
while (true)
{
var readLine = Console.ReadLine() ?? "";
switch (readLine.ToLowerInvariant())
{
case "cls":
Console.Clear();
break;
case "reset":
Console.Clear();
return true;
default:
return false;
}
}
}
}
public async Task ItemsShouldPreserveTheOrderInStorageAfterReplicatingToDestinationsWithRevisionsConfig()
{
var reasonableWaitTime = Debugger.IsAttached ? (int)TimeSpan.FromMinutes(15).TotalMilliseconds : (int)TimeSpan.FromSeconds(15).TotalMilliseconds;
using (var server = GetNewServer(new ServerCreationOptions
{
CustomSettings = new Dictionary <string, string> {
[RavenConfiguration.GetKey(x => x.Replication.MaxItemsCount)] = 1.ToString()
},
RegisterForDisposal = false
}))
{
using (var source = GetDocumentStore(new Options {
Server = server
}))
{
await SetupRevisionsForTest(server, source, source.Database);
var id = "user/0";
using (var session = source.OpenSession())
{
session.Store(new User {
Name = "EGR_0"
}, id);
session.SaveChanges();
}
using (var session = source.OpenSession())
{
using var stream = new MemoryStream(new byte[] { 1, 2, 3, 4, 5 });
session.Advanced.Attachments.Store(id, 0.ToString(), stream, "image/png");
session.SaveChanges();
}
using (var session = source.OpenAsyncSession())
{
var u = await session.LoadAsync <User>(id);
u.Name = "RGE" + u.Name.Substring(3);
await session.SaveChangesAsync();
}
List <ReplicationBatchItem> firstReplicationOrder = new List <ReplicationBatchItem>();
List <ReplicationBatchItem> secondReplicationOrder = new List <ReplicationBatchItem>();
var etag = 0L;
using var disposable1 = server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx);
using (var firstDestination = GetDocumentStore(new Options {
ModifyDatabaseName = s => GetDatabaseName() + "_destination", Server = server
}))
{
await SetupRevisionsForTest(server, firstDestination, firstDestination.Database);
var stats = source.Maintenance.Send(new GetStatisticsOperation());
var database = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(source.Database);
using var disposable2 = database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context1);
using (context1.OpenReadTransaction())
{
Assert.NotNull(database);
firstReplicationOrder.AddRange(ReplicationDocumentSender.GetReplicationItems(database, context1, etag, new ReplicationDocumentSender.ReplicationStats
{
DocumentRead = new OutgoingReplicationStatsScope(new OutgoingReplicationRunStats()),
AttachmentRead = new OutgoingReplicationStatsScope(new OutgoingReplicationRunStats())
}, false).Select(item => item.Clone(ctx)));
etag = firstReplicationOrder.Select(x => x.Etag).Max();
}
// Replication from source to firstDestination
await SetupReplicationAsync(source, firstDestination);
Assert.True(WaitForValue(() => AssertReplication(firstDestination, firstDestination.Database, stats), true, reasonableWaitTime, 333));
using (var session = source.OpenAsyncSession())
{
var u = await session.LoadAsync <User>(id);
u.Age = 30;
await session.SaveChangesAsync();
}
stats = source.Maintenance.Send(new GetStatisticsOperation());
Assert.True(WaitForValue(() => AssertReplication(firstDestination, firstDestination.Database, stats), true, reasonableWaitTime, 333));
using (context1.OpenReadTransaction())
{
firstReplicationOrder.AddRange(ReplicationDocumentSender.GetReplicationItems(database, context1, etag, new ReplicationDocumentSender.ReplicationStats
{
DocumentRead = new OutgoingReplicationStatsScope(new OutgoingReplicationRunStats()),
AttachmentRead = new OutgoingReplicationStatsScope(new OutgoingReplicationRunStats())
}, false).Select(item => item.Clone(ctx)));
}
var database2 = await server.ServerStore.DatabasesLandlord.TryGetOrCreateResourceStore(firstDestination.Database);
Assert.NotNull(database2);
// record the replicated items order
using var disposable3 = database2.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context);
using (context.OpenReadTransaction())
{
secondReplicationOrder.AddRange(ReplicationDocumentSender.GetReplicationItems(database2, context, 0, new ReplicationDocumentSender.ReplicationStats
{
DocumentRead = new OutgoingReplicationStatsScope(new OutgoingReplicationRunStats()),
AttachmentRead = new OutgoingReplicationStatsScope(new OutgoingReplicationRunStats())
}, false).Select(item => item.Clone(ctx)));
}
// remove the 1st doc sent, because of its change in the middle of the 1st replication, so it was sent twice
Assert.True(firstReplicationOrder.Remove(firstReplicationOrder.First(y => y is DocumentReplicationItem doc && doc.Flags.Contain(DocumentFlags.Revision) == false)));
Assert.Equal(firstReplicationOrder.Count, secondReplicationOrder.Count);
for (int i = 0; i < secondReplicationOrder.Count; i++)
{
var item = firstReplicationOrder[i];
var item2 = secondReplicationOrder[i];
if (item is AttachmentReplicationItem a1 && item2 is AttachmentReplicationItem a2)
{
Assert.Equal(a1.Key.ToString(), a2.Key.ToString());
Assert.Equal(a1.ChangeVector, a2.ChangeVector);
Assert.Equal(a1.Type, a2.Type);
}
private static void InteractiveRun(string[] args)
{
switch (GetArgument(args))
{
case "install":
AdminRequired(InstallAndStart, "/install");
break;
case "uninstall":
AdminRequired(EnsureStoppedAndUninstall, "/uninstall");
break;
case "start":
AdminRequired(StartService, "/start");
break;
case "restart":
AdminRequired(RestartService, "/restart");
break;
case "stop":
AdminRequired(StopService, "/stop");
break;
case "restore":
if (args.Length != 3)
{
PrintUsage();
break;
}
RunRestoreOperation(args[1], args[2]);
break;
case "debug":
RunInDebugMode(anonymousUserAccessMode: null, ravenConfiguration: new RavenConfiguration());
break;
case "ram":
RunInDebugMode(anonymousUserAccessMode: AnonymousUserAccessMode.All, ravenConfiguration: new RavenConfiguration
{
RunInMemory = true,
Settings =
{
{ "Raven/RunInMemory", "true" }
}
});
break;
#if DEBUG
case "test":
var dataDirectory = new RavenConfiguration().DataDirectory;
IOExtensions.DeleteDirectory(dataDirectory);
RunInDebugMode(anonymousUserAccessMode: AnonymousUserAccessMode.All, ravenConfiguration: new RavenConfiguration());
break;
#endif
default:
PrintUsage();
break;
}
}
internal void ValidateAllowedDestinations()
{
if (AllowedDestinations == null)
{
return;
}
if (AllowedDestinations.Contains(_noneDestination, StringComparer.OrdinalIgnoreCase))
{
if (AllowedDestinations.Length > 1)
{
throw new ArgumentException($"If you specify \"None\" under '{RavenConfiguration.GetKey(x => x.Backup.AllowedDestinations)}' then it must be the only value.");
}
return;
}
foreach (var dest in AllowedDestinations)
{
if (_allDestinations.Contains(dest, StringComparer.OrdinalIgnoreCase))
{
continue;
}
throw new ArgumentException($"The destination '{dest}' defined in the configuration under '{RavenConfiguration.GetKey(x => x.Backup.AllowedDestinations)}' is unknown. Make sure to use the following destinations: {string.Join(", ", _allDestinations)}.");
}
}
public static void Main(string[] args)
{
string path = string.Concat(Environment.CurrentDirectory, "\\settings.json");
var configuration = new RavenConfiguration(null, ResourceType.Server, path);
configuration.Initialize();
LoggingSource.Instance.SetupLogMode(configuration.Logs.Mode, configuration.Logs.Path.FullPath);
if (Logger.IsInfoEnabled)
{
Logger.Info($"Logging to {configuration.Logs.Path} set to {configuration.Logs.Mode} level.");
}
if (Logger.IsOperationsEnabled)
{
Logger.Operations(RavenCli.GetInfoText());
}
RestartServer = () =>
{
ResetServerMre.Set();
ShutdownServerMre.Set();
};
var rerun = false;
RavenConfiguration configBeforeRestart = configuration;
do
{
try
{
using (var server = new RavenServer(configuration))
{
try
{
try
{
server.OpenPipes();
}
catch (Exception e)
{
if (Logger.IsInfoEnabled)
{
Logger.Info("Unable to OpenPipe. Admin Channel will not be available to the user", e);
}
Console.WriteLine("Warning: Admin Channel is not available");
}
server.Initialize();
var prevColor = Console.ForegroundColor;
Console.Write("Server available on: ");
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"{server.ServerStore.GetNodeHttpServerUrl()}");
Console.ForegroundColor = prevColor;
var tcpServerStatus = server.GetTcpServerStatus();
prevColor = Console.ForegroundColor;
Console.Write("Tcp listening on ");
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($"{string.Join(", ", tcpServerStatus.Listeners.Select(l => l.LocalEndpoint))}");
Console.ForegroundColor = prevColor;
IsRunningNonInteractive = false;
Console.WriteLine("Server start completed");
Console.ReadLine();
Console.WriteLine("Starting shut down...");
if (Logger.IsInfoEnabled)
{
Logger.Info("Server is shutting down");
}
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations("Failed to initialize the server", e);
}
Console.WriteLine(e);
}
}
Console.WriteLine("Shutdown completed");
}
catch (Exception e)
{
Console.WriteLine("Error during shutdown");
Console.WriteLine(e);
}
finally
{
if (Logger.IsOperationsEnabled)
{
Logger.OperationsAsync("Server has shut down").Wait(TimeSpan.FromSeconds(15));
}
}
} while (rerun);
}
public InMemoryRavenPersistenceFactory(RavenConfiguration config) : base(config)
{
}
public InMemoryRavenPersistenceFactory(RavenConfiguration config)
: base(config)
{
}
public ScriptRunnerCache(DocumentDatabase database, RavenConfiguration configuration)
{
_database = database;
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
}