public DatabaseBulkOperations(DocumentDatabase database, TransactionInformation transactionInformation, CancellationTokenSource tokenSource, CancellationTimeout timeout)
{
this.database = database;
this.transactionInformation = transactionInformation;
this.tokenSource = tokenSource;
this.timeout = timeout;
}
protected override bool TryGetOrCreateResourceStore(string tenantId, out IResourceStore database)
{
if (ResourcesStoresCache.TryGetValue(tenantId, out database))
return true;
JsonDocument jsonDocument;
using (DocumentRetriever.DisableReadTriggers())
jsonDocument = DefaultDatabase.Get("Raven/Databases/" + tenantId, null);
if (jsonDocument == null)
return false;
var document = jsonDocument.DataAsJson.JsonDeserialization<DatabaseDocument>();
database = ResourcesStoresCache.GetOrAdd(tenantId, s =>
{
var config = new InMemoryRavenConfiguration
{
Settings = DefaultConfiguration.Settings,
};
config.Settings["Raven/VirtualDir"] = config.Settings["Raven/VirtualDir"] + "/" + tenantId;
foreach (var setting in document.Settings)
{
config.Settings[setting.Key] = setting.Value;
}
config.Initialize();
var documentDatabase = new DocumentDatabase(config);
documentDatabase.SpinBackgroundWorkers();
return documentDatabase;
});
return true;
}
public BackupOperation(DocumentDatabase database, string backupSourceDirectory, string backupDestinationDirectory, bool incrementalBackup,
DatabaseDocument databaseDocument)
: base(database, backupSourceDirectory, backupDestinationDirectory, incrementalBackup, databaseDocument)
{
instance = ((TransactionalStorage) database.TransactionalStorage).Instance;
backupConfigPath = Path.Combine(backupDestinationDirectory, "RavenDB.Backup");
}
public void SilverlightWasRequested(DocumentDatabase database)
{
if (database.GetIndexDefinition("Raven/DocumentsByEntityName") == null)
{
database.PutIndex("Raven/DocumentsByEntityName", new IndexDefinition
{
Map =
@"from doc in docs
let Tag = doc[""@metadata""][""Raven-Entity-Name""]
select new { Tag, LastModified = (DateTime)doc[""@metadata""][""Last-Modified""] };",
Indexes =
{
{"Tag", FieldIndexing.NotAnalyzed},
{"LastModified", FieldIndexing.NotAnalyzed},
},
Stores =
{
{"Tag", FieldStorage.No},
{"LastModified", FieldStorage.No}
},
TermVectors =
{
{"Tag", FieldTermVector.No},
{"LastModified", FieldTermVector.No}
}
});
}
}
private void OnTransformerChange(DocumentDatabase documentDatabase, TransformerChangeNotification notification)
{
var transformerName = notification.Name;
switch (notification.Type)
{
case TransformerChangeTypes.TransformerAdded:
//if created transformer with the same name as deleted one, we should prevent its deletion replication
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.Remove(Constants.RavenReplicationTransformerTombstones, transformerName));
break;
case TransformerChangeTypes.TransformerRemoved:
//If we don't have any destination to replicate to (we are probably slave node)
//we shouldn't keep a tombstone since we are not going to remove it anytime
//and keeping it prevents us from getting that transformer created again.
if (GetReplicationDestinations().Count == 0)
return;
var metadata = new RavenJObject
{
{Constants.RavenTransformerDeleteMarker, true},
{Constants.RavenReplicationSource, Database.TransactionalStorage.Id.ToString()},
{Constants.RavenReplicationVersion, ReplicationHiLo.NextId(Database)},
{"TransformerVersion", notification.Version }
};
Database.TransactionalStorage.Batch(accessor =>
accessor.Lists.Set(Constants.RavenReplicationTransformerTombstones, transformerName, metadata, UuidType.Transformers));
break;
}
}
public BackupOperation(DocumentDatabase database, string src, string to)
{
instance = database.TransactionalStorage.Instance;
this.database = database;
this.to = to;
this.src = src;
}
public CurrentIndexingScope(DocumentDatabase database, string index)
{
this.database = database;
this.index = index;
LoadDocumentCount = 0;
LoadDocumentDuration = new Stopwatch();
}
public void Execute(DocumentDatabase database)
{
HashSet<Guid> resourceManagersRequiringRecovery = new HashSet<Guid>();
database.TransactionalStorage.Batch(actions =>
{
foreach (var txId in actions.Transactions.GetTransactionIds())
{
var attachment = actions.Attachments.GetAttachment("transactions/recoveryInformation/" + txId);
if (attachment == null)//Prepare was not called, there is no recovery information
{
actions.Transactions.RollbackTransaction(txId);
}
else
{
Guid resourceManagerId;
if (Guid.TryParse(attachment.Metadata.Value<string>("Resource-Manager-Id"), out resourceManagerId) == false)
{
actions.Transactions.RollbackTransaction(txId);
}
else
{
resourceManagersRequiringRecovery.Add(resourceManagerId);
TransactionManager.Reenlist(resourceManagerId, attachment.Data, new InternalEnlistment(database, txId));
}
}
}
});
foreach (var rm in resourceManagersRequiringRecovery)
{
TransactionManager.RecoveryComplete(rm);
}
}
public OwinHttpServer(InMemoryRavenConfiguration config, DocumentDatabase db = null, bool useHttpServer = true, Action<RavenDBOptions> configure = null)
{
startup = new Startup(config, db);
if (configure != null)
configure(startup.Options);
owinEmbeddedHost = OwinEmbeddedHost.Create(app => startup.Configuration(app));
if (!useHttpServer)
{
return;
}
server = WebApp.Start("http://+:" + config.Port, app => //TODO DH: configuration.ServerUrl doesn't bind properly
{
var listener = (HttpListener) app.Properties["System.Net.HttpListener"];
if (listener != null)
{
new WindowsAuthConfigureHttpListener().Configure(listener, config);
}
startup.Configuration(app);
app.Use(async (context, _) =>
{
context.Response.StatusCode = 404;
context.Response.ReasonPhrase = "Not Found";
await context.Response.Body.WriteAsync(NotFoundBody, 0, NotFoundBody.Length);
});
});
}
public static void Execute(this ICommandData self, DocumentDatabase database)
{
var deleteCommandData = self as DeleteCommandData;
if (deleteCommandData != null)
{
database.Delete(deleteCommandData.Key, deleteCommandData.Etag, deleteCommandData.TransactionInformation);
return;
}
var putCommandData = self as PutCommandData;
if (putCommandData != null)
{
var putResult = database.Put(putCommandData.Key, putCommandData.Etag, putCommandData.Document, putCommandData.Metadata, putCommandData.TransactionInformation);
putCommandData.Etag = putResult.ETag;
putCommandData.Key = putResult.Key;
return;
}
var patchCommandData = self as PatchCommandData;
if (patchCommandData != null)
{
database.ApplyPatch(patchCommandData.Key, patchCommandData.Etag, patchCommandData.Patches, patchCommandData.TransactionInformation);
var doc = database.Get(patchCommandData.Key, patchCommandData.TransactionInformation);
if (doc != null)
{
patchCommandData.Metadata = doc.Metadata;
patchCommandData.Etag = doc.Etag;
}
return;
}
}
protected ActionsBase(DocumentDatabase database, SizeLimitedConcurrentDictionary<string, TouchedDocumentInfo> recentTouches, IUuidGenerator uuidGenerator, ILog log)
{
Database = database;
RecentTouches = recentTouches;
UuidGenerator = uuidGenerator;
Log = log;
}
public AnonymousObjectToLuceneDocumentConverter(DocumentDatabase database, IndexDefinition indexDefinition, AbstractViewGenerator viewGenerator, ILog log)
{
this.database = database;
this.indexDefinition = indexDefinition;
this.viewGenerator = viewGenerator;
this.log = log;
}
public IndexReplacer(DocumentDatabase database)
{
Database = database;
database.Notifications.OnDocumentChange += (db, notification, metadata) =>
{
if (notification.Id == null)
return;
if (notification.Id.StartsWith(Constants.IndexReplacePrefix, StringComparison.OrdinalIgnoreCase) == false)
return;
var replaceIndexName = notification.Id.Substring(Constants.IndexReplacePrefix.Length);
if (notification.Type == DocumentChangeTypes.Delete)
{
HandleIndexReplaceDocumentDelete(replaceIndexName);
return;
}
var document = db.Documents.Get(notification.Id, null);
var replaceIndexId = HandleIndexReplaceDocument(document);
if (replaceIndexId != null)
ReplaceIndexes(new []{ replaceIndexId.Value });
};
Initialize();
}
public void Execute(DocumentDatabase database)
{
if (database.Configuration.RunInMemory)
return;
var dataDirectory = Path.GetFullPath(database.Configuration.DataDirectory);
var desktopIni = Path.Combine(dataDirectory, "desktop.ini");
var icon = Path.Combine(dataDirectory, "raven-data.ico");
if (File.Exists(desktopIni) && File.Exists(icon))
return;
using (var iconFile = typeof(CreateFolderIcon).Assembly.GetManifestResourceStream("Raven.Database.Server.WebUI.raven-data.ico"))
{
File.WriteAllBytes(icon, iconFile.ReadData());
}
File.WriteAllText(desktopIni, string.Format(@"
[.ShellClassInfo]
IconResource={0},0
[ViewState]
Mode=
Vid=
FolderType=Generic
",
icon));
File.SetAttributes(desktopIni, FileAttributes.Hidden | FileAttributes.System | FileAttributes.Archive);
File.SetAttributes(dataDirectory, FileAttributes.ReadOnly);
}
public IndexReplicationTask(DocumentDatabase database, HttpRavenRequestFactory httpRavenRequestFactory, ReplicationTask replication)
: base(database, httpRavenRequestFactory, replication)
{
replicationFrequency = TimeSpan.FromSeconds(database.Configuration.IndexAndTransformerReplicationLatencyInSec); //by default 10 min
lastQueriedFrequency = TimeSpan.FromSeconds(database.Configuration.TimeToWaitBeforeRunningIdleIndexes.TotalSeconds / 2);
TimeToWaitBeforeSendingDeletesOfIndexesToSiblings = TimeSpan.FromMinutes(1);
}
public void Execute(DocumentDatabase database)
{
if (database.Configuration.IsSystemDatabase() == false)
return;
if (ValidateLicense.CurrentLicense.IsCommercial && ValidateLicense.CurrentLicense.Error == false)
{
SetValidCommercialLicenseMarker(database);
ValidateLicense.CurrentLicense.ValidCommercialLicenseSeen = true;
}
else
{
var marker = GetLastSeenValidCommercialLicenseMarker(database);
if (marker != null)
{
ValidateLicense.CurrentLicense.ValidCommercialLicenseSeen = true;
}
}
if (Authentication.IsEnabled == false && database.Configuration.AnonymousUserAccessMode != AnonymousUserAccessMode.Admin)
{
throw new InvalidOperationException("Cannot set Raven/AnonymousAccess to '" + database.Configuration.AnonymousUserAccessMode+"' without a valid license.\r\n" +
"This RavenDB instance doesn't have a license, and the only valid Raven/AnonymousAccess setting is: Admin\r\n" +
"Please change to Raven/AnonymousAccess to Admin, or install a valid license.");
}
}
public void RecordWriteError(Exception e, DocumentDatabase database, int count = 1, DateTime? newErrorTime = null)
{
WriteErrorCount += count;
if (WriteErrorCount < 100)
return;
if (WriteErrorCount <= SuccessCount)
return;
if (newErrorTime != null)
{
LastErrorTime = newErrorTime.Value;
return;
}
database.AddAlert(LastAlert = new Alert
{
AlertLevel = AlertLevel.Error,
CreatedAt = SystemTime.UtcNow,
Message = "Could not tolerate write error ratio and stopped current replication cycle for " + name + Environment.NewLine + this,
Title = "Sql Replication write error hit ratio too high",
Exception = e.ToString(),
UniqueKey = "Sql Replication Write Error Ratio: " + name
});
throw new InvalidOperationException("Could not tolerate write error ratio and stopped current replication cycle for " + name + Environment.NewLine + this, e);
}
public ScriptedJsonPatcher(DocumentDatabase database = null)
{
if (database == null)
{
maxSteps = 10 * 1000;
additionalStepsPerSize = 5;
loadDocument = (s =>
{
throw new InvalidOperationException(
"Cannot load by id without database context");
});
}
else
{
maxSteps = database.Configuration.MaxStepsForScript;
additionalStepsPerSize = database.Configuration.AdditionalStepsForScriptBasedOnDocumentSize;
loadDocument = id =>
{
JsonDocument jsonDocument;
if (!createdDocDict.TryGetValue(id, out jsonDocument))
jsonDocument = database.Get(id, null);
return jsonDocument == null ? null : jsonDocument.ToJson();
};
}
}
public IndexStorage(IndexDefinitionStorage indexDefinitionStorage, InMemoryRavenConfiguration configuration, DocumentDatabase documentDatabase)
{
this.indexDefinitionStorage = indexDefinitionStorage;
this.configuration = configuration;
path = configuration.IndexStoragePath;
if (Directory.Exists(path) == false && configuration.RunInMemory == false)
Directory.CreateDirectory(path);
if (configuration.RunInMemory == false)
{
var crashMarkerPath = Path.Combine(path, "indexing.crash-marker");
if (File.Exists(crashMarkerPath))
{
// the only way this can happen is if we crashed because of a power outage
// in this case, we consider all open indexes to be corrupt and force them
// to be reset. This is because to get better perf, we don't flush the files to disk,
// so in the case of a power outage, we can't be sure that there wasn't still stuff in
// the OS buffer that wasn't written yet.
configuration.ResetIndexOnUncleanShutdown = true;
}
// The delete on close ensures that the only way this file will exists is if there was
// a power outage while the server was running.
crashMarker = File.Create(crashMarkerPath, 16, FileOptions.DeleteOnClose);
}
foreach (var indexName in indexDefinitionStorage.IndexNames)
{
OpenIndexOnStartup(documentDatabase, indexName);
}
}
public SmugglerExporter(DocumentDatabase database, ExportOptions options)
{
this.database = database;
this.options = options;
FillExportTypes();
}
public void Execute(DocumentDatabase database)
{
Database = database;
Database.OnDocumentChange += (sender, notification) =>
{
if (notification.Id == null)
return;
if (!notification.Id.StartsWith("Raven/SqlReplication/Configuration/", StringComparison.InvariantCultureIgnoreCase))
return;
replicationConfigs = null;
statistics.Clear();
};
GetReplicationStatus();
prefetchingBehavior = new PrefetchingBehavior(Database.WorkContext, new IndexBatchSizeAutoTuner(Database.WorkContext));
var task = Task.Factory.StartNew(() =>
{
using (LogContext.WithDatabase(database.Name))
{
try
{
BackgroundSqlReplication();
}
catch (Exception e)
{
log.ErrorException("Fatal failure when replicating to SQL. All SQL Replication activity STOPPED", e);
}
}
}, TaskCreationOptions.LongRunning);
database.ExtensionsState.GetOrAdd(typeof(SqlReplicationTask).FullName, k => new DisposableAction(task.Wait));
}
public BackupOperation(DocumentDatabase systemDatabase, RavenFileSystem filesystem, string backupSourceDirectory, string backupDestinationDirectory, bool incrementalBackup,
FileSystemDocument filesystemDocument)
: base(systemDatabase, filesystem, backupSourceDirectory, backupDestinationDirectory, incrementalBackup, filesystemDocument)
{
instance = ((TransactionalStorage)filesystem.Storage).Instance;
backupConfigPath = Path.Combine(backupDestinationDirectory, "RavenDB.Backup");
}
public RavenDBOptions(InMemoryRavenConfiguration configuration, DocumentDatabase db = null)
{
if (configuration == null)
throw new ArgumentNullException("configuration");
try
{
HttpEndpointRegistration.RegisterHttpEndpointTarget();
HttpEndpointRegistration.RegisterAdminLogsTarget();
if (db == null)
{
configuration.UpdateDataDirForLegacySystemDb();
systemDatabase = new DocumentDatabase(configuration);
systemDatabase.SpinBackgroundWorkers();
}
else
{
systemDatabase = db;
}
fileSystemLandlord = new FileSystemsLandlord(systemDatabase);
databasesLandlord = new DatabasesLandlord(systemDatabase);
countersLandlord = new CountersLandlord(systemDatabase);
requestManager = new RequestManager(databasesLandlord);
mixedModeRequestAuthorizer = new MixedModeRequestAuthorizer();
mixedModeRequestAuthorizer.Initialize(systemDatabase, new RavenServer(databasesLandlord.SystemDatabase, configuration));
}
catch
{
if (systemDatabase != null)
systemDatabase.Dispose();
throw;
}
}
public ReplicationTopologyDiscoverer(DocumentDatabase database, RavenJArray @from, int ttl, ILog log)
{
this.database = database;
this.ttl = ttl;
this.log = log;
this.@from = @from;
requestFactory = new HttpRavenRequestFactory();
}
private void SetValidCommercialLicenseMarker(DocumentDatabase database)
{
database.TransactionalStorage.Batch(
accessor => accessor.Lists.Set(LicensingListName, ValidationMarkerName, new RavenJObject()
{
{"ValidationTime", SystemTime.UtcNow}
}, UuidType.Documents));
}
public EsentInFlightTransactionalState(DocumentDatabase database, TransactionalStorage storage, CommitTransactionGrbit txMode, Func<string, Etag, RavenJObject, RavenJObject, TransactionInformation, PutResult> databasePut, Func<string, Etag, TransactionInformation, bool> databaseDelete)
: base(databasePut, databaseDelete)
{
_database = database;
this.storage = storage;
this.txMode = txMode;
timer = database.TimerManager.NewTimer(CleanupOldTransactions, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30));
}
public void Initialize(DocumentDatabase database, IRavenServer theServer)
{
this.database = database;
this.settings = database.Configuration;
this.server = theServer;
Initialize();
}
public void Execute(DocumentDatabase database)
{
var oldBackup = database.Documents.Get(RestoreInProgress.RavenRestoreInProgressDocumentKey,null);
if (oldBackup == null)
return;
database.Documents.Delete(RestoreInProgress.RavenRestoreInProgressDocumentKey, null, null);
}
private ListItem GetLastSeenValidCommercialLicenseMarker(DocumentDatabase database)
{
ListItem lastSeenValidCommercialLicense = null;
database.TransactionalStorage.Batch(
accessor => { lastSeenValidCommercialLicense = accessor.Lists.Read(LicensingListName, ValidationMarkerName); });
return lastSeenValidCommercialLicense;
}
public CurrentTransformationScope(DocumentDatabase database, DocumentRetriever documentRetriever)
{
this.database = database;
retriever = documentRetriever;
old = current;
current = this;
}
private static void ActualTest(int numberOfUsers, string[] locations, Index index,
MapReduceIndexingContext mapReduceContext, IIndexingWork reducer, DocumentDatabase database, string outputToCollectionName)
{
using (index._contextPool.AllocateOperationContext(out TransactionOperationContext indexContext))
{
ulong hashOfReduceKey = 73493;
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < numberOfUsers; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 1,
["Location"] = locations[i % locations.Length]
}, $"users/{i}"))
{
store.Add(i, mappedResult);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, indexContext));
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
tx.Commit();
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
using (context.OpenReadTransaction())
{
var results = database.DocumentsStorage.GetDocumentsFrom(context, outputToCollectionName, 0, 0, int.MaxValue).ToList();
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected, results[i].Data["Count"]);
}
}
// update
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < locations.Length; i++)
{
using (var mappedResult = indexContext.ReadObject(new DynamicJsonValue
{
["Count"] = 2, // increased by 1
["Location"] = locations[i % locations.Length]
}, $"users/{i}"))
{
store.Add(i, mappedResult);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, indexContext));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
tx.Commit();
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
using (context.OpenReadTransaction())
{
var results = database.DocumentsStorage.GetDocumentsFrom(context, outputToCollectionName, 0, 0, int.MaxValue).ToList();
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected + 1, results[i].Data["Count"]);
}
}
// delete
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < locations.Length; i++)
{
store.Delete(i);
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, indexContext));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
tx.Commit();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
using (context.OpenReadTransaction())
{
var results = database.DocumentsStorage.GetDocumentsFrom(context, outputToCollectionName, 0, 0, int.MaxValue).ToList();
Assert.Equal(locations.Length, results.Count);
for (int i = 0; i < locations.Length; i++)
{
Assert.Equal(locations[i], results[i].Data["Location"].ToString());
long expected = numberOfUsers / locations.Length + numberOfUsers % (locations.Length - i);
Assert.Equal(expected - 1, results[i].Data["Count"]);
}
}
// delete entries for one reduce key
using (var tx = indexContext.OpenWriteTransaction())
{
mapReduceContext.MapPhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .MapPhaseTreeName);
mapReduceContext.ReducePhaseTree = tx.InnerTransaction.CreateTree(MapReduceIndexBase <MapIndexDefinition, IndexField> .ReducePhaseTreeName);
var store = new MapReduceResultsStore(hashOfReduceKey, MapResultsStorageType.Tree, indexContext, mapReduceContext, true);
for (int i = 0; i < numberOfUsers; i++)
{
if (i % locations.Length == 0)
{
store.Delete(i);
}
}
mapReduceContext.StoreByReduceKeyHash.Add(hashOfReduceKey, store);
var writeOperation =
new Lazy <IndexWriteOperation>(() => index.IndexPersistence.OpenIndexWriter(tx.InnerTransaction, indexContext));
try
{
var stats = new IndexingStatsScope(new IndexingRunStats());
reducer.Execute(null, indexContext,
writeOperation,
stats, CancellationToken.None);
using (var indexWriteOperation = writeOperation.Value)
{
indexWriteOperation.Commit(stats);
}
index.IndexPersistence.RecreateSearcher(tx.InnerTransaction);
mapReduceContext.Dispose();
tx.Commit();
}
finally
{
if (writeOperation.IsValueCreated)
{
writeOperation.Value.Dispose();
}
}
}
using (var context = DocumentsOperationContext.ShortTermSingleUse(database))
using (context.OpenReadTransaction())
{
var results = database.DocumentsStorage.GetDocumentsFrom(context, outputToCollectionName, 0, 0, int.MaxValue).ToList();
Assert.Equal(locations.Length - 1, results.Count);
}
}
}
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
{
var databaseName = RestoreFromConfiguration.DatabaseName;
var result = new RestoreResult
{
DataDirectory = RestoreFromConfiguration.DataDirectory
};
try
{
var filesToRestore = await GetOrderedFilesToRestore();
using (_serverStore.ContextPool.AllocateOperationContext(out JsonOperationContext serverContext))
{
if (onProgress == null)
{
onProgress = _ => { }
}
;
Stopwatch sw = null;
RestoreSettings restoreSettings = null;
var firstFile = filesToRestore[0];
var extension = Path.GetExtension(firstFile);
var snapshotRestore = false;
if ((extension == Constants.Documents.PeriodicBackup.SnapshotExtension) ||
(extension == Constants.Documents.PeriodicBackup.EncryptedSnapshotExtension))
{
onProgress.Invoke(result.Progress);
snapshotRestore = true;
sw = Stopwatch.StartNew();
if (extension == Constants.Documents.PeriodicBackup.EncryptedSnapshotExtension)
{
_hasEncryptionKey = RestoreFromConfiguration.EncryptionKey != null ||
RestoreFromConfiguration.BackupEncryptionSettings?.Key != null;
}
// restore the snapshot
restoreSettings = await SnapshotRestore(serverContext, firstFile, onProgress, result);
if (restoreSettings != null && RestoreFromConfiguration.SkipIndexes)
{
// remove all indexes from the database record
restoreSettings.DatabaseRecord.AutoIndexes = null;
restoreSettings.DatabaseRecord.Indexes = null;
}
// removing the snapshot from the list of files
filesToRestore.RemoveAt(0);
}
else
{
result.SnapshotRestore.Skipped = true;
result.SnapshotRestore.Processed = true;
onProgress.Invoke(result.Progress);
}
if (restoreSettings == null)
{
restoreSettings = new RestoreSettings
{
DatabaseRecord = new DatabaseRecord(databaseName)
{
// we only have a smuggler restore
// use the encryption key to encrypt the database
Encrypted = _hasEncryptionKey
}
};
DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
}
var databaseRecord = restoreSettings.DatabaseRecord;
databaseRecord.Settings ??= new Dictionary <string, string>();
var runInMemoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.RunInMemory);
databaseRecord.Settings.Remove(runInMemoryConfigurationKey);
if (_serverStore.Configuration.Core.RunInMemory)
{
databaseRecord.Settings[runInMemoryConfigurationKey] = "false";
}
var dataDirectoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.DataDirectory);
databaseRecord.Settings.Remove(dataDirectoryConfigurationKey); // removing because we want to restore to given location, not to serialized in backup one
if (_restoringToDefaultDataDirectory == false)
{
databaseRecord.Settings[dataDirectoryConfigurationKey] = RestoreFromConfiguration.DataDirectory;
}
if (_hasEncryptionKey)
{
// save the encryption key so we'll be able to access the database
_serverStore.PutSecretKey(RestoreFromConfiguration.EncryptionKey,
databaseName, overwrite: false);
}
var addToInitLog = new Action <string>(txt => // init log is not save in mem during RestoreBackup
{
var msg = $"[RestoreBackup] {DateTime.UtcNow} :: Database '{databaseName}' : {txt}";
if (Logger.IsInfoEnabled)
{
Logger.Info(msg);
}
});
var configuration = _serverStore
.DatabasesLandlord
.CreateDatabaseConfiguration(databaseName, ignoreDisabledDatabase: true, ignoreBeenDeleted: true, ignoreNotRelevant: true, databaseRecord);
using (var database = new DocumentDatabase(databaseName, configuration, _serverStore, addToInitLog))
{
// smuggler needs an existing document database to operate
var options = InitializeOptions.SkipLoadingDatabaseRecord;
if (snapshotRestore)
{
options |= InitializeOptions.GenerateNewDatabaseId;
}
database.Initialize(options);
databaseRecord.Topology = new DatabaseTopology();
// restoring to the current node only
databaseRecord.Topology.Members.Add(_nodeTag);
// we are currently restoring, shouldn't try to access it
databaseRecord.DatabaseState = DatabaseStateStatus.RestoreInProgress;
await SaveDatabaseRecordAsync(databaseName, databaseRecord, restoreSettings.DatabaseValues, result, onProgress);
database.ClusterTransactionId = databaseRecord.Topology.ClusterTransactionIdBase64;
database.DatabaseGroupId = databaseRecord.Topology.DatabaseTopologyIdBase64;
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
if (snapshotRestore)
{
await RestoreFromSmugglerFile(onProgress, database, firstFile, context);
await SmugglerRestore(database, filesToRestore, context, databaseRecord, onProgress, result);
result.SnapshotRestore.Processed = true;
var summary = database.GetDatabaseSummary();
result.Documents.ReadCount += summary.DocumentsCount;
result.Documents.Attachments.ReadCount += summary.AttachmentsCount;
result.Counters.ReadCount += summary.CounterEntriesCount;
result.RevisionDocuments.ReadCount += summary.RevisionsCount;
result.Conflicts.ReadCount += summary.ConflictsCount;
result.Indexes.ReadCount += databaseRecord.GetIndexesCount();
result.CompareExchange.ReadCount += summary.CompareExchangeCount;
result.CompareExchangeTombstones.ReadCount += summary.CompareExchangeTombstonesCount;
result.Identities.ReadCount += summary.IdentitiesCount;
result.TimeSeries.ReadCount += summary.TimeSeriesSegmentsCount;
result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms");
onProgress.Invoke(result.Progress);
}
else
{
await SmugglerRestore(database, filesToRestore, context, databaseRecord, onProgress, result);
}
DisableOngoingTasksIfNeeded(databaseRecord);
Raven.Server.Smuggler.Documents.DatabaseSmuggler.EnsureProcessed(result, skipped: false);
onProgress.Invoke(result.Progress);
}
if (snapshotRestore)
{
RegenerateDatabaseIdInIndexes(configuration, database);
}
}
// after the db for restore is done, we can safely set the db state to normal and write the DatabaseRecord
databaseRecord.DatabaseState = DatabaseStateStatus.Normal;
await SaveDatabaseRecordAsync(databaseName, databaseRecord, null, result, onProgress);
return(result);
}
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations("Failed to restore database", e);
}
var alert = AlertRaised.Create(
RestoreFromConfiguration.DatabaseName,
"Failed to restore database",
$"Could not restore database named {RestoreFromConfiguration.DatabaseName}",
AlertType.RestoreError,
NotificationSeverity.Error,
details: new ExceptionDetails(e));
_serverStore.NotificationCenter.Add(alert);
using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
{
bool databaseExists;
using (context.OpenReadTransaction())
{
databaseExists = _serverStore.Cluster.DatabaseExists(context, RestoreFromConfiguration.DatabaseName);
}
if (databaseExists == false)
{
// delete any files that we already created during the restore
IOExtensions.DeleteDirectory(RestoreFromConfiguration.DataDirectory);
}
else
{
try
{
var deleteResult = await _serverStore.DeleteDatabaseAsync(RestoreFromConfiguration.DatabaseName, true, new[] { _serverStore.NodeTag },
RaftIdGenerator.DontCareId);
await _serverStore.Cluster.WaitForIndexNotification(deleteResult.Index, TimeSpan.FromSeconds(60));
}
catch (TimeoutException te)
{
result.AddError($"Failed to delete the database {databaseName} after a failed restore. " +
$"In order to restart the restore process this database needs to be deleted manually. Exception: {te}.");
onProgress.Invoke(result.Progress);
}
}
}
result.AddError($"Error occurred during restore of database {databaseName}. Exception: {e}");
onProgress.Invoke(result.Progress);
throw;
}
finally
{
Dispose();
}
void RegenerateDatabaseIdInIndexes(RavenConfiguration configuration, DocumentDatabase database)
{
// this code will generate new DatabaseId for each index.
// This is something that we need to do when snapshot restore is executed to match the newly generated database id
var indexesPath = configuration.Indexing.StoragePath.FullPath;
if (Directory.Exists(indexesPath) == false)
{
return;
}
foreach (var indexPath in Directory.GetDirectories(indexesPath))
{
Index index = null;
try
{
index = Index.Open(indexPath, database, generateNewDatabaseId: true);
}
catch (Exception e)
{
result.AddError($"Could not open index from path '{indexPath}'. Error: {e.Message}");
}
finally
{
index?.Dispose();
}
}
}
}
private bool TryAuthorize(RouteInformation route, HttpContext context, DocumentDatabase database, out RavenServer.AuthenticationStatus authenticationStatus)
{
var feature = context.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection;
if (feature.WrittenToAuditLog == 0) // intentionally racy, we'll check it again later
{
var auditLog = LoggingSource.AuditLog.IsInfoEnabled ? LoggingSource.AuditLog.GetLogger("RequestRouter", "Audit") : null;
if (auditLog != null)
{
// only one thread will win it, technically, there can't really be threading
// here, because there is a single connection, but better to be safe
if (Interlocked.CompareExchange(ref feature.WrittenToAuditLog, 1, 0) == 0)
{
if (feature.WrongProtocolMessage != null)
{
auditLog.Info($"Connection from {context.Connection.RemoteIpAddress}:{context.Connection.RemotePort} " +
$"used the wrong protocol and will be rejected. {feature.WrongProtocolMessage}");
}
else
{
auditLog.Info($"Connection from {context.Connection.RemoteIpAddress}:{context.Connection.RemotePort} " +
$"with certificate '{feature.Certificate?.Subject} ({feature.Certificate?.Thumbprint})', status: {feature.StatusForAudit}, " +
$"databases: [{string.Join(", ", feature.AuthorizedDatabases.Keys)}]");
var conLifetime = context.Features.Get <IConnectionLifetimeFeature>();
if (conLifetime != null)
{
var msg = $"Connection {context.Connection.RemoteIpAddress}:{context.Connection.RemotePort} closed. Was used with: " +
$"with certificate '{feature.Certificate?.Subject} ({feature.Certificate?.Thumbprint})', status: {feature.StatusForAudit}, " +
$"databases: [{string.Join(", ", feature.AuthorizedDatabases.Keys)}]";
CancellationTokenRegistration cancellationTokenRegistration = default;
cancellationTokenRegistration = conLifetime.ConnectionClosed.Register(() =>
{
auditLog.Info(msg);
cancellationTokenRegistration.Dispose();
});
}
}
}
}
}
authenticationStatus = feature?.Status ?? RavenServer.AuthenticationStatus.None;
switch (route.AuthorizationStatus)
{
case AuthorizationStatus.UnauthenticatedClients:
var userWantsToAccessStudioMainPage = context.Request.Path == "/studio/index.html";
if (userWantsToAccessStudioMainPage)
{
switch (authenticationStatus)
{
case RavenServer.AuthenticationStatus.NoCertificateProvided:
case RavenServer.AuthenticationStatus.Expired:
case RavenServer.AuthenticationStatus.NotYetValid:
case RavenServer.AuthenticationStatus.None:
case RavenServer.AuthenticationStatus.UnfamiliarCertificate:
case RavenServer.AuthenticationStatus.UnfamiliarIssuer:
UnlikelyFailAuthorization(context, database?.Name, feature, route.AuthorizationStatus);
return(false);
}
}
return(true);
case AuthorizationStatus.ClusterAdmin:
case AuthorizationStatus.Operator:
case AuthorizationStatus.ValidUser:
case AuthorizationStatus.DatabaseAdmin:
case AuthorizationStatus.RestrictedAccess:
switch (authenticationStatus)
{
case RavenServer.AuthenticationStatus.NoCertificateProvided:
case RavenServer.AuthenticationStatus.Expired:
case RavenServer.AuthenticationStatus.NotYetValid:
case RavenServer.AuthenticationStatus.None:
UnlikelyFailAuthorization(context, database?.Name, feature, route.AuthorizationStatus);
return(false);
case RavenServer.AuthenticationStatus.UnfamiliarCertificate:
case RavenServer.AuthenticationStatus.UnfamiliarIssuer:
// we allow an access to the restricted endpoints with an unfamiliar certificate, since we will authorize it at the endpoint level
if (route.AuthorizationStatus == AuthorizationStatus.RestrictedAccess)
{
return(true);
}
goto case RavenServer.AuthenticationStatus.None;
case RavenServer.AuthenticationStatus.Allowed:
if (route.AuthorizationStatus == AuthorizationStatus.Operator || route.AuthorizationStatus == AuthorizationStatus.ClusterAdmin)
{
goto case RavenServer.AuthenticationStatus.None;
}
if (database == null)
{
return(true);
}
if (feature.CanAccess(database.Name, route.AuthorizationStatus == AuthorizationStatus.DatabaseAdmin))
{
return(true);
}
goto case RavenServer.AuthenticationStatus.None;
case RavenServer.AuthenticationStatus.Operator:
if (route.AuthorizationStatus == AuthorizationStatus.ClusterAdmin)
{
goto case RavenServer.AuthenticationStatus.None;
}
return(true);
case RavenServer.AuthenticationStatus.ClusterAdmin:
return(true);
default:
throw new ArgumentOutOfRangeException();
}
default:
ThrowUnknownAuthStatus(route);
return(false); // never hit
}
}
public InFlightTransactionalState GetInFlightTransactionalState(DocumentDatabase self, Func <string, Etag, RavenJObject, RavenJObject, TransactionInformation, PutResult> put, Func <string, Etag, TransactionInformation, bool> delete)
{
return(new DtcNotSupportedTransactionalState(FriendlyName, put, delete));
}
///<summary>
/// Create a new instance
///</summary>
public EmbeddedDatabaseCommands(DocumentDatabase database, DocumentConvention convention)
{
this.database = database;
this.convention = convention;
OperationsHeaders = new NameValueCollection();
}
protected void SetupDatabase(DocumentDatabase documentDatabase)
{
DocumentDatabase = documentDatabase;
}
public RavenServer(DocumentDatabase systemDatabase, InMemoryRavenConfiguration systemConfiguration)
{
this.systemDatabase = systemDatabase;
this.systemConfiguration = systemConfiguration;
}
public IndexActions(DocumentDatabase database, SizeLimitedConcurrentDictionary <string, TouchedDocumentInfo> recentTouches, IUuidGenerator uuidGenerator, ILog log)
: base(database, recentTouches, uuidGenerator, log)
{
}
public OutgoingReplicationHandler.UpdateSiblingCurrentEtag ToCommand(DocumentsOperationContext context, DocumentDatabase database)
{
var command = new OutgoingReplicationHandler.UpdateSiblingCurrentEtag(ReplicationBatchReply, new AsyncManualResetEvent());
command.Init();
return(command);
}
public void Initialize([CallerMemberName] string caller = null)
{
_documentDatabase = CreateDocumentDatabase(caller: caller);
_sut = new DynamicQueryToIndexMatcher(_documentDatabase.IndexStore);
}
protected async Task SmugglerRestore(DocumentDatabase database, List <string> filesToRestore, DocumentsOperationContext context,
DatabaseRecord databaseRecord, Action <IOperationProgress> onProgress, RestoreResult result)
{
Debug.Assert(onProgress != null);
// the files are already ordered by name
// take only the files that are relevant for smuggler restore
if (filesToRestore.Count == 0)
{
return;
}
// we do have at least one smuggler backup, we'll take the indexes from the last file
databaseRecord.AutoIndexes = new Dictionary <string, AutoIndexDefinition>();
databaseRecord.Indexes = new Dictionary <string, IndexDefinition>();
// restore the smuggler backup
var options = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
SkipRevisionCreation = true
};
options.OperateOnTypes |= DatabaseItemType.LegacyDocumentDeletions;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachments;
options.OperateOnTypes |= DatabaseItemType.LegacyAttachmentDeletions;
#pragma warning disable 618
options.OperateOnTypes |= DatabaseItemType.Counters;
#pragma warning restore 618
var oldOperateOnTypes = Raven.Client.Documents.Smuggler.DatabaseSmuggler.ConfigureOptionsForIncrementalImport(options);
var destination = new DatabaseDestination(database);
for (var i = 0; i < filesToRestore.Count - 1; i++)
{
result.AddInfo($"Restoring file {(i + 1):#,#;;0}/{filesToRestore.Count:#,#;;0}");
onProgress.Invoke(result.Progress);
var filePath = GetBackupPath(filesToRestore[i]);
await ImportSingleBackupFile(database, onProgress, result, filePath, context, destination, options, isLastFile : false,
onDatabaseRecordAction : smugglerDatabaseRecord =>
{
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
options.OperateOnTypes = oldOperateOnTypes;
var lastFilePath = GetBackupPath(filesToRestore.Last());
result.AddInfo($"Restoring file {filesToRestore.Count:#,#;;0}/{filesToRestore.Count:#,#;;0}");
onProgress.Invoke(result.Progress);
await ImportSingleBackupFile(database, onProgress, result, lastFilePath, context, destination, options, isLastFile : true,
onIndexAction : indexAndType =>
{
if (this.RestoreFromConfiguration.SkipIndexes)
{
return;
}
switch (indexAndType.Type)
{
case IndexType.AutoMap:
case IndexType.AutoMapReduce:
var autoIndexDefinition = (AutoIndexDefinitionBaseServerSide)indexAndType.IndexDefinition;
databaseRecord.AutoIndexes[autoIndexDefinition.Name] =
PutAutoIndexCommand.GetAutoIndexDefinition(autoIndexDefinition, indexAndType.Type);
break;
case IndexType.Map:
case IndexType.MapReduce:
case IndexType.JavaScriptMap:
case IndexType.JavaScriptMapReduce:
var indexDefinition = (IndexDefinition)indexAndType.IndexDefinition;
databaseRecord.Indexes[indexDefinition.Name] = indexDefinition;
break;
case IndexType.None:
case IndexType.Faulty:
break;
default:
throw new ArgumentOutOfRangeException();
}
},
onDatabaseRecordAction : smugglerDatabaseRecord =>
{
databaseRecord.ConflictSolverConfig = smugglerDatabaseRecord.ConflictSolverConfig;
foreach (var setting in smugglerDatabaseRecord.Settings)
{
databaseRecord.Settings[setting.Key] = setting.Value;
}
databaseRecord.SqlEtls = smugglerDatabaseRecord.SqlEtls;
databaseRecord.RavenEtls = smugglerDatabaseRecord.RavenEtls;
databaseRecord.PeriodicBackups = smugglerDatabaseRecord.PeriodicBackups;
databaseRecord.ExternalReplications = smugglerDatabaseRecord.ExternalReplications;
databaseRecord.Sorters = smugglerDatabaseRecord.Sorters;
databaseRecord.Analyzers = smugglerDatabaseRecord.Analyzers;
databaseRecord.SinkPullReplications = smugglerDatabaseRecord.SinkPullReplications;
databaseRecord.HubPullReplications = smugglerDatabaseRecord.HubPullReplications;
databaseRecord.Revisions = smugglerDatabaseRecord.Revisions;
databaseRecord.Expiration = smugglerDatabaseRecord.Expiration;
databaseRecord.RavenConnectionStrings = smugglerDatabaseRecord.RavenConnectionStrings;
databaseRecord.SqlConnectionStrings = smugglerDatabaseRecord.SqlConnectionStrings;
databaseRecord.Client = smugglerDatabaseRecord.Client;
databaseRecord.TimeSeries = smugglerDatabaseRecord.TimeSeries;
databaseRecord.DocumentsCompression = smugglerDatabaseRecord.DocumentsCompression;
databaseRecord.LockMode = smugglerDatabaseRecord.LockMode;
databaseRecord.OlapConnectionStrings = smugglerDatabaseRecord.OlapConnectionStrings;
databaseRecord.OlapEtls = smugglerDatabaseRecord.OlapEtls;
// need to enable revisions before import
database.DocumentsStorage.RevisionsStorage.InitializeFromDatabaseRecord(smugglerDatabaseRecord);
});
}
protected AbstractQueryRunner(DocumentDatabase database)
{
Database = database;
}
public static bool IsAdministrator(this IPrincipal principal, DocumentDatabase database)
{
return(IsAdministrator(principal, database.Name));
}
public DatabaseLiveIoStatsCollector(DocumentDatabase database) : base(database.IoChanges, database.GetAllStoragesEnvironment().ToList(), database.GetAllPerformanceMetrics(), database.DocumentsStorage.ContextPool, database.DatabaseShutdown)
{
}
public static MapReduceIndex CreateNew <TStaticIndex>(IndexDefinition definition, DocumentDatabase documentDatabase, bool isIndexReset = false)
where TStaticIndex : MapReduceIndex
{
TStaticIndex instance;
if (typeof(TStaticIndex) == typeof(MapReduceIndex))
{
instance = (TStaticIndex)CreateIndexInstance <MapReduceIndex>(definition, documentDatabase.Configuration, IndexDefinitionBaseServerSide.IndexVersion.CurrentVersion, (staticMapIndexDefinition, staticIndex) => new MapReduceIndex(staticMapIndexDefinition, staticIndex));
}
else if (typeof(TStaticIndex) == typeof(MapReduceTimeSeriesIndex))
{
instance = (TStaticIndex)(MapReduceIndex)CreateIndexInstance <MapReduceTimeSeriesIndex>(definition, documentDatabase.Configuration, IndexDefinitionBaseServerSide.IndexVersion.CurrentVersion, (staticMapIndexDefinition, staticIndex) => new MapReduceTimeSeriesIndex(staticMapIndexDefinition, staticIndex));
}
else if (typeof(TStaticIndex) == typeof(MapReduceCountersIndex))
{
instance = (TStaticIndex)(MapReduceIndex)CreateIndexInstance <MapReduceCountersIndex>(definition, documentDatabase.Configuration, IndexDefinitionBaseServerSide.IndexVersion.CurrentVersion, (staticMapIndexDefinition, staticIndex) => new MapReduceCountersIndex(staticMapIndexDefinition, staticIndex));
}
else
{
throw new NotSupportedException($"Not supported index type {typeof(TStaticIndex).Name}");
}
ValidateReduceResultsCollectionName(definition, instance._compiled, documentDatabase,
checkIfCollectionEmpty: isIndexReset == false);
instance.Initialize(documentDatabase,
new SingleIndexConfiguration(definition.Configuration, documentDatabase.Configuration),
documentDatabase.Configuration.PerformanceHints);
return(instance);
}
public static void ValidateReduceResultsCollectionName(IndexDefinition definition, AbstractStaticIndexBase index, DocumentDatabase database, bool checkIfCollectionEmpty)
{
var outputReduceToCollection = definition.OutputReduceToCollection;
if (string.IsNullOrWhiteSpace(outputReduceToCollection))
{
return;
}
if (outputReduceToCollection.Equals(definition.PatternReferencesCollectionName, StringComparison.OrdinalIgnoreCase))
{
throw new IndexInvalidException($"Collection defined in {nameof(definition.PatternReferencesCollectionName)} must not be the same as in {nameof(definition.OutputReduceToCollection)}. Collection name: '{outputReduceToCollection}'");
}
var collections = index.Maps.Keys.ToHashSet(StringComparer.OrdinalIgnoreCase);
if (collections.Contains(Constants.Documents.Collections.AllDocumentsCollection))
{
throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
$"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
$"as this index is mapping all documents " +
$"and this will result in an infinite loop.");
}
foreach (var referencedCollection in index.ReferencedCollections)
{
foreach (var collectionName in referencedCollection.Value)
{
collections.Add(collectionName.Name);
}
}
if (collections.Contains(outputReduceToCollection))
{
throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
$"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
$"as this index is mapping or referencing the '{outputReduceToCollection}' collection " +
$"and this will result in an infinite loop.");
}
var indexes = database.IndexStore.GetIndexes()
.Where(x => x.Type.IsStatic() && x.Type.IsMapReduce())
.Cast <MapReduceIndex>()
.Where(mapReduceIndex =>
{
// we have handling for side by side indexing with OutputReduceToCollection so we're checking only other indexes
string existingIndexName = mapReduceIndex.Name.Replace(Constants.Documents.Indexing.SideBySideIndexNamePrefix, string.Empty,
StringComparison.OrdinalIgnoreCase);
string newIndexName = definition.Name.Replace(Constants.Documents.Indexing.SideBySideIndexNamePrefix, string.Empty,
StringComparison.OrdinalIgnoreCase);
return(string.IsNullOrWhiteSpace(mapReduceIndex.Definition.OutputReduceToCollection) == false && string.Equals(existingIndexName, newIndexName, StringComparison.OrdinalIgnoreCase) == false);
})
.ToList();
foreach (var otherIndex in indexes)
{
if (otherIndex.Definition.OutputReduceToCollection.Equals(outputReduceToCollection, StringComparison.OrdinalIgnoreCase))
{
throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
$"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
$"as there is another index named '{otherIndex.Name}' " +
$"which also output reduce results to documents in the same '{outputReduceToCollection}' collection. " +
$"{nameof(IndexDefinition.OutputReduceToCollection)} must by set to unique value for each index or be null.");
}
var otherIndexCollections = new HashSet <string>(otherIndex.Collections);
foreach (var referencedCollection in otherIndex.GetReferencedCollections())
{
foreach (var collectionName in referencedCollection.Value)
{
otherIndexCollections.Add(collectionName.Name);
}
}
if (otherIndexCollections.Contains(outputReduceToCollection) &&
CheckIfThereIsAnIndexWhichWillOutputReduceDocumentsWhichWillBeUsedAsMapOnTheSpecifiedIndex(otherIndex, collections, indexes, out string description))
{
description += Environment.NewLine + $"--> {definition.Name}: {string.Join(",", collections)} => *{outputReduceToCollection}*";
throw new IndexInvalidException($"It is forbidden to create the '{definition.Name}' index " +
$"which would output reduce results to documents in the '{outputReduceToCollection}' collection, " +
$"as '{outputReduceToCollection}' collection is consumed by other index in a way that would " +
$"lead to an infinite loop." +
Environment.NewLine + description);
}
}
var existingIndexOrSideBySide = database.IndexStore.GetIndexes()
.Where(x => x.Type.IsStatic() && x.Type.IsMapReduce())
.Cast <MapReduceIndex>()
.FirstOrDefault(x =>
{
var name = definition.Name.Replace(Constants.Documents.Indexing.SideBySideIndexNamePrefix, string.Empty,
StringComparison.OrdinalIgnoreCase);
return(x.Name.Equals(name, StringComparison.OrdinalIgnoreCase) &&
x.Definition.ReduceOutputIndex != null); // legacy index definitions don't have this field - side by side indexing isn't supported then
});
if (existingIndexOrSideBySide != null)
{
if (definition.OutputReduceToCollection.Equals(existingIndexOrSideBySide.Definition.OutputReduceToCollection, StringComparison.OrdinalIgnoreCase)) // we have handling for side by side indexing with OutputReduceToCollection
{
checkIfCollectionEmpty = false;
}
}
if (checkIfCollectionEmpty)
{
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
using (context.OpenReadTransaction())
{
var stats = database.DocumentsStorage.GetCollection(outputReduceToCollection, context);
if (stats.Count > 0)
{
throw new IndexInvalidException(
$"Index '{definition.Name}' is defined to output the Reduce results to documents in Collection '{outputReduceToCollection}'. " +
$"This collection currently has {stats.Count} document{(stats.Count == 1 ? ' ' : 's')}. " +
$"All documents in Collection '{stats.Name}' must be deleted first.");
}
}
}
}
public CreateUpdateDeleteDocuments()
{
store = NewDocumentStore();
db = store.DocumentDatabase;
}
public long StartMigratingSingleDatabase(DatabaseMigrationSettings databaseMigrationSettings, DocumentDatabase database)
{
var operationId = database.Operations.GetNextOperationId();
var cancelToken = new OperationCancelToken(database.DatabaseShutdown);
var result = new SmugglerResult();
var databaseName = databaseMigrationSettings.DatabaseName;
database.Operations.AddOperation(null,
$"Database name: '{databaseName}' from url: {_serverUrl}",
Operations.OperationType.DatabaseMigrationRavenDb,
taskFactory: onProgress => Task.Run(async() =>
{
onProgress?.Invoke(result.Progress);
var message = $"Importing from RavenDB {EnumHelper.GetDescription(_buildMajorVersion)}";
result.AddInfo(message);
using (cancelToken)
{
try
{
var migrationStateKey = $"{MigrationStateKeyBase}" +
$"{EnumHelper.GetDescription(_buildMajorVersion)}/" +
$"{databaseName}/" +
$"{_serverUrl}";
var options = new MigratorOptions
{
MigrationStateKey = migrationStateKey,
ServerUrl = _serverUrl,
DatabaseName = databaseName,
HttpClient = _httpClient,
ApiKey = _apiKey,
EnableBasicAuthenticationOverUnsecuredHttp = _enableBasicAuthenticationOverUnsecuredHttp,
SkipServerCertificateValidation = _skipServerCertificateValidation,
OperateOnTypes = databaseMigrationSettings.OperateOnTypes,
RemoveAnalyzers = databaseMigrationSettings.RemoveAnalyzers,
ImportRavenFs = databaseMigrationSettings.ImportRavenFs,
Result = result,
OnProgress = onProgress,
Database = database,
CancelToken = cancelToken
};
AbstractMigrator migrator;
switch (_buildMajorVersion)
{
case MajorVersion.V2:
migrator = new Migrator_V2(database, options);
break;
case MajorVersion.V30:
case MajorVersion.V35:
migrator = new Migrator_V3(database, options, _buildMajorVersion, _buildVersion);
break;
case MajorVersion.V4:
migrator = new Importer(options);
break;
default:
throw new ArgumentOutOfRangeException(nameof(_buildMajorVersion), _buildMajorVersion, null);
}
await migrator.Execute();
}
catch (Exception e)
{
result.AddError($"Error occurred during database migration named: {databaseName}." +
$"Exception: {e.Message}");
throw;
}
}
return((IOperationResult)result);
}, cancelToken.Token), id: operationId, token: cancelToken);
return(operationId);
}
protected virtual void ConfigureDatabase(DocumentDatabase database)
{
}
private static BackupInfo GetBackupInfo(DocumentDatabase db)
{
var periodicBackupRunner = db?.PeriodicBackupRunner;
return(periodicBackupRunner?.GetBackupInfo());
}
public GeneralStorage()
{
store = NewDocumentStore();
db = store.SystemDatabase;
}
private static TimeSpan GetUptime(DocumentDatabase db)
{
return(SystemTime.UtcNow - db.StartTime);
}
/// <summary>
/// Restore CompareExchange, Identities and Subscriptions from smuggler file when restoring snapshot.
/// </summary>
/// <param name="onProgress"></param>
/// <param name="database"></param>
/// <param name="smugglerFile"></param>
/// <param name="context"></param>
protected async Task RestoreFromSmugglerFile(Action <IOperationProgress> onProgress, DocumentDatabase database, string smugglerFile, DocumentsOperationContext context)
{
var destination = new DatabaseDestination(database);
var smugglerOptions = new DatabaseSmugglerOptionsServerSide
{
AuthorizationStatus = AuthorizationStatus.DatabaseAdmin,
OperateOnTypes = DatabaseItemType.CompareExchange | DatabaseItemType.Identities | DatabaseItemType.Subscriptions,
SkipRevisionCreation = true
};
var lastPath = GetSmugglerBackupPath(smugglerFile);
using (var zip = await GetZipArchiveForSnapshot(lastPath))
{
foreach (var entry in zip.Entries)
{
if (entry.Name == RestoreSettings.SmugglerValuesFileName)
{
await using (var input = entry.Open())
await using (var inputStream = GetSnapshotInputStream(input, database.Name))
await using (var uncompressed = new GZipStream(inputStream, CompressionMode.Decompress))
{
var source = new StreamSource(uncompressed, context, database);
var smuggler = new Smuggler.Documents.DatabaseSmuggler(database, source, destination,
database.Time, smugglerOptions, onProgress: onProgress, token: _operationCancelToken.Token);
await smuggler.ExecuteAsync(ensureStepsProcessed : true, isLastFile : true);
}
break;
}
}
}
}
public static bool IsSystemDatabase(this DocumentDatabase database)
{
return(database.Name == null || database.Name == Constants.SystemDatabase);
}
public void Execute(DocumentDatabase database)
{
Database = database;
Initialize();
Task.Factory.StartNew(BackgroundTask, TaskCreationOptions.LongRunning);
}
public DeleteIndexes()
{
store = NewDocumentStore();
db = store.SystemDatabase;
}
public virtual void Initialize(DocumentDatabase database)
{
}
public static AutoMapReduceIndex CreateNew(AutoMapReduceIndexDefinition definition, DocumentDatabase documentDatabase)
{
var instance = new AutoMapReduceIndex(definition);
instance.Initialize(documentDatabase, documentDatabase.Configuration.Indexing, documentDatabase.Configuration.PerformanceHints);
return(instance);
}
public SnapshotShoppingCartBatcher(string indexName, DocumentDatabase database)
{
this.indexName = indexName;
this.database = database;
}
private bool TryAuthorize(RouteInformation route, HttpContext context, DocumentDatabase database)
{
var feature = context.Features.Get <IHttpAuthenticationFeature>() as RavenServer.AuthenticateConnection;
switch (route.AuthorizationStatus)
{
case AuthorizationStatus.UnauthenticatedClients:
var userWantsToAccessStudioMainPage = context.Request.Path == "/studio/index.html";
if (userWantsToAccessStudioMainPage)
{
switch (feature?.Status)
{
case null:
case RavenServer.AuthenticationStatus.NoCertificateProvided:
case RavenServer.AuthenticationStatus.Expired:
case RavenServer.AuthenticationStatus.NotYetValid:
case RavenServer.AuthenticationStatus.None:
case RavenServer.AuthenticationStatus.UnfamiliarCertificate:
UnlikelyFailAuthorization(context, database?.Name, feature);
return(false);
}
}
return(true);
case AuthorizationStatus.ClusterAdmin:
case AuthorizationStatus.Operator:
case AuthorizationStatus.ValidUser:
case AuthorizationStatus.DatabaseAdmin:
switch (feature?.Status)
{
case null:
case RavenServer.AuthenticationStatus.NoCertificateProvided:
case RavenServer.AuthenticationStatus.Expired:
case RavenServer.AuthenticationStatus.NotYetValid:
case RavenServer.AuthenticationStatus.None:
case RavenServer.AuthenticationStatus.UnfamiliarCertificate:
UnlikelyFailAuthorization(context, database?.Name, feature);
return(false);
case RavenServer.AuthenticationStatus.Allowed:
if (route.AuthorizationStatus == AuthorizationStatus.Operator || route.AuthorizationStatus == AuthorizationStatus.ClusterAdmin)
{
goto case RavenServer.AuthenticationStatus.None;
}
if (database == null)
{
return(true);
}
if (feature.CanAccess(database.Name, route.AuthorizationStatus == AuthorizationStatus.DatabaseAdmin))
{
return(true);
}
goto case RavenServer.AuthenticationStatus.None;
case RavenServer.AuthenticationStatus.Operator:
if (route.AuthorizationStatus == AuthorizationStatus.ClusterAdmin)
{
goto case RavenServer.AuthenticationStatus.None;
}
return(true);
case RavenServer.AuthenticationStatus.ClusterAdmin:
return(true);
default:
throw new ArgumentOutOfRangeException();
}
default:
ThrowUnknownAuthStatus(route);
return(false); // never hit
}
}
