private static void SetOfflineDatabaseInfo(
ServerStore serverStore,
string databaseName,
DatabasesInfo existingDatabasesInfo,
DrivesUsage existingDrivesUsage,
bool disabled)
{
var databaseRecord = serverStore.LoadDatabaseRecord(databaseName, out var _);
if (databaseRecord == null)
{
// database doesn't exist
return;
}
var databaseInfoItem = new DatabaseInfoItem
{
Database = databaseName,
Online = false,
Disabled = disabled,
Irrelevant = databaseRecord.Topology?.AllNodes.Contains(serverStore.NodeTag) ?? true
};
DatabaseInfo databaseInfo = null;
if (serverStore.DatabaseInfoCache.TryGet(databaseName,
databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)))
{
Debug.Assert(databaseInfo != null);
databaseInfoItem.DocumentsCount = databaseInfo.DocumentsCount ?? 0;
databaseInfoItem.IndexesCount = databaseInfo.IndexesCount ?? databaseRecord.Indexes.Count;
databaseInfoItem.ReplicationFactor = databaseRecord.Topology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;
if (databaseInfo.MountPointsUsage != null)
{
var drives = DriveInfo.GetDrives();
foreach (var mountPointUsage in databaseInfo.MountPointsUsage)
{
var diskSpaceResult = DiskSpaceChecker.GetFreeDiskSpace(mountPointUsage.DiskSpaceResult.DriveName, drives);
if (diskSpaceResult != null)
{
// update the latest drive info
mountPointUsage.DiskSpaceResult = new Client.ServerWide.Operations.DiskSpaceResult
{
DriveName = diskSpaceResult.DriveName,
VolumeLabel = diskSpaceResult.VolumeLabel,
TotalFreeSpaceInBytes = diskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
TotalSizeInBytes = diskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
};
}
UpdateMountPoint(mountPointUsage, databaseName, existingDrivesUsage);
}
}
}
existingDatabasesInfo.Items.Add(databaseInfoItem);
}
public static DynamicJsonValue ToJson(ServerStore serverStore, TransactionOperationContext context)
{
var djv = new DynamicJsonValue
{
[$"@{nameof(General)}"] = General.ToJson()
};
var mapping = SnmpWatcher.GetMapping(serverStore, context);
foreach (var kvp in mapping)
{
var record = serverStore.LoadDatabaseRecord(kvp.Key, out _);
if (record == null)
{
continue;
}
var array = new DynamicJsonArray();
foreach (var field in typeof(Databases).GetFields())
{
var fieldValue = GetFieldValue(field);
var oid = string.Format(fieldValue.Oid, kvp.Value);
array.Add(CreateJsonItem(Root + oid, fieldValue.Description));
}
djv[kvp.Key] = new DynamicJsonValue
{
[$"@{nameof(General)}"] = array,
[nameof(Indexes)] = Indexes.ToJson(serverStore, context, record, kvp.Value)
};
}
return(djv);
}
public static SqlEtlSimulationResult SimulateSqlEtl(SimulateSqlEtl simulateSqlEtl, DocumentDatabase database, ServerStore serverStore, DocumentsOperationContext context)
{
var document = database.DocumentsStorage.Get(context, simulateSqlEtl.DocumentId);
if (document == null)
{
throw new InvalidOperationException($"Document {simulateSqlEtl.DocumentId} does not exist");
}
if (serverStore.LoadDatabaseRecord(database.Name, out _).SqlConnectionStrings.TryGetValue(simulateSqlEtl.Configuration.ConnectionStringName, out var connectionString) == false)
{
throw new InvalidOperationException($"Connection string named {simulateSqlEtl.Configuration.ConnectionStringName} was not found in the database record");
}
simulateSqlEtl.Configuration.Initialize(connectionString);
if (simulateSqlEtl.Configuration.Validate(out List <string> errors) == false)
{
throw new InvalidOperationException($"Invalid ETL configuration for '{simulateSqlEtl.Configuration.Name}'. " +
$"Reason{(errors.Count > 1 ? "s" : string.Empty)}: {string.Join(";", errors)}.");
}
// TODO arek - those constraints can be changed later on
if (simulateSqlEtl.Configuration.Transforms.Count != 1)
{
throw new InvalidOperationException($"Invalid number of transformations. You have provided {simulateSqlEtl.Configuration.Transforms.Count} " +
"while SQL ETL simulation expects to get exactly 1 transformation script");
}
if (simulateSqlEtl.Configuration.Transforms[0].Collections.Count != 1)
{
throw new InvalidOperationException($"Invalid number of collections specified in the transformation script. You have provided {simulateSqlEtl.Configuration.Transforms[0].Collections.Count} " +
"while SQL ETL simulation is supposed to work with exactly 1 collection");
}
using (var etl = new SqlEtl(simulateSqlEtl.Configuration.Transforms[0], simulateSqlEtl.Configuration, database, null))
{
etl.EnsureThreadAllocationStats();
var collection = simulateSqlEtl.Configuration.Transforms[0].Collections[0];
var transformed = etl.Transform(new[] { new ToSqlItem(document, collection) }, context, new EtlStatsScope(new EtlRunStats()), new EtlProcessState());
return(etl.Simulate(simulateSqlEtl, context, transformed));
}
}
public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
{
var databaseName = _restoreConfiguration.DatabaseName;
var result = new RestoreResult
{
DataDirectory = _restoreConfiguration.DataDirectory
};
try
{
if (onProgress == null)
{
onProgress = _ => { }
}
;
Stopwatch sw = null;
RestoreSettings restoreSettings = null;
var firstFile = _filesToRestore[0];
var lastFile = _filesToRestore.Last();
var extension = Path.GetExtension(firstFile);
var snapshotRestore = false;
if (extension == Constants.Documents.PeriodicBackup.SnapshotExtension)
{
onProgress.Invoke(result.Progress);
snapshotRestore = true;
sw = Stopwatch.StartNew();
// restore the snapshot
restoreSettings = SnapshotRestore(firstFile,
_restoreConfiguration.DataDirectory,
onProgress,
result);
// removing the snapshot from the list of files
_filesToRestore.RemoveAt(0);
}
else
{
result.SnapshotRestore.Skipped = true;
result.SnapshotRestore.Processed = true;
onProgress.Invoke(result.Progress);
}
if (restoreSettings == null)
{
restoreSettings = new RestoreSettings
{
DatabaseRecord = new DatabaseRecord(databaseName)
{
// we only have a smuggler restore
// use the encryption key to encrypt the database
Encrypted = _hasEncryptionKey
}
};
DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
}
var databaseRecord = restoreSettings.DatabaseRecord;
if (databaseRecord.Settings == null)
{
databaseRecord.Settings = new Dictionary <string, string>();
}
var runInMemoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.RunInMemory);
databaseRecord.Settings.Remove(runInMemoryConfigurationKey);
if (_serverStore.Configuration.Core.RunInMemory)
{
databaseRecord.Settings[runInMemoryConfigurationKey] = "false";
}
var dataDirectoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.DataDirectory);
databaseRecord.Settings.Remove(dataDirectoryConfigurationKey); // removing because we want to restore to given location, not to serialized in backup one
if (_restoringToDefaultDataDirectory == false)
{
databaseRecord.Settings[dataDirectoryConfigurationKey] = _restoreConfiguration.DataDirectory;
}
if (_hasEncryptionKey)
{
// save the encryption key so we'll be able to access the database
_serverStore.PutSecretKey(_restoreConfiguration.EncryptionKey,
databaseName, overwrite: false);
}
var addToInitLog = new Action <string>(txt => // init log is not save in mem during RestoreBackup
{
var msg = $"[RestoreBackup] {DateTime.UtcNow} :: Database '{databaseName}' : {txt}";
if (Logger.IsInfoEnabled)
{
Logger.Info(msg);
}
});
using (var database = new DocumentDatabase(databaseName,
new RavenConfiguration(databaseName, ResourceType.Database)
{
Core =
{
DataDirectory = new PathSetting(_restoreConfiguration.DataDirectory),
RunInMemory = false
}
}, _serverStore, addToInitLog))
{
// smuggler needs an existing document database to operate
var options = InitializeOptions.SkipLoadingDatabaseRecord;
if (snapshotRestore)
{
options |= InitializeOptions.GenerateNewDatabaseId;
}
database.Initialize(options);
if (snapshotRestore)
{
result.SnapshotRestore.Processed = true;
var summary = database.GetDatabaseSummary();
result.Documents.ReadCount += summary.DocumentsCount;
result.Documents.Attachments.ReadCount += summary.AttachmentsCount;
result.Counters.ReadCount += summary.CountersCount;
result.RevisionDocuments.ReadCount += summary.RevisionsCount;
result.Conflicts.ReadCount += summary.ConflictsCount;
result.Indexes.ReadCount += databaseRecord.GetIndexesCount();
result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} " +
$"files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms");
onProgress.Invoke(result.Progress);
}
using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
{
SmugglerRestore(_restoreConfiguration.BackupLocation, database, context, databaseRecord, onProgress, result);
result.DatabaseRecord.Processed = true;
result.Documents.Processed = true;
result.RevisionDocuments.Processed = true;
result.Conflicts.Processed = true;
result.Indexes.Processed = true;
result.Counters.Processed = true;
onProgress.Invoke(result.Progress);
databaseRecord.Topology = new DatabaseTopology();
// restoring to the current node only
databaseRecord.Topology.Members.Add(_nodeTag);
databaseRecord.Disabled = true; // we are currently restoring, shouldn't try to access it
_serverStore.EnsureNotPassive();
DisableOngoingTasksIfNeeded(databaseRecord);
var(index, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, restoreSettings.DatabaseValues, isRestore : true);
await _serverStore.Cluster.WaitForIndexNotification(index);
// restore identities & cmpXchg values
RestoreFromLastFile(onProgress, database, lastFile, context, result);
}
}
// after the db for restore is done, we can safely set the db status to active
databaseRecord = _serverStore.LoadDatabaseRecord(databaseName, out _);
databaseRecord.Disabled = false;
var(updateIndex, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null);
await _serverStore.Cluster.WaitForIndexNotification(updateIndex);
return(result);
}
catch (Exception e)
{
if (Logger.IsOperationsEnabled)
{
Logger.Operations("Failed to restore database", e);
}
var alert = AlertRaised.Create(
_restoreConfiguration.DatabaseName,
"Failed to restore database",
$"Could not restore database named {_restoreConfiguration.DatabaseName}",
AlertType.RestoreError,
NotificationSeverity.Error,
details: new ExceptionDetails(e));
_serverStore.NotificationCenter.Add(alert);
if (_serverStore.LoadDatabaseRecord(_restoreConfiguration.DatabaseName, out var _) == null)
{
// delete any files that we already created during the restore
IOExtensions.DeleteDirectory(_restoreConfiguration.DataDirectory);
}
else
{
var deleteResult = await _serverStore.DeleteDatabaseAsync(_restoreConfiguration.DatabaseName, true, new[] { _serverStore.NodeTag });
await _serverStore.Cluster.WaitForIndexNotification(deleteResult.Index);
}
result.AddError($"Error occurred during restore of database {databaseName}. Exception: {e.Message}");
onProgress.Invoke(result.Progress);
throw;
}
finally
{
_operationCancelToken.Dispose();
}
}
public DatabaseRecord LoadDatabaseRecord()
{
return(_server.LoadDatabaseRecord(Database.Name, out _));
}
public static IEnumerable <AbstractDashboardNotification> FetchDatabasesInfo(ServerStore serverStore, CancellationTokenSource cts)
{
var databasesInfo = new DatabasesInfo();
var indexingSpeed = new IndexingSpeed();
var trafficWatch = new TrafficWatch();
var drivesUsage = new DrivesUsage();
using (serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionContext))
using (transactionContext.OpenReadTransaction())
{
foreach (var databaseTuple in serverStore.Cluster.ItemsStartingWith(transactionContext, Constants.Documents.Prefix, 0, int.MaxValue))
{
var databaseName = databaseTuple.ItemName.Substring(3);
if (cts.IsCancellationRequested)
{
yield break;
}
if (serverStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out var databaseTask) == false)
{
// database does not exist in this server or disabled
continue;
}
var databaseOnline = IsDatabaseOnline(databaseTask, out var database);
if (databaseOnline == false)
{
var databaseRecord = serverStore.LoadDatabaseRecord(databaseName, out var _);
if (databaseRecord == null)
{
// database doesn't exist
continue;
}
var databaseInfoItem = new DatabaseInfoItem
{
Database = databaseName,
Online = false
};
DatabaseInfo databaseInfo = null;
if (serverStore.DatabaseInfoCache.TryGet(databaseName,
databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)))
{
Debug.Assert(databaseInfo != null);
databaseInfoItem.DocumentsCount = databaseInfo.DocumentsCount ?? 0;
databaseInfoItem.IndexesCount = databaseInfo.IndexesCount ?? databaseRecord.Indexes.Count;
databaseInfoItem.ReplicationFactor = databaseRecord.Topology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;
}
databasesInfo.Items.Add(databaseInfoItem);
continue;
}
var indexingSpeedItem = new IndexingSpeedItem
{
Database = database.Name,
IndexedPerSecond = database.Metrics.MapIndexes.IndexedPerSec.FiveSecondRate,
MappedPerSecond = database.Metrics.MapReduceIndexes.MappedPerSec.FiveSecondRate,
ReducedPerSecond = database.Metrics.MapReduceIndexes.ReducedPerSec.FiveSecondRate
};
indexingSpeed.Items.Add(indexingSpeedItem);
var replicationFactor = GetReplicationFactor(databaseTuple.Value);
var documentsStorage = database.DocumentsStorage;
var indexStorage = database.IndexStore;
using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
using (documentsContext.OpenReadTransaction())
{
var databaseInfoItem = new DatabaseInfoItem
{
Database = databaseName,
DocumentsCount = documentsStorage.GetNumberOfDocuments(documentsContext),
IndexesCount = database.IndexStore.Count,
AlertsCount = database.NotificationCenter.GetAlertCount(),
ReplicationFactor = replicationFactor,
ErroredIndexesCount = indexStorage.GetIndexes().Count(index => index.GetErrorCount() > 0),
Online = true
};
databasesInfo.Items.Add(databaseInfoItem);
}
var trafficWatchItem = new TrafficWatchItem
{
Database = databaseName,
RequestsPerSecond = (int)database.Metrics.Requests.RequestsPerSec.FiveSecondRate,
WritesPerSecond = (int)database.Metrics.Docs.PutsPerSec.FiveSecondRate,
WriteBytesPerSecond = database.Metrics.Docs.BytesPutsPerSec.FiveSecondRate
};
trafficWatch.Items.Add(trafficWatchItem);
foreach (var mountPointUsage in database.GetMountPointsUsage())
{
if (cts.IsCancellationRequested)
{
yield break;
}
var mountPoint = mountPointUsage.DiskSpaceResult.DriveName;
var usage = drivesUsage.Items.FirstOrDefault(x => x.MountPoint == mountPoint);
if (usage == null)
{
usage = new MountPointUsage
{
MountPoint = mountPoint,
VolumeLabel = mountPointUsage.DiskSpaceResult.VolumeLabel,
FreeSpace = mountPointUsage.DiskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
TotalCapacity = mountPointUsage.DiskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
};
drivesUsage.Items.Add(usage);
}
var existingDatabaseUsage = usage.Items.FirstOrDefault(x => x.Database == databaseName);
if (existingDatabaseUsage == null)
{
existingDatabaseUsage = new DatabaseDiskUsage
{
Database = databaseName
};
usage.Items.Add(existingDatabaseUsage);
}
existingDatabaseUsage.Size += mountPointUsage.UsedSpace;
}
}
}
yield return(databasesInfo);
yield return(indexingSpeed);
yield return(trafficWatch);
yield return(drivesUsage);
}
