private static void SetOfflineDatabaseInfo(
            ServerStore serverStore,
            string databaseName,
            DatabasesInfo existingDatabasesInfo,
            DrivesUsage existingDrivesUsage,
            bool disabled)
        {
            var databaseRecord = serverStore.LoadDatabaseRecord(databaseName, out var _);

            if (databaseRecord == null)
            {
                // database doesn't exist
                return;
            }

            var databaseInfoItem = new DatabaseInfoItem
            {
                Database   = databaseName,
                Online     = false,
                Disabled   = disabled,
                Irrelevant = databaseRecord.Topology?.AllNodes.Contains(serverStore.NodeTag) ?? true
            };

            DatabaseInfo databaseInfo = null;

            if (serverStore.DatabaseInfoCache.TryGet(databaseName,
                                                     databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)))
            {
                Debug.Assert(databaseInfo != null);

                databaseInfoItem.DocumentsCount      = databaseInfo.DocumentsCount ?? 0;
                databaseInfoItem.IndexesCount        = databaseInfo.IndexesCount ?? databaseRecord.Indexes.Count;
                databaseInfoItem.ReplicationFactor   = databaseRecord.Topology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
                databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;

                if (databaseInfo.MountPointsUsage != null)
                {
                    var drives = DriveInfo.GetDrives();
                    foreach (var mountPointUsage in databaseInfo.MountPointsUsage)
                    {
                        var diskSpaceResult = DiskSpaceChecker.GetFreeDiskSpace(mountPointUsage.DiskSpaceResult.DriveName, drives);
                        if (diskSpaceResult != null)
                        {
                            // update the latest drive info
                            mountPointUsage.DiskSpaceResult = new Client.ServerWide.Operations.DiskSpaceResult
                            {
                                DriveName             = diskSpaceResult.DriveName,
                                VolumeLabel           = diskSpaceResult.VolumeLabel,
                                TotalFreeSpaceInBytes = diskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
                                TotalSizeInBytes      = diskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
                            };
                        }

                        UpdateMountPoint(mountPointUsage, databaseName, existingDrivesUsage);
                    }
                }
            }

            existingDatabasesInfo.Items.Add(databaseInfoItem);
        }
Ejemplo n.º 2
0
            public static DynamicJsonValue ToJson(ServerStore serverStore, TransactionOperationContext context)
            {
                var djv = new DynamicJsonValue
                {
                    [$"@{nameof(General)}"] = General.ToJson()
                };

                var mapping = SnmpWatcher.GetMapping(serverStore, context);

                foreach (var kvp in mapping)
                {
                    var record = serverStore.LoadDatabaseRecord(kvp.Key, out _);
                    if (record == null)
                    {
                        continue;
                    }

                    var array = new DynamicJsonArray();
                    foreach (var field in typeof(Databases).GetFields())
                    {
                        var fieldValue = GetFieldValue(field);
                        var oid        = string.Format(fieldValue.Oid, kvp.Value);
                        array.Add(CreateJsonItem(Root + oid, fieldValue.Description));
                    }

                    djv[kvp.Key] = new DynamicJsonValue
                    {
                        [$"@{nameof(General)}"] = array,
                        [nameof(Indexes)]       = Indexes.ToJson(serverStore, context, record, kvp.Value)
                    };
                }

                return(djv);
            }
Ejemplo n.º 3
0
        public static SqlEtlSimulationResult SimulateSqlEtl(SimulateSqlEtl simulateSqlEtl, DocumentDatabase database, ServerStore serverStore, DocumentsOperationContext context)
        {
            var document = database.DocumentsStorage.Get(context, simulateSqlEtl.DocumentId);

            if (document == null)
            {
                throw new InvalidOperationException($"Document {simulateSqlEtl.DocumentId} does not exist");
            }

            if (serverStore.LoadDatabaseRecord(database.Name, out _).SqlConnectionStrings.TryGetValue(simulateSqlEtl.Configuration.ConnectionStringName, out var connectionString) == false)
            {
                throw new InvalidOperationException($"Connection string named {simulateSqlEtl.Configuration.ConnectionStringName} was not found in the database record");
            }

            simulateSqlEtl.Configuration.Initialize(connectionString);

            if (simulateSqlEtl.Configuration.Validate(out List <string> errors) == false)
            {
                throw new InvalidOperationException($"Invalid ETL configuration for '{simulateSqlEtl.Configuration.Name}'. " +
                                                    $"Reason{(errors.Count > 1 ? "s" : string.Empty)}: {string.Join(";", errors)}.");
            }

            // TODO arek - those constraints can be changed later on

            if (simulateSqlEtl.Configuration.Transforms.Count != 1)
            {
                throw new InvalidOperationException($"Invalid number of transformations. You have provided {simulateSqlEtl.Configuration.Transforms.Count} " +
                                                    "while SQL ETL simulation expects to get exactly 1 transformation script");
            }

            if (simulateSqlEtl.Configuration.Transforms[0].Collections.Count != 1)
            {
                throw new InvalidOperationException($"Invalid number of collections specified in the transformation script. You have provided {simulateSqlEtl.Configuration.Transforms[0].Collections.Count} " +
                                                    "while SQL ETL simulation is supposed to work with exactly 1 collection");
            }

            using (var etl = new SqlEtl(simulateSqlEtl.Configuration.Transforms[0], simulateSqlEtl.Configuration, database, null))
            {
                etl.EnsureThreadAllocationStats();

                var collection  = simulateSqlEtl.Configuration.Transforms[0].Collections[0];
                var transformed = etl.Transform(new[] { new ToSqlItem(document, collection) }, context, new EtlStatsScope(new EtlRunStats()), new EtlProcessState());

                return(etl.Simulate(simulateSqlEtl, context, transformed));
            }
        }
Ejemplo n.º 4
0
        public async Task <IOperationResult> Execute(Action <IOperationProgress> onProgress)
        {
            var databaseName = _restoreConfiguration.DatabaseName;
            var result       = new RestoreResult
            {
                DataDirectory = _restoreConfiguration.DataDirectory
            };

            try
            {
                if (onProgress == null)
                {
                    onProgress = _ => { }
                }
                ;

                Stopwatch       sw = null;
                RestoreSettings restoreSettings = null;
                var             firstFile       = _filesToRestore[0];
                var             lastFile        = _filesToRestore.Last();

                var extension       = Path.GetExtension(firstFile);
                var snapshotRestore = false;
                if (extension == Constants.Documents.PeriodicBackup.SnapshotExtension)
                {
                    onProgress.Invoke(result.Progress);

                    snapshotRestore = true;
                    sw = Stopwatch.StartNew();
                    // restore the snapshot
                    restoreSettings = SnapshotRestore(firstFile,
                                                      _restoreConfiguration.DataDirectory,
                                                      onProgress,
                                                      result);

                    // removing the snapshot from the list of files
                    _filesToRestore.RemoveAt(0);
                }
                else
                {
                    result.SnapshotRestore.Skipped   = true;
                    result.SnapshotRestore.Processed = true;

                    onProgress.Invoke(result.Progress);
                }

                if (restoreSettings == null)
                {
                    restoreSettings = new RestoreSettings
                    {
                        DatabaseRecord = new DatabaseRecord(databaseName)
                        {
                            // we only have a smuggler restore
                            // use the encryption key to encrypt the database
                            Encrypted = _hasEncryptionKey
                        }
                    };

                    DatabaseHelper.Validate(databaseName, restoreSettings.DatabaseRecord, _serverStore.Configuration);
                }

                var databaseRecord = restoreSettings.DatabaseRecord;
                if (databaseRecord.Settings == null)
                {
                    databaseRecord.Settings = new Dictionary <string, string>();
                }

                var runInMemoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.RunInMemory);
                databaseRecord.Settings.Remove(runInMemoryConfigurationKey);
                if (_serverStore.Configuration.Core.RunInMemory)
                {
                    databaseRecord.Settings[runInMemoryConfigurationKey] = "false";
                }

                var dataDirectoryConfigurationKey = RavenConfiguration.GetKey(x => x.Core.DataDirectory);
                databaseRecord.Settings.Remove(dataDirectoryConfigurationKey); // removing because we want to restore to given location, not to serialized in backup one
                if (_restoringToDefaultDataDirectory == false)
                {
                    databaseRecord.Settings[dataDirectoryConfigurationKey] = _restoreConfiguration.DataDirectory;
                }

                if (_hasEncryptionKey)
                {
                    // save the encryption key so we'll be able to access the database
                    _serverStore.PutSecretKey(_restoreConfiguration.EncryptionKey,
                                              databaseName, overwrite: false);
                }

                var addToInitLog = new Action <string>(txt => // init log is not save in mem during RestoreBackup
                {
                    var msg = $"[RestoreBackup] {DateTime.UtcNow} :: Database '{databaseName}' : {txt}";
                    if (Logger.IsInfoEnabled)
                    {
                        Logger.Info(msg);
                    }
                });

                using (var database = new DocumentDatabase(databaseName,
                                                           new RavenConfiguration(databaseName, ResourceType.Database)
                {
                    Core =
                    {
                        DataDirectory = new PathSetting(_restoreConfiguration.DataDirectory),
                        RunInMemory   = false
                    }
                }, _serverStore, addToInitLog))
                {
                    // smuggler needs an existing document database to operate
                    var options = InitializeOptions.SkipLoadingDatabaseRecord;
                    if (snapshotRestore)
                    {
                        options |= InitializeOptions.GenerateNewDatabaseId;
                    }

                    database.Initialize(options);

                    if (snapshotRestore)
                    {
                        result.SnapshotRestore.Processed = true;

                        var summary = database.GetDatabaseSummary();
                        result.Documents.ReadCount             += summary.DocumentsCount;
                        result.Documents.Attachments.ReadCount += summary.AttachmentsCount;
                        result.Counters.ReadCount          += summary.CountersCount;
                        result.RevisionDocuments.ReadCount += summary.RevisionsCount;
                        result.Conflicts.ReadCount         += summary.ConflictsCount;
                        result.Indexes.ReadCount           += databaseRecord.GetIndexesCount();
                        result.AddInfo($"Successfully restored {result.SnapshotRestore.ReadCount} " +
                                       $"files during snapshot restore, took: {sw.ElapsedMilliseconds:#,#;;0}ms");
                        onProgress.Invoke(result.Progress);
                    }
                    using (database.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext context))
                    {
                        SmugglerRestore(_restoreConfiguration.BackupLocation, database, context, databaseRecord, onProgress, result);

                        result.DatabaseRecord.Processed    = true;
                        result.Documents.Processed         = true;
                        result.RevisionDocuments.Processed = true;
                        result.Conflicts.Processed         = true;
                        result.Indexes.Processed           = true;
                        result.Counters.Processed          = true;
                        onProgress.Invoke(result.Progress);

                        databaseRecord.Topology = new DatabaseTopology();
                        // restoring to the current node only
                        databaseRecord.Topology.Members.Add(_nodeTag);
                        databaseRecord.Disabled = true; // we are currently restoring, shouldn't try to access it
                        _serverStore.EnsureNotPassive();

                        DisableOngoingTasksIfNeeded(databaseRecord);

                        var(index, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null, restoreSettings.DatabaseValues, isRestore : true);

                        await _serverStore.Cluster.WaitForIndexNotification(index);

                        // restore identities & cmpXchg values
                        RestoreFromLastFile(onProgress, database, lastFile, context, result);
                    }
                }

                // after the db for restore is done, we can safely set the db status to active
                databaseRecord          = _serverStore.LoadDatabaseRecord(databaseName, out _);
                databaseRecord.Disabled = false;

                var(updateIndex, _) = await _serverStore.WriteDatabaseRecordAsync(databaseName, databaseRecord, null);

                await _serverStore.Cluster.WaitForIndexNotification(updateIndex);

                return(result);
            }
            catch (Exception e)
            {
                if (Logger.IsOperationsEnabled)
                {
                    Logger.Operations("Failed to restore database", e);
                }

                var alert = AlertRaised.Create(
                    _restoreConfiguration.DatabaseName,
                    "Failed to restore database",
                    $"Could not restore database named {_restoreConfiguration.DatabaseName}",
                    AlertType.RestoreError,
                    NotificationSeverity.Error,
                    details: new ExceptionDetails(e));
                _serverStore.NotificationCenter.Add(alert);

                if (_serverStore.LoadDatabaseRecord(_restoreConfiguration.DatabaseName, out var _) == null)
                {
                    // delete any files that we already created during the restore
                    IOExtensions.DeleteDirectory(_restoreConfiguration.DataDirectory);
                }
                else
                {
                    var deleteResult = await _serverStore.DeleteDatabaseAsync(_restoreConfiguration.DatabaseName, true, new[] { _serverStore.NodeTag });

                    await _serverStore.Cluster.WaitForIndexNotification(deleteResult.Index);
                }

                result.AddError($"Error occurred during restore of database {databaseName}. Exception: {e.Message}");
                onProgress.Invoke(result.Progress);
                throw;
            }
            finally
            {
                _operationCancelToken.Dispose();
            }
        }
Ejemplo n.º 5
0
 public DatabaseRecord LoadDatabaseRecord()
 {
     return(_server.LoadDatabaseRecord(Database.Name, out _));
 }
Ejemplo n.º 6
0
        public static IEnumerable <AbstractDashboardNotification> FetchDatabasesInfo(ServerStore serverStore, CancellationTokenSource cts)
        {
            var databasesInfo = new DatabasesInfo();
            var indexingSpeed = new IndexingSpeed();
            var trafficWatch  = new TrafficWatch();
            var drivesUsage   = new DrivesUsage();

            using (serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionContext))
                using (transactionContext.OpenReadTransaction())
                {
                    foreach (var databaseTuple in serverStore.Cluster.ItemsStartingWith(transactionContext, Constants.Documents.Prefix, 0, int.MaxValue))
                    {
                        var databaseName = databaseTuple.ItemName.Substring(3);
                        if (cts.IsCancellationRequested)
                        {
                            yield break;
                        }

                        if (serverStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out var databaseTask) == false)
                        {
                            // database does not exist in this server or disabled
                            continue;
                        }

                        var databaseOnline = IsDatabaseOnline(databaseTask, out var database);
                        if (databaseOnline == false)
                        {
                            var databaseRecord = serverStore.LoadDatabaseRecord(databaseName, out var _);
                            if (databaseRecord == null)
                            {
                                // database doesn't exist
                                continue;
                            }

                            var databaseInfoItem = new DatabaseInfoItem
                            {
                                Database = databaseName,
                                Online   = false
                            };

                            DatabaseInfo databaseInfo = null;
                            if (serverStore.DatabaseInfoCache.TryGet(databaseName,
                                                                     databaseInfoJson => databaseInfo = JsonDeserializationServer.DatabaseInfo(databaseInfoJson)))
                            {
                                Debug.Assert(databaseInfo != null);

                                databaseInfoItem.DocumentsCount      = databaseInfo.DocumentsCount ?? 0;
                                databaseInfoItem.IndexesCount        = databaseInfo.IndexesCount ?? databaseRecord.Indexes.Count;
                                databaseInfoItem.ReplicationFactor   = databaseRecord.Topology?.ReplicationFactor ?? databaseInfo.ReplicationFactor;
                                databaseInfoItem.ErroredIndexesCount = databaseInfo.IndexingErrors ?? 0;
                            }

                            databasesInfo.Items.Add(databaseInfoItem);
                            continue;
                        }

                        var indexingSpeedItem = new IndexingSpeedItem
                        {
                            Database         = database.Name,
                            IndexedPerSecond = database.Metrics.MapIndexes.IndexedPerSec.FiveSecondRate,
                            MappedPerSecond  = database.Metrics.MapReduceIndexes.MappedPerSec.FiveSecondRate,
                            ReducedPerSecond = database.Metrics.MapReduceIndexes.ReducedPerSec.FiveSecondRate
                        };
                        indexingSpeed.Items.Add(indexingSpeedItem);

                        var replicationFactor = GetReplicationFactor(databaseTuple.Value);
                        var documentsStorage  = database.DocumentsStorage;
                        var indexStorage      = database.IndexStore;
                        using (documentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext documentsContext))
                            using (documentsContext.OpenReadTransaction())
                            {
                                var databaseInfoItem = new DatabaseInfoItem
                                {
                                    Database            = databaseName,
                                    DocumentsCount      = documentsStorage.GetNumberOfDocuments(documentsContext),
                                    IndexesCount        = database.IndexStore.Count,
                                    AlertsCount         = database.NotificationCenter.GetAlertCount(),
                                    ReplicationFactor   = replicationFactor,
                                    ErroredIndexesCount = indexStorage.GetIndexes().Count(index => index.GetErrorCount() > 0),
                                    Online = true
                                };
                                databasesInfo.Items.Add(databaseInfoItem);
                            }

                        var trafficWatchItem = new TrafficWatchItem
                        {
                            Database            = databaseName,
                            RequestsPerSecond   = (int)database.Metrics.Requests.RequestsPerSec.FiveSecondRate,
                            WritesPerSecond     = (int)database.Metrics.Docs.PutsPerSec.FiveSecondRate,
                            WriteBytesPerSecond = database.Metrics.Docs.BytesPutsPerSec.FiveSecondRate
                        };
                        trafficWatch.Items.Add(trafficWatchItem);

                        foreach (var mountPointUsage in database.GetMountPointsUsage())
                        {
                            if (cts.IsCancellationRequested)
                            {
                                yield break;
                            }

                            var mountPoint = mountPointUsage.DiskSpaceResult.DriveName;
                            var usage      = drivesUsage.Items.FirstOrDefault(x => x.MountPoint == mountPoint);
                            if (usage == null)
                            {
                                usage = new MountPointUsage
                                {
                                    MountPoint    = mountPoint,
                                    VolumeLabel   = mountPointUsage.DiskSpaceResult.VolumeLabel,
                                    FreeSpace     = mountPointUsage.DiskSpaceResult.TotalFreeSpace.GetValue(SizeUnit.Bytes),
                                    TotalCapacity = mountPointUsage.DiskSpaceResult.TotalSize.GetValue(SizeUnit.Bytes)
                                };
                                drivesUsage.Items.Add(usage);
                            }

                            var existingDatabaseUsage = usage.Items.FirstOrDefault(x => x.Database == databaseName);
                            if (existingDatabaseUsage == null)
                            {
                                existingDatabaseUsage = new DatabaseDiskUsage
                                {
                                    Database = databaseName
                                };
                                usage.Items.Add(existingDatabaseUsage);
                            }

                            existingDatabaseUsage.Size += mountPointUsage.UsedSpace;
                        }
                    }
                }

            yield return(databasesInfo);

            yield return(indexingSpeed);

            yield return(trafficWatch);

            yield return(drivesUsage);
        }