private void ScheduleNextBackup(PeriodicBackup periodicBackup)
        {
            try
            {
                _serverStore.ConcurrentBackupsCounter.FinishBackup();

                periodicBackup.RunningTask         = null;
                periodicBackup.RunningBackupTaskId = null;
                periodicBackup.CancelToken         = null;
                periodicBackup.RunningBackupStatus = null;

                if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
                {
                    var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
                    periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
                }
            }
            catch (Exception e)
            {
                var message = $"Failed to schedule next backup for task: '{periodicBackup.Configuration.Name}'";
                if (_logger.IsOperationsEnabled)
                {
                    _logger.Operations(message, e);
                }

                _database.NotificationCenter.Add(AlertRaised.Create(
                                                     _database.Name,
                                                     "Couldn't schedule next backup",
                                                     message,
                                                     AlertType.PeriodicBackup,
                                                     NotificationSeverity.Warning,
                                                     details: new ExceptionDetails(e)));
            }
        }
Esempio n. 2
0
 private void AddAlertOnFailureToReachOtherSide(string msg, Exception e)
 {
     _database.NotificationCenter.Add(
         AlertRaised.Create(
             _database.Name,
             AlertTitle, msg, AlertType.Replication, NotificationSeverity.Warning, key: FromToString, details: new ExceptionDetails(e)));
 }
Esempio n. 3
0
        public RelationalDatabaseWriter(SqlEtl etl, DocumentDatabase database)
            : base(etl.Configuration.FactoryName)
        {
            _etl                         = etl;
            _database                    = database;
            _logger                      = LoggingSource.Instance.GetLogger <RelationalDatabaseWriter>(_database.Name);
            _providerFactory             = GetDbProviderFactory(etl.Configuration);
            _commandBuilder              = _providerFactory.InitializeCommandBuilder();
            _connection                  = _providerFactory.CreateConnection();
            _connection.ConnectionString = etl.Configuration.Connection.ConnectionString;

            try
            {
                _connection.Open();
            }
            catch (Exception e)
            {
                database.NotificationCenter.Add(AlertRaised.Create(
                                                    database.Name,
                                                    SqlEtl.SqlEtlTag,
                                                    $"SQL ETL could not open connection to {_connection.ConnectionString}",
                                                    AlertType.SqlEtl_ConnectionError,
                                                    NotificationSeverity.Error,
                                                    key: _connection.ConnectionString,
                                                    details: new ExceptionDetails(e)));

                throw;
            }

            _tx = _connection.BeginTransaction();

            _stringParserList = GenerateStringParsers();
        }
Esempio n. 4
0
        private void RunBackupThread(PeriodicBackup periodicBackup, BackupTask backupTask, Action <IOperationProgress> onProgress, TaskCompletionSource <IOperationResult> tcs)
        {
            try
            {
                Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
                NativeMemory.EnsureRegistered();

                using (_database.PreventFromUnloading())
                {
                    tcs.SetResult(backupTask.RunPeriodicBackup(onProgress));
                }
            }
            catch (OperationCanceledException)
            {
                tcs.SetCanceled();
            }
            catch (Exception e)
            {
                if (_logger.IsOperationsEnabled)
                {
                    _logger.Operations($"Failed to run the backup thread: '{periodicBackup.Configuration.Name}'", e);
                }

                tcs.SetException(e);
            }
            finally
            {
                try
                {
                    _serverStore.ConcurrentBackupsSemaphore.Release();

                    periodicBackup.RunningTask         = null;
                    periodicBackup.RunningBackupTaskId = null;
                    periodicBackup.CancelToken         = null;
                    periodicBackup.RunningBackupStatus = null;

                    if (periodicBackup.HasScheduledBackup() && _cancellationToken.IsCancellationRequested == false)
                    {
                        var newBackupTimer = GetTimer(periodicBackup.Configuration, periodicBackup.BackupStatus);
                        periodicBackup.UpdateTimer(newBackupTimer, discardIfDisabled: true);
                    }
                }
                catch (Exception e)
                {
                    var msg = $"Failed to schedule next backup for backup thread: '{periodicBackup.Configuration.Name}'";
                    if (_logger.IsOperationsEnabled)
                    {
                        _logger.Operations(msg, e);
                    }

                    _database.NotificationCenter.Add(AlertRaised.Create(
                                                         _database.Name,
                                                         "Couldn't schedule next backup.",
                                                         msg,
                                                         AlertType.PeriodicBackup,
                                                         NotificationSeverity.Warning,
                                                         details: new ExceptionDetails(e)));
                }
            }
        }
Esempio n. 5
0
        public void RecordLoadError(Exception e, int count = 1)
        {
            LoadErrors += count;

            LastErrorTime = SystemTime.UtcNow;

            LastAlert = AlertRaised.Create(_processType,
                                           $"[{_name}] Write error: {e.Message}",
                                           AlertType.Etl_LoadError,
                                           NotificationSeverity.Error,
                                           key: _name, details: new ExceptionDetails(e));

            if (LoadErrors < 100)
            {
                return;
            }

            if (LoadErrors <= LoadSuccesses)
            {
                return;
            }

            var message = $"[{_name}] Write error hit ratio too high. Could not tolerate write error ratio and stopped current ETL cycle";

            LastAlert = AlertRaised.Create(_processType,
                                           message,
                                           AlertType.Etl_WriteErrorRatio,
                                           NotificationSeverity.Error,
                                           key: _name,
                                           details: new ExceptionDetails(e));

            _notificationCenter.Add(LastAlert);

            throw new InvalidOperationException($"{message}. Current stats: {this}", e);
        }
Esempio n. 6
0
        private DbProviderFactory GetDbProviderFactory(SqlEtlConfiguration configuration)
        {
            DbProviderFactory providerFactory;

            try
            {
                providerFactory = DbProviderFactories.GetFactory(configuration.FactoryName);
            }
            catch (Exception e)
            {
                var message = $"Could not find provider factory {configuration.FactoryName} to replicate to sql for {configuration.Name}, ignoring.";

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info(message, e);
                }

                _database.NotificationCenter.Add(AlertRaised.Create(
                                                     _database.Name,
                                                     SqlEtl.SqlEtlTag,
                                                     message,
                                                     AlertType.SqlEtl_ProviderError,
                                                     NotificationSeverity.Error,
                                                     details: new ExceptionDetails(e)));

                throw;
            }
            return(providerFactory);
        }
Esempio n. 7
0
        private void HandleOnRecoveryError(StorageEnvironmentWithType.StorageEnvironmentType type, string resourceName, object environment, RecoveryErrorEventArgs e)
        {
            NotificationCenter.NotificationCenter nc;
            string title;

            switch (type)
            {
            case StorageEnvironmentWithType.StorageEnvironmentType.Configuration:
            case StorageEnvironmentWithType.StorageEnvironmentType.Documents:
                nc    = _serverStore?.NotificationCenter;
                title = $"Database Recovery Error - {resourceName ?? "Unknown Database"}";

                if (type == StorageEnvironmentWithType.StorageEnvironmentType.Configuration)
                {
                    title += " (configuration storage)";
                }
                break;

            case StorageEnvironmentWithType.StorageEnvironmentType.Index:
                nc    = NotificationCenter;
                title = $"Index Recovery Error - {resourceName ?? "Unknown Index"}";
                break;

            default:
                throw new ArgumentOutOfRangeException(nameof(type), type.ToString());
            }

            nc?.Add(AlertRaised.Create(Name,
                                       title,
                                       $"{e.Message}{Environment.NewLine}{Environment.NewLine}Environment: {environment}",
                                       AlertType.RecoveryError,
                                       NotificationSeverity.Error,
                                       key: resourceName));
        }
Esempio n. 8
0
        private void AlertIfDocumentStoreCreationRateIsNotReasonable(string applicationIdentifier, string name)
        {
            var q   = ServerStore.ClientCreationRate.GetOrCreate(applicationIdentifier);
            var now = DateTime.UtcNow;

            q.Enqueue(now);
            while (q.Count > 20)
            {
                if (q.TryDequeue(out var last) && (now - last).TotalMinutes < 1)
                {
                    q.Clear();

                    ServerStore.NotificationCenter.Add(
                        AlertRaised.Create(
                            name,
                            "Too many clients creations",
                            "There has been a lot of topology updates (more than 20) for the same client id in less than a minute. " +
                            $"Last one from ({HttpContext.Connection.RemoteIpAddress} as " +
                            $"{HttpContext.Connection.ClientCertificate?.FriendlyName ?? HttpContext.Connection.ClientCertificate?.Thumbprint ?? "<unsecured>"})" +
                            "This is usually an indication that you are creating a large number of DocumentStore instance. " +
                            "Are you creating a Document Store per request, instead of using DocumentStore as a singleton? ",
                            AlertType.HighClientCreationRate,
                            NotificationSeverity.Warning
                            ));
                }
            }
        }
Esempio n. 9
0
 internal void HandleOnRecoveryError(object sender, RecoveryErrorEventArgs e)
 {
     _serverStore?.NotificationCenter.Add(AlertRaised.Create($"Database Recovery Error - {Name ?? "Unknown Database"}",
                                                             e.Message,
                                                             AlertType.RecoveryError,
                                                             NotificationSeverity.Error,
                                                             Name));
 }
Esempio n. 10
0
        public static TimeSeriesPolicyRunner LoadConfigurations(DocumentDatabase database, DatabaseRecord dbRecord, TimeSeriesPolicyRunner policyRunner)
        {
            try
            {
                if (dbRecord.TimeSeries?.Collections == null || dbRecord.TimeSeries.Collections.Count == 0)
                {
                    policyRunner?.Dispose();
                    return null;
                }

                database.ServerStore.LicenseManager.AssertCanAddTimeSeriesRollupsAndRetention(dbRecord.TimeSeries);

                if (policyRunner != null)
                {
                    // no changes
                    if (policyRunner.Configuration.PolicyConfigurationChanged(dbRecord.TimeSeries) == false)
                        return policyRunner;
                }

                policyRunner?.Dispose();

                var runner = new TimeSeriesPolicyRunner(database, dbRecord.TimeSeries);
                runner.Start();
                return runner;
            }
            catch (Exception e)
            {
                const string msg = "Cannot enable policy runner as the configuration record is not valid.";

                if (e is LicenseLimitException lle)
                {
                    LicenseLimitWarning.AddLicenseLimitNotification(database.ServerStore.NotificationCenter, lle);
                }
                else
                {
                    database.NotificationCenter.Add(AlertRaised.Create(
                        database.Name,
                        $"Time series policy runner for database '{database.Name}' encountered an error", msg,
                        AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, database.Name));
                }

                var logger = LoggingSource.Instance.GetLogger<TimeSeriesPolicyRunner>(database.Name);
                if (logger.IsOperationsEnabled)
                    logger.Operations(msg, e);

                try
                {
                    policyRunner?.Dispose();
                }
                catch (Exception ex)
                {
                    if (logger.IsOperationsEnabled)
                        logger.Operations("Failed to dispose previous time-series policy runner", ex);
                }

                return null;
            }
        }
Esempio n. 11
0
 private static List <Notification> CreateSampleNotificationsForFilterOutTest()
 {
     return(new List <Notification>
     {
         AlertRaised.Create(
             null,
             "DatabaseTopologyWarning",
             "DatabaseTopologyWarning_MSG",
             AlertType.DatabaseTopologyWarning,
             NotificationSeverity.Info),
         DatabaseChanged.Create(null, DatabaseChangeType.Put), // filtered out, DatabaseChange
         AlertRaised.Create(
             null,
             "LicenseManager_AGPL3",
             "LicenseManager_AGPL3_MSG",
             AlertType.ClusterTransactionFailure,
             NotificationSeverity.Info),
         AlertRaised.Create(
             null,
             "LicenseManager_AGPL3",
             "LicenseManager_AGPL3_MSG",
             AlertType.LicenseManager_AGPL3, // filtered out explicitly
             NotificationSeverity.Info),
         AlertRaised.Create(
             null,
             "RevisionsConfigurationNotValid",
             "RevisionsConfigurationNotValid_MSG",
             AlertType.RevisionsConfigurationNotValid, // filtered out explicitly
             NotificationSeverity.Info),
         AlertRaised.Create(
             null,
             "Certificates_ReplaceError",
             "Certificates_ReplaceError_MSG",
             AlertType.Certificates_ReplaceError,
             NotificationSeverity.Info),
         PerformanceHint.Create(
             null,
             "SlowIO",
             "SlowIO_MSG",
             PerformanceHintType.SlowIO, // filtered out, PerformanceHint
             NotificationSeverity.Info,
             "test"),
         PerformanceHint.Create(
             null,
             "SqlEtl_SlowSql",
             "SqlEtl_SlowSql_MSG",
             PerformanceHintType.SqlEtl_SlowSql, // filtered out, PerformanceHint
             NotificationSeverity.Info,
             "test"),
         OperationChanged.Create(null, 1, new Operations.OperationDescription(), new OperationState()
         {
             Result = new PersistableResult()
         }, false),
         DatabaseChanged.Create(null, DatabaseChangeType.Delete) // filtered out, DatabaseChange
     });
 }
        public void Execute(string databaseName, Exception e, Guid environmentId)
        {
            var stats = _errorsPerEnvironment.GetOrAdd(environmentId, x => FailureStats.Create(MaxDatabaseUnloads));

            if (stats.WillUnloadDatabase == false)
            {
                if (DateTime.UtcNow - stats.LastUnloadTime > NoFailurePeriod)
                {
                    // let it unload again after it was working fine for a given time with no failure

                    stats.NumberOfUnloads = 0;
                    stats.LastUnloadTime  = DateTime.MinValue;
                }
                else
                {
                    return;
                }
            }

            stats.DatabaseUnloadTask = Task.Run(async() =>
            {
                var title            = $"Critical error in '{databaseName}'";
                const string message = "Database is about to be unloaded due to an encountered error";

                try
                {
                    _serverStore.NotificationCenter.Add(AlertRaised.Create(
                                                            databaseName,
                                                            title,
                                                            message,
                                                            AlertType.CatastrophicDatabaseFailure,
                                                            NotificationSeverity.Error,
                                                            key: databaseName,
                                                            details: new ExceptionDetails(e)));
                }
                catch (Exception)
                {
                    // exception in raising an alert can't prevent us from unloading a database
                }

                if (_logger.IsOperationsEnabled)
                {
                    _logger.Operations($"{title}. {message}", e);
                }

                // let it propagate the exception to the client first and do
                // the internal failure handling e.g. Index.HandleIndexCorruption
                await Task.Delay(TimeToWaitBeforeUnloadingDatabase);

                stats.NumberOfUnloads++;
                stats.LastUnloadTime = DateTime.UtcNow;
                (await _databasesLandlord.UnloadAndLockDatabase(databaseName, "CatastrophicFailure"))?.Dispose();

                stats.DatabaseUnloadTask = null;
            });
        }
Esempio n. 13
0
 private static AlertRaised GetSampleAlert(string customMessage = null, string customKey = null)
 {
     return(AlertRaised.Create(
                "title",
                customMessage ?? "Alert #1",
                0, //use any type
                NotificationSeverity.Info,
                key: customKey ?? "Key",
                details: new ExceptionDetails(new Exception("Error message"))));
 }
Esempio n. 14
0
        private void HandleSlowSql(long elapsedMiliseconds, string stmt)
        {
            var message = $"[{_etl.Name}] Slow SQL detected. Execution took: {elapsedMiliseconds}ms, statement: {stmt}";

            if (_logger.IsInfoEnabled)
            {
                _logger.Info(message);
            }

            _database.NotificationCenter.Add(AlertRaised.Create(_database.Name, _etl.Tag, message, AlertType.SqlEtl_SlowSql, NotificationSeverity.Warning));
        }
Esempio n. 15
0
        public void InitializeFromDatabaseRecord(DatabaseRecord dbRecord)
        {
            try
            {
                var revisions = dbRecord.Revisions;
                if (revisions == null ||
                    (revisions.Default == null && revisions.Collections.Count == 0))
                {
                    Configuration = null;
                    return;
                }

                if (revisions.Equals(Configuration))
                {
                    return;
                }

                Configuration = revisions;

                using (var tx = _database.DocumentsStorage.Environment.WriteTransaction())
                {
                    foreach (var collection in Configuration.Collections)
                    {
                        if (collection.Value.Disabled)
                        {
                            continue;
                        }
                        EnsureRevisionTableCreated(tx, new CollectionName(collection.Key));
                    }

                    CreateTrees(tx);

                    tx.Commit();
                }

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info("Revisions configuration changed");
                }
            }
            catch (Exception e)
            {
                var msg = "Cannot enable revisions for documents as the revisions configuration " +
                          "in the database record is missing or not valid.";
                _database.NotificationCenter.Add(AlertRaised.Create(
                                                     _database.Name,
                                                     $"Revisions error in {_database.Name}", msg,
                                                     AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, _database.Name));
                if (_logger.IsOperationsEnabled)
                {
                    _logger.Operations(msg, e);
                }
            }
        }
Esempio n. 16
0
        public static void AddLicenseLimitNotification(ServerStore serverStore, LicenseLimit licenseLimit)
        {
            var alert = AlertRaised.Create(
                "You've reached your license limit",
                licenseLimit.Message,
                AlertType.LicenseManager_LicenseLimit,
                NotificationSeverity.Warning,
                details: new LicenseLimitWarning(licenseLimit));

            serverStore.NotificationCenter.Add(alert);
        }
Esempio n. 17
0
 internal void HandleNonDurableFileSystemError(object sender, NonDurabilitySupportEventArgs e)
 {
     _serverStore?.NotificationCenter.Add(AlertRaised.Create($"Non Durable File System - {Name ?? "Unknown Database"}",
                                                             e.Message,
                                                             AlertType.NonDurableFileSystem,
                                                             NotificationSeverity.Warning,
                                                             Name,
                                                             details: new MessageDetails {
         Message = e.Details
     }));
 }
Esempio n. 18
0
        public static void AddLicenseLimitNotification(NotificationCenter notificationCenter, LicenseLimitException licenseLimit)
        {
            var alert = AlertRaised.Create(
                null,
                $@"You've reached your license limit ({EnumHelper.GetDescription(licenseLimit.Type)})",
                licenseLimit.Message,
                AlertType.LicenseManager_LicenseLimit,
                NotificationSeverity.Warning,
                details: new LicenseLimitWarning(licenseLimit));

            notificationCenter.Add(alert);
        }
Esempio n. 19
0
        private void AddAlertOnFailureToReachOtherSide(string msg, Exception e)
        {
            using (_database.ConfigurationStorage.ContextPool.AllocateOperationContext(out TransactionOperationContext configurationContext))
                using (var txw = configurationContext.OpenWriteTransaction())
                {
                    _database.NotificationCenter.AddAfterTransactionCommit(
                        AlertRaised.Create(AlertTitle, msg, AlertType.Replication, NotificationSeverity.Warning, key: FromToString, details: new ExceptionDetails(e)),
                        txw);

                    txw.Commit();
                }
        }
Esempio n. 20
0
        private void OpenIndex(PathSetting path, string indexPath, List <Exception> exceptions, string name)
        {
            Index index = null;

            try
            {
                index = Index.Open(indexPath, _documentDatabase);
                index.Start();
                if (_logger.IsInfoEnabled)
                {
                    _logger.Info($"Started {index.Name} from {indexPath}");
                }

                _indexes.Add(index);
            }
            catch (Exception e)
            {
                var alreadyFaulted = false;
                if (index != null && _indexes.TryGetByName(index.Name, out Index i))
                {
                    if (i is FaultyInMemoryIndex)
                    {
                        alreadyFaulted = true;
                    }
                }
                index?.Dispose();
                exceptions?.Add(e);
                if (alreadyFaulted)
                {
                    return;
                }
                var configuration = new FaultyInMemoryIndexConfiguration(path, _documentDatabase.Configuration);

                var fakeIndex = new FaultyInMemoryIndex(e, name, configuration);

                var message = $"Could not open index at '{indexPath}'. Created in-memory, fake instance: {fakeIndex.Name}";

                if (_logger.IsInfoEnabled)
                {
                    _logger.Info(message, e);
                }

                _documentDatabase.NotificationCenter.Add(AlertRaised.Create(
                                                             _documentDatabase.Name,
                                                             "Indexes store initialization error",
                                                             message,
                                                             AlertType.IndexStore_IndexCouldNotBeOpened,
                                                             NotificationSeverity.Error,
                                                             key: fakeIndex.Name,
                                                             details: new ExceptionDetails(e)));
                _indexes.Add(fakeIndex);
            }
        }
Esempio n. 21
0
        public static ExpiredDocumentsCleaner LoadConfigurations(DocumentDatabase database, DatabaseRecord dbRecord, ExpiredDocumentsCleaner expiredDocumentsCleaner)
        {
            try
            {
                if (dbRecord.Expiration == null && dbRecord.Refresh == null)
                {
                    expiredDocumentsCleaner?.Dispose();
                    return(null);
                }

                if (expiredDocumentsCleaner != null)
                {
                    // no changes
                    if (Equals(expiredDocumentsCleaner.ExpirationConfiguration, dbRecord.Expiration) &&
                        Equals(expiredDocumentsCleaner.RefreshConfiguration, dbRecord.Refresh))
                    {
                        return(expiredDocumentsCleaner);
                    }
                }

                expiredDocumentsCleaner?.Dispose();

                var hasExpiration = dbRecord.Expiration?.Disabled == false;
                var hasRefresh    = dbRecord.Refresh?.Disabled == false;

                if (hasExpiration == false && hasRefresh == false)
                {
                    return(null);
                }

                var cleaner = new ExpiredDocumentsCleaner(database, dbRecord.Expiration, dbRecord.Refresh);
                cleaner.Start();
                return(cleaner);
            }
            catch (Exception e)
            {
                const string msg = "Cannot enable expired documents cleaner as the configuration record is not valid.";
                database.NotificationCenter.Add(AlertRaised.Create(
                                                    database.Name,
                                                    $"Expiration error in {database.Name}", msg,
                                                    AlertType.RevisionsConfigurationNotValid, NotificationSeverity.Error, database.Name));

                var logger = LoggingSource.Instance.GetLogger <ExpiredDocumentsCleaner>(database.Name);
                if (logger.IsOperationsEnabled)
                {
                    logger.Operations(msg, e);
                }

                return(null);
            }
        }
Esempio n. 22
0
        private void LogConfigurationWarning <T>(EtlConfiguration <T> config, List <string> warnings) where T : ConnectionString
        {
            var warnMessage = $"Warning about ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " +
                              $"Reason{(warnings.Count > 1 ? "s" : string.Empty)}: {string.Join(";", warnings)}.";

            if (Logger.IsInfoEnabled)
            {
                Logger.Info(warnMessage);
            }

            var alert = AlertRaised.Create(_database.Name, AlertTitle, warnMessage, AlertType.Etl_Warning, NotificationSeverity.Warning);

            _database.NotificationCenter.Add(alert);
        }
Esempio n. 23
0
        private void LogConfigurationError <T>(EtlConfiguration <T> config, List <string> errors) where T : ConnectionString
        {
            var errorMessage = $"Invalid ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " +
                               $"Reason{(errors.Count > 1 ? "s" : string.Empty)}: {string.Join(";", errors)}.";

            if (Logger.IsInfoEnabled)
            {
                Logger.Info(errorMessage);
            }

            var alert = AlertRaised.Create(_database.Name, AlertTitle, errorMessage, AlertType.Etl_Error, NotificationSeverity.Error);

            _database.NotificationCenter.Add(alert);
        }
Esempio n. 24
0
        public void Initialize(StorageEnvironment environment, TransactionContextPool contextPool)
        {
            try
            {
                _licenseStorage.Initialize(environment, contextPool);

                var firstServerStartDate = _licenseStorage.GetFirstServerStartDate();
                if (firstServerStartDate == null)
                {
                    firstServerStartDate = SystemTime.UtcNow;
                    _licenseStorage.SetFirstServerStartDate(firstServerStartDate.Value);
                }

                _licenseStatus.FirstServerStartDate = firstServerStartDate.Value;

                var license = _serverStore.LoadLicense();
                if (license == null)
                {
                    return;
                }

                _leaseLicenseTimer = new Timer(state =>
                                               AsyncHelpers.RunSync(LeaseLicense), null, 0, (int)TimeSpan.FromHours(24).TotalMilliseconds);

                _licenseStatus.Attributes = LicenseValidator.Validate(license, RSAParameters);
                _licenseStatus.Error      = false;
                _licenseStatus.Message    = null;
            }
            catch (Exception e)
            {
                _licenseStatus.Attributes = null;
                _licenseStatus.Error      = true;
                _licenseStatus.Message    = e.Message;

                if (Logger.IsInfoEnabled)
                {
                    Logger.Info("Could not validate license", e);
                }

                var alert = AlertRaised.Create(
                    "License manager initialization error",
                    "Could not intitalize the license manager",
                    AlertType.LicenseManager_InitializationError,
                    NotificationSeverity.Warning,
                    details: new ExceptionDetails(e));

                _serverStore.NotificationCenter.Add(alert);
            }
        }
Esempio n. 25
0
        private void RaiseNoLivingNodesAlert(string alertMsg)
        {
            var alert = AlertRaised.Create(
                "No living nodes in the database topology",
                alertMsg,
                AlertType.DatabaseTopologyWarning,
                NotificationSeverity.Warning
                );

            NotificationCenter.Add(alert);
            if (_logger.IsOperationsEnabled)
            {
                _logger.Operations(alertMsg);
            }
        }
Esempio n. 26
0
        private void RaiseNoLivingNodesAlert(string alertMsg, string dbName)
        {
            var alert = AlertRaised.Create(
                $"Could not reach any node of {dbName} database",
                $"{alertMsg}. {ThingsToCheck}",
                AlertType.DatabaseTopologyWarning,
                NotificationSeverity.Warning
                );

            NotificationCenter.Add(alert);
            if (_logger.IsOperationsEnabled)
            {
                _logger.Operations(alertMsg);
            }
        }
Esempio n. 27
0
        public void Add(StorageEnvironment environment, Exception exception)
        {
            var notificationsMetadata = _notificationsMetadataTable.GetOrCreateValue(environment);

            if (notificationsMetadata.TryGetValue(exception.GetType(), out var notificationMetadata))
            {
                if (DateTime.Now - notificationMetadata.Time < _updateFrequency)
                {
                    return;
                }

                if (Interlocked.CompareExchange(ref notificationMetadata.IsInProgress, 1, 0) == 1)
                {
                    return;
                }

                notificationMetadata.Time = DateTime.Now;
            }
            else
            {
                notificationMetadata = new NotificationTime
                {
                    Time         = DateTime.Now,
                    IsInProgress = 1
                };
                if (notificationsMetadata.TryAdd(exception.GetType(), notificationMetadata) == false)
                {
                    return;
                }

                //We are in low of memory so we want to minimize allocations
                notificationMetadata.Key   = $"{environment}:{exception.GetType()}";
                notificationMetadata.Title = $"Out of memory occurred for '{environment}'";
            }

            var alert = AlertRaised.Create(
                null,
                notificationMetadata.Title,
                exception.Message,
                AlertType.OutOfMemoryException,
                NotificationSeverity.Error,
                notificationMetadata.Key,
                OutOfMemoryDetails(exception));

            _notificationsCenter.Add(alert);

            Volatile.Write(ref notificationMetadata.IsInProgress, 0);
        }
Esempio n. 28
0
        private void RaiseNodeNotFoundAlert(string alertMsg, string node)
        {
            var alert = AlertRaised.Create(
                null,
                $"Node {node} not found.",
                $"{alertMsg}",
                AlertType.DatabaseTopologyWarning,
                NotificationSeverity.Warning
                );

            NotificationCenter.Add(alert);
            if (_logger.IsOperationsEnabled)
            {
                _logger.Operations(alertMsg);
            }
        }
Esempio n. 29
0
        private TaskStatus GetTaskStatus(
            DatabaseRecord databaseRecord,
            PeriodicBackupConfiguration configuration,
            bool skipErrorLog = false)
        {
            if (configuration.Disabled)
            {
                return(TaskStatus.Disabled);
            }

            if (configuration.HasBackup() == false)
            {
                if (skipErrorLog == false)
                {
                    var message = $"All backup destinations are disabled for backup task id: {configuration.TaskId}";
                    _database.NotificationCenter.Add(AlertRaised.Create(
                                                         _database.Name,
                                                         "Periodic Backup",
                                                         message,
                                                         AlertType.PeriodicBackup,
                                                         NotificationSeverity.Info));
                }

                return(TaskStatus.Disabled);
            }

            var backupStatus  = GetBackupStatus(configuration.TaskId);
            var whoseTaskIsIt = _database.WhoseTaskIsIt(databaseRecord.Topology, configuration, backupStatus, useLastResponsibleNodeIfNoAvailableNodes: true);

            if (whoseTaskIsIt == null)
            {
                return(TaskStatus.Disabled);
            }

            if (whoseTaskIsIt == _serverStore.NodeTag)
            {
                return(TaskStatus.ActiveByCurrentNode);
            }

            if (_logger.IsInfoEnabled)
            {
                _logger.Info($"Backup job is skipped at {SystemTime.UtcNow}, because it is managed " +
                             $"by '{whoseTaskIsIt}' node and not the current node ({_serverStore.NodeTag})");
            }

            return(TaskStatus.ActiveByOtherNode);
        }
Esempio n. 30
0
        private NextBackup GetNextBackupDetails(
            PeriodicBackupConfiguration configuration,
            PeriodicBackupStatus backupStatus,
            bool skipErrorLog = false)
        {
            var now                   = SystemTime.UtcNow;
            var lastFullBackup        = backupStatus.LastFullBackupInternal ?? now;
            var lastIncrementalBackup = backupStatus.LastIncrementalBackupInternal ?? backupStatus.LastFullBackupInternal ?? now;
            var nextFullBackup        = GetNextBackupOccurrence(configuration.FullBackupFrequency,
                                                                lastFullBackup, configuration, skipErrorLog: skipErrorLog);
            var nextIncrementalBackup = GetNextBackupOccurrence(configuration.IncrementalBackupFrequency,
                                                                lastIncrementalBackup, configuration, skipErrorLog: skipErrorLog);

            if (nextFullBackup == null && nextIncrementalBackup == null)
            {
                var message = "Couldn't schedule next backup " +
                              $"full backup frequency: {configuration.FullBackupFrequency}, " +
                              $"incremental backup frequency: {configuration.IncrementalBackupFrequency}";
                if (string.IsNullOrWhiteSpace(configuration.Name) == false)
                {
                    message += $", backup name: {configuration.Name}";
                }

                _database.NotificationCenter.Add(AlertRaised.Create(
                                                     _database.Name,
                                                     "Couldn't schedule next backup, this shouldn't happen",
                                                     message,
                                                     AlertType.PeriodicBackup,
                                                     NotificationSeverity.Warning));

                return(null);
            }

            Debug.Assert(configuration.TaskId != 0);

            var isFullBackup       = IsFullBackup(backupStatus, configuration, nextFullBackup, nextIncrementalBackup);
            var nextBackupDateTime = GetNextBackupDateTime(nextFullBackup, nextIncrementalBackup);
            var nowLocalTime       = now.ToLocalTime();
            var nextBackupTimeSpan = (nextBackupDateTime - nowLocalTime).Ticks <= 0 ? TimeSpan.Zero : nextBackupDateTime - nowLocalTime;

            return(new NextBackup
            {
                TimeSpan = nextBackupTimeSpan,
                DateTime = DateTime.UtcNow.Add(nextBackupTimeSpan),
                IsFull = isFullBackup
            });
        }