예제 #1
0
        public async Task PromoteNode()
        {
            if (ServerStore.LeaderTag == null)
            {
                NoContentStatus();
                return;
            }

            SetupCORSHeaders();

            if (ServerStore.IsLeader() == false)
            {
                RedirectToLeader();
                return;
            }

            var nodeTag = GetStringQueryString("nodeTag");

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var topology = ServerStore.GetClusterTopology(context);
                    if (topology.Watchers.ContainsKey(nodeTag) == false)
                    {
                        throw new InvalidOperationException(
                                  $"Failed to promote node {nodeTag} beacuse {nodeTag} is not a watcher in the cluster topology");
                    }

                    var url = topology.GetUrlFromTag(nodeTag);
                    await ServerStore.Engine.ModifyTopologyAsync(nodeTag, url, Leader.TopologyModification.Promotable);

                    NoContentStatus();
                }
        }
예제 #2
0
        public Task GetNodeInfo()
        {
            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    var json = new DynamicJsonValue();
                    using (context.OpenReadTransaction())
                    {
                        json[nameof(NodeInfo.NodeTag)]     = ServerStore.NodeTag;
                        json[nameof(NodeInfo.TopologyId)]  = ServerStore.GetClusterTopology(context).TopologyId;
                        json[nameof(NodeInfo.Certificate)] = ServerStore.Server.Certificate.CertificateForClients;
                        json[nameof(ServerStore.Engine.LastStateChangeReason)] = ServerStore.LastStateChangeReason();
                        json[nameof(NodeInfo.NumberOfCores)] = ProcessorInfo.ProcessorCount;

                        var memoryInformation = MemoryInformation.GetMemoryInfo();
                        json[nameof(NodeInfo.InstalledMemoryInGb)] = memoryInformation.InstalledMemory.GetDoubleValue(SizeUnit.Gigabytes);
                        json[nameof(NodeInfo.UsableMemoryInGb)]    = memoryInformation.TotalPhysicalMemory.GetDoubleValue(SizeUnit.Gigabytes);
                        json[nameof(NodeInfo.BuildInfo)]           = LicenseManager.BuildInfo;
                        json[nameof(NodeInfo.ServerId)]            = ServerStore.GetServerId().ToString();
                        json[nameof(NodeInfo.CurrentState)]        = ServerStore.CurrentRachisState;
                    }
                    context.Write(writer, json);
                    writer.Flush();
                }
            return(Task.CompletedTask);
        }
예제 #3
0
        protected void RedirectToLeader()
        {
            if (ServerStore.LeaderTag == null)
            {
                throw new NoLeaderException();
            }

            ClusterTopology topology;

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    topology = ServerStore.GetClusterTopology(context);
                }
            var url = topology.GetUrlFromTag(ServerStore.LeaderTag);

            if (string.Equals(url, ServerStore.GetNodeHttpServerUrl(), StringComparison.OrdinalIgnoreCase))
            {
                throw new NoLeaderException($"This node is not the leader, but the current topology does mark it as the leader. Such confusion is usually an indication of a network or configuration problem.");
            }
            var leaderLocation = url + HttpContext.Request.Path + HttpContext.Request.QueryString;

            HttpContext.Response.StatusCode = (int)HttpStatusCode.TemporaryRedirect;
            HttpContext.Response.Headers.Remove("Content-Type");
            HttpContext.Response.Headers.Add("Location", leaderLocation);
        }
예제 #4
0
        public Task GetClusterTopology()
        {
            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var topology = ServerStore.GetClusterTopology(context);
                    var nodeTag  = ServerStore.NodeTag;

                    if (topology.Members.Count == 0)
                    {
                        var tag       = ServerStore.NodeTag ?? "A";
                        var serverUrl = ServerStore.NodeHttpServerUrl;

                        topology = new ClusterTopology(
                            "dummy",
                            new Dictionary <string, string>
                        {
                            [tag] = serverUrl
                        },
                            new Dictionary <string, string>(),
                            new Dictionary <string, string>(),
                            tag
                            );
                        nodeTag = tag;
                    }

                    HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;

                    using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        var loadLicenseLimits  = ServerStore.LoadLicenseLimits();
                        var nodeLicenseDetails = loadLicenseLimits == null ?
                                                 null : DynamicJsonValue.Convert(loadLicenseLimits.NodeLicenseDetails);
                        var json = new DynamicJsonValue
                        {
                            ["Topology"]           = topology.ToSortedJson(),
                            ["Leader"]             = ServerStore.LeaderTag,
                            ["CurrentState"]       = ServerStore.CurrentRachisState,
                            ["NodeTag"]            = nodeTag,
                            ["CurrentTerm"]        = ServerStore.Engine.CurrentTerm,
                            ["NodeLicenseDetails"] = nodeLicenseDetails,
                            [nameof(ServerStore.Engine.LastStateChangeReason)] = ServerStore.LastStateChangeReason()
                        };
                        var clusterErrors = ServerStore.GetClusterErrors();
                        if (clusterErrors.Count > 0)
                        {
                            json["Errors"] = clusterErrors;
                        }

                        var nodesStatues = ServerStore.GetNodesStatuses();
                        json["Status"] = DynamicJsonValue.Convert(nodesStatues);

                        context.Write(writer, json);
                        writer.Flush();
                    }
                }

            return(Task.CompletedTask);
        }
예제 #5
0
 public ClusterTopology GetClusterTopology()
 {
     using (_server.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
         using (ctx.OpenReadTransaction())
         {
             return(_server.GetClusterTopology(ctx));
         }
 }
예제 #6
0
        public async Task DistributeKeyInCluster()
        {
            await ServerStore.EnsureNotPassiveAsync();

            var name  = GetStringQueryString("name");
            var nodes = GetStringValuesQueryString("node");

            using (var reader = new StreamReader(HttpContext.Request.Body))
            {
                var base64 = await reader.ReadToEndAsync();

                using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
                {
                    ClusterTopology clusterTopology;
                    using (ctx.OpenReadTransaction())
                        clusterTopology = ServerStore.GetClusterTopology(ctx);

                    foreach (var node in nodes)
                    {
                        if (string.IsNullOrEmpty(node))
                        {
                            continue;
                        }

                        if (string.Equals(node, "?") || string.Equals(node, ServerStore.NodeTag, StringComparison.OrdinalIgnoreCase))
                        {
                            var key = Convert.FromBase64String(base64);

                            if (key.Length != 256 / 8)
                            {
                                throw new ArgumentException($"Key size must be 256 bits, but was {key.Length * 8}", nameof(key));
                            }

                            StoreKeyLocally(name, key, ctx);
                        }
                        else
                        {
                            var url = clusterTopology.GetUrlFromTag(node);
                            if (url == null)
                            {
                                throw new InvalidOperationException($"Node {node} is not a part of the cluster, cannot send secret key.");
                            }

                            if (url.StartsWith("https:", StringComparison.OrdinalIgnoreCase) == false)
                            {
                                throw new InvalidOperationException($"Cannot put secret key for {name} on node {node} with url {url} because it is not using HTTPS");
                            }

                            await SendKeyToNodeAsync(name, base64, ctx, ServerStore, node, url).ConfigureAwait(false);
                        }
                    }
                }
            }

            HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;
        }
예제 #7
0
        public async Task Watch()
        {
            var nodeTag = GetStringQueryString("node", required: true);

            using (var webSocket = await HttpContext.WebSockets.AcceptWebSocketAsync())
            {
                try
                {
                    if (nodeTag.Equals(ServerStore.NodeTag, StringComparison.OrdinalIgnoreCase))
                    {
                        var canAccessDatabase = GetDatabaseAccessValidationFunc();

                        await SendNotifications(canAccessDatabase, webSocket);
                    }
                    else
                    {
                        ClusterTopology topology;

                        using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext txOperationContext))
                            using (txOperationContext.OpenReadTransaction())
                            {
                                topology = ServerStore.GetClusterTopology(txOperationContext);
                            }

                        var remoteNodeUrl = topology.GetUrlFromTag(nodeTag);

                        if (string.IsNullOrEmpty(remoteNodeUrl))
                        {
                            throw new InvalidOperationException($"Could not find node url for node tag '{nodeTag}'");
                        }

                        var currentCert = GetCurrentCertificate();

                        using (var connection = new ProxyWebSocketConnection(webSocket, remoteNodeUrl, $"/admin/cluster-dashboard/remote/watch?thumbprint={currentCert?.Thumbprint}", ServerStore.ContextPool, ServerStore.ServerShutdown))
                        {
                            await connection.Establish(Server.Certificate?.Certificate);

                            await connection.RelayData();
                        }
                    }
                }
                catch (OperationCanceledException)
                {
                    // ignored
                }
                catch (Exception ex)
                {
                    await HandleException(ex, webSocket);
                }
            }
        }
예제 #8
0
        private async Task CreateInternal(BlittableJsonReaderObject bjro, SubscriptionCreationOptions options, DocumentsOperationContext context, long?id, bool?disabled)
        {
            if (TrafficWatchManager.HasRegisteredClients)
            {
                AddStringToHttpContext(bjro.ToString(), TrafficWatchChangeType.Subscriptions);
            }

            var sub = SubscriptionConnection.ParseSubscriptionQuery(options.Query);

            if (Enum.TryParse(options.ChangeVector, out Constants.Documents.SubscriptionChangeVectorSpecialStates changeVectorSpecialValue))
            {
                switch (changeVectorSpecialValue)
                {
                case Constants.Documents.SubscriptionChangeVectorSpecialStates.BeginningOfTime:

                    options.ChangeVector = null;
                    break;

                case Constants.Documents.SubscriptionChangeVectorSpecialStates.LastDocument:
                    options.ChangeVector = Database.DocumentsStorage.GetLastDocumentChangeVector(context.Transaction.InnerTransaction, context, sub.Collection);
                    break;
                }
            }

            var mentor         = options.MentorNode;
            var subscriptionId = await Database.SubscriptionStorage.PutSubscription(options, GetRaftRequestIdFromQuery(), id, disabled, mentor);

            var name = options.Name ?? subscriptionId.ToString();

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext serverContext))
                using (serverContext.OpenReadTransaction())
                {
                    // need to wait on the relevant remote node
                    var node = Database.SubscriptionStorage.GetResponsibleNode(serverContext, name);
                    if (node != null && node != ServerStore.NodeTag)
                    {
                        await WaitForExecutionOnSpecificNode(serverContext, ServerStore.GetClusterTopology(serverContext), node, subscriptionId);
                    }
                }

            HttpContext.Response.StatusCode = (int)HttpStatusCode.Created;

            await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
            {
                context.Write(writer, new DynamicJsonValue
                {
                    [nameof(CreateSubscriptionResult.Name)] = name
                });
            }
        }
예제 #9
0
        public Task ClusterDomains()
        {
            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                List <string> domains = null;
                if (ServerStore.CurrentRachisState != RachisState.Passive)
                {
                    ClusterTopology clusterTopology;
                    using (context.OpenReadTransaction())
                        clusterTopology = ServerStore.GetClusterTopology(context);

                    domains = clusterTopology.AllNodes.Select(node => new Uri(node.Value).DnsSafeHost).ToList();
                }
                else
                {
                    var myUrl = Server.Configuration.Core.PublicServerUrl.HasValue
                        ? Server.Configuration.Core.PublicServerUrl.Value.UriValue
                        : Server.Configuration.Core.ServerUrls[0];
                    var myDomain = new Uri(myUrl).DnsSafeHost;
                    domains = new List <string>
                    {
                        myDomain
                    };
                }

                using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    writer.WriteStartObject();
                    writer.WritePropertyName("ClusterDomains");

                    writer.WriteStartArray();
                    var first = true;
                    foreach (var domain in domains)
                    {
                        if (first == false)
                        {
                            writer.WriteComma();
                        }
                        first = false;
                        writer.WriteString(domain);
                    }
                    writer.WriteEndArray();

                    writer.WriteEndObject();
                }
            }

            return(Task.CompletedTask);
        }
        private Dictionary <string, (HashSet <string>, string)> CreateUrlToDatabaseNamesMapping(TransactionOperationContext transactionOperationContext, IEnumerable <string> databaseNames)
        {
            var nodeUrlToDatabaseNames = new Dictionary <string, (HashSet <string>, string)>();
            var clusterTopology        = ServerStore.GetClusterTopology(transactionOperationContext);

            foreach (var databaseName in databaseNames)
            {
                var topology        = ServerStore.Cluster.ReadDatabaseTopology(transactionOperationContext, databaseName);
                var nodeUrlsAndTags = topology.AllNodes.Select(tag => (clusterTopology.GetUrlFromTag(tag), tag));
                foreach (var urlAndTag in nodeUrlsAndTags)
                {
                    if (nodeUrlToDatabaseNames.TryGetValue(urlAndTag.Item1, out (HashSet <string>, string)databaseNamesWithNodeTag))
                    {
                        databaseNamesWithNodeTag.Item1.Add(databaseName);
                    }
예제 #11
0
 public Task GetNodeInfo()
 {
     using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
         using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
         {
             var json = new DynamicJsonValue();
             using (context.OpenReadTransaction())
             {
                 json[nameof(NodeInfo.NodeTag)]     = ServerStore.NodeTag;
                 json[nameof(NodeInfo.TopologyId)]  = ServerStore.GetClusterTopology(context).TopologyId;
                 json[nameof(NodeInfo.Certificate)] = ServerStore.RavenServer.ClusterCertificateHolder.CertificateForClients;
                 json[nameof(ServerStore.Engine.LastStateChangeReason)] = ServerStore.LastStateChangeReason();
             }
             context.Write(writer, json);
             writer.Flush();
         }
     return(Task.CompletedTask);
 }
예제 #12
0
        public async Task PingTest()
        {
            var dest     = GetStringQueryString("url", false) ?? GetStringQueryString("node", false);
            var topology = ServerStore.GetClusterTopology();
            var tasks    = new List <Task <PingResult> >();

            if (string.IsNullOrEmpty(dest))
            {
                foreach (var node in topology.AllNodes)
                {
                    tasks.Add(PingOnce(node.Value));
                }
            }
            else
            {
                var url = topology.GetUrlFromTag(dest);
                tasks.Add(PingOnce(url ?? dest));
            }

            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
                using (var write = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    write.WriteStartObject();
                    write.WritePropertyName("Result");

                    write.WriteStartArray();
                    while (tasks.Count > 0)
                    {
                        var task = await Task.WhenAny(tasks);

                        tasks.Remove(task);
                        context.Write(write, task.Result.ToJson());
                        if (tasks.Count > 0)
                        {
                            write.WriteComma();
                        }
                        write.Flush();
                    }
                    write.WriteEndArray();
                    write.WriteEndObject();
                    write.Flush();
                }
        }
        public async Task GetClusterWideInfoPackage()
        {
            var contentDisposition = $"attachment; filename={DateTime.UtcNow:yyyy-MM-dd H:mm:ss} Cluster Wide.zip";

            HttpContext.Response.Headers["Content-Disposition"] = contentDisposition;
            HttpContext.Response.Headers["Content-Type"]        = "application/zip";

            var token       = CreateOperationToken();
            var operationId = GetLongQueryString("operationId", false) ?? ServerStore.Operations.GetNextOperationId();

            await ServerStore.Operations.AddOperation(null, "Created debug package for all cluster nodes", Operations.Operations.OperationType.DebugPackage, async _ =>
            {
                using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
                    using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext jsonOperationContext))
                        using (transactionOperationContext.OpenReadTransaction())
                        {
                            await using (var ms = new MemoryStream())
                            {
                                using (var archive = new ZipArchive(ms, ZipArchiveMode.Create, true))
                                {
                                    var topology = ServerStore.GetClusterTopology(transactionOperationContext);

                                    foreach (var(tag, url) in topology.AllNodes)
                                    {
                                        try
                                        {
                                            await WriteDebugInfoPackageForNodeAsync(jsonOperationContext, archive, tag, url, Server.Certificate.Certificate);
                                        }
                                        catch (Exception e)
                                        {
                                            await DebugInfoPackageUtils.WriteExceptionAsZipEntryAsync(e, archive, $"Node - [{tag}]");
                                        }
                                    }
                                }

                                ms.Position = 0;
                                await ms.CopyToAsync(ResponseBodyStream(), token.Token);
                            }
                        }

                return(null);
            }, operationId, token : token);
        }
예제 #14
0
        public static OngoingTasksResult GetOngoingTasksFor(string dbName, ServerStore store)
        {
            var ongoingTasksResult = new OngoingTasksResult();

            using (store.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                DatabaseTopology dbTopology;
                ClusterTopology  clusterTopology;
                DatabaseRecord   databaseRecord;

                using (context.OpenReadTransaction())
                {
                    databaseRecord = store.Cluster.ReadDatabase(context, dbName);

                    if (databaseRecord == null)
                    {
                        return(ongoingTasksResult);
                    }

                    dbTopology      = databaseRecord.Topology;
                    clusterTopology = store.GetClusterTopology(context);

                    ongoingTasksResult.OngoingTasksList.AddRange(CollectSubscriptionTasks(context, databaseRecord, clusterTopology, store));
                }

                foreach (var tasks in new[]
                {
                    CollectExternalReplicationTasks(databaseRecord.ExternalReplication, dbTopology, clusterTopology, store),
                    CollectEtlTasks(databaseRecord, dbTopology, clusterTopology, store),
                    CollectBackupTasks(databaseRecord, dbTopology, clusterTopology, store)
                })
                {
                    ongoingTasksResult.OngoingTasksList.AddRange(tasks);
                }

                if (store.DatabasesLandlord.DatabasesCache.TryGetValue(dbName, out var database) && database.Status == TaskStatus.RanToCompletion)
                {
                    ongoingTasksResult.SubscriptionsCount = (int)database.Result.SubscriptionStorage.GetAllSubscriptionsCount();
                }

                return(ongoingTasksResult);
            }
        }
예제 #15
0
        public async Task RemoveEntryFromLog()
        {
            var index    = GetLongQueryString("index");
            var first    = GetBoolValueQueryString("first", false) ?? true;
            var nodeList = new List <string>();

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var removed = ServerStore.Engine.RemoveEntryFromRaftLog(index);
                    if (removed)
                    {
                        nodeList.Add(ServerStore.NodeTag);
                    }

                    if (first)
                    {
                        foreach (var node in ServerStore.GetClusterTopology(context).AllNodes)
                        {
                            if (node.Value == Server.WebUrl)
                            {
                                continue;
                            }

                            var cmd = new RemoveEntryFromRaftLogCommand(index);
                            using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(node.Value, Server.Certificate.Certificate))
                            {
                                await requestExecutor.ExecuteAsync(cmd, context);

                                nodeList.AddRange(cmd.Result);
                            }
                        }
                    }

                    await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                    {
                        writer.WriteStartObject();
                        writer.WriteArray("Nodes", nodeList);
                        writer.WriteEndObject();
                    }
                }
        }
예제 #16
0
        public Task GetRemoteTaskTopology()
        {
            var database        = GetStringQueryString("database");
            var databaseGroupId = GetStringQueryString("groupId");
            var remoteTask      = GetStringQueryString("remote-task");

            if (Authenticate(HttpContext, ServerStore, database, remoteTask) == false)
            {
                return(Task.CompletedTask);
            }

            List <string> nodes;

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var pullReplication = ServerStore.Cluster.ReadPullReplicationDefinition(database, remoteTask, context);
                    if (pullReplication.Disabled)
                    {
                        throw new InvalidOperationException($"The pull replication '{remoteTask}' is disabled.");
                    }

                    var topology = ServerStore.Cluster.ReadDatabaseTopology(context, database);
                    nodes = GetResponsibleNodes(topology, databaseGroupId, pullReplication);
                }

            using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
                using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                {
                    var output          = new DynamicJsonArray();
                    var clusterTopology = ServerStore.GetClusterTopology();
                    foreach (var node in nodes)
                    {
                        output.Add(clusterTopology.GetUrlFromTag(node));
                    }
                    context.Write(writer, new DynamicJsonValue
                    {
                        ["Results"] = output
                    });
                }
            return(Task.CompletedTask);
        }
예제 #17
0
        private void RedirectToLeader()
        {
            if (ServerStore.LeaderTag == null)
            {
                throw new NoLeaderException();
            }

            ClusterTopology topology;

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    topology = ServerStore.GetClusterTopology(context);
                }
            var url            = topology.GetUrlFromTag(ServerStore.LeaderTag);
            var leaderLocation = url + HttpContext.Request.Path + HttpContext.Request.QueryString;

            HttpContext.Response.StatusCode = (int)HttpStatusCode.TemporaryRedirect;
            HttpContext.Response.Headers.Remove("Content-Type");
            HttpContext.Response.Headers.Add("Location", leaderLocation);
        }
예제 #18
0
        public async Task DemoteNode()
        {
            if (ServerStore.LeaderTag == null)
            {
                NoContentStatus();
                return;
            }

            SetupCORSHeaders();

            if (ServerStore.IsLeader() == false)
            {
                RedirectToLeader();
                return;
            }

            var nodeTag = GetStringQueryString("nodeTag");

            if (nodeTag == ServerStore.LeaderTag)
            {
                throw new InvalidOperationException(
                          $"Failed to demote node {nodeTag} because {nodeTag} is the current leader in the cluster topology. In order to demote {nodeTag} perform a Step-Down first");
            }

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    var topology = ServerStore.GetClusterTopology(context);
                    if (topology.Promotables.ContainsKey(nodeTag) == false && topology.Members.ContainsKey(nodeTag) == false)
                    {
                        throw new InvalidOperationException(
                                  $"Failed to demote node {nodeTag} because {nodeTag} is not a voter in the cluster topology");
                    }

                    var url = topology.GetUrlFromTag(nodeTag);
                    await ServerStore.Engine.ModifyTopologyAsync(nodeTag, url, Leader.TopologyModification.NonVoter);

                    NoContentStatus();
                }
        }
예제 #19
0
        public OngoingTasksResult GetOngoingTasksInternal()
        {
            var ongoingTasksResult = new OngoingTasksResult();

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                DatabaseTopology dbTopology;
                ClusterTopology  clusterTopology;
                DatabaseRecord   databaseRecord;

                using (context.OpenReadTransaction())
                {
                    databaseRecord = ServerStore.Cluster.ReadDatabase(context, Database.Name);

                    if (databaseRecord == null)
                    {
                        return(ongoingTasksResult);
                    }

                    dbTopology      = databaseRecord.Topology;
                    clusterTopology = ServerStore.GetClusterTopology(context);
                    ongoingTasksResult.OngoingTasksList.AddRange(CollectSubscriptionTasks(context, databaseRecord, clusterTopology));
                }

                foreach (var tasks in new[]
                {
                    CollectExternalReplicationTasks(databaseRecord.ExternalReplication, dbTopology, clusterTopology),
                    CollectEtlTasks(databaseRecord, dbTopology, clusterTopology),
                    CollectBackupTasks(databaseRecord, dbTopology, clusterTopology)
                })
                {
                    ongoingTasksResult.OngoingTasksList.AddRange(tasks);
                }

                ongoingTasksResult.SubscriptionsCount = (int)Database.SubscriptionStorage.GetAllSubscriptionsCount();

                return(ongoingTasksResult);
            }
        }
예제 #20
0
        private static bool IsOriginAllowed(string origin, ServerStore serverStore)
        {
            if (serverStore.Server.Certificate.Certificate == null)
            {
                // running in unsafe mode - since server can be access via multiple urls/aliases accept them
                return(true);
            }

            var topology = serverStore.GetClusterTopology();

            // check explicitly each topology type to avoid allocations in topology.AllNodes
            foreach (var kvp in topology.Members)
            {
                if (kvp.Value.Equals(origin, StringComparison.OrdinalIgnoreCase))
                {
                    return(true);
                }
            }

            foreach (var kvp in topology.Watchers)
            {
                if (kvp.Value.Equals(origin, StringComparison.OrdinalIgnoreCase))
                {
                    return(true);
                }
            }

            foreach (var kvp in topology.Promotables)
            {
                if (kvp.Value.Equals(origin, StringComparison.OrdinalIgnoreCase))
                {
                    return(true);
                }
            }

            return(false);
        }
예제 #21
0
        public async Task AddNode()
        {
            SetupCORSHeaders();

            var nodeUrl       = GetQueryStringValueAndAssertIfSingleAndNotEmpty("url");
            var watcher       = GetBoolValueQueryString("watcher", false);
            var assignedCores = GetIntValueQueryString("assignedCores", false);

            if (assignedCores <= 0)
            {
                throw new ArgumentException("Assigned cores must be greater than 0!");
            }

            nodeUrl = UrlHelper.TryGetLeftPart(nodeUrl);
            var remoteIsHttps = nodeUrl.StartsWith("https:", StringComparison.OrdinalIgnoreCase);

            if (HttpContext.Request.IsHttps != remoteIsHttps)
            {
                throw new InvalidOperationException($"Cannot add node '{nodeUrl}' to cluster because it will create invalid mix of HTTPS & HTTP endpoints. A cluster must be only HTTPS or only HTTP.");
            }

            NodeInfo nodeInfo;

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
                using (var requestExecutor = ClusterRequestExecutor.CreateForSingleNode(nodeUrl, Server.Certificate.Certificate))
                {
                    requestExecutor.DefaultTimeout = ServerStore.Engine.OperationTimeout;

                    var infoCmd = new GetNodeInfoCommand();
                    try
                    {
                        await requestExecutor.ExecuteAsync(infoCmd, ctx);
                    }
                    catch (AllTopologyNodesDownException e)
                    {
                        throw new InvalidOperationException($"Couldn't contact node at {nodeUrl}", e);
                    }

                    nodeInfo = infoCmd.Result;

                    if (ServerStore.IsPassive() && nodeInfo.TopologyId != null)
                    {
                        throw new TopologyMismatchException("You can't add new node to an already existing cluster");
                    }
                }

            if (assignedCores != null && assignedCores > nodeInfo.NumberOfCores)
            {
                throw new ArgumentException("Cannot add node because the assigned cores is larger " +
                                            $"than the available cores on that machine: {nodeInfo.NumberOfCores}");
            }

            ServerStore.EnsureNotPassive();

            if (assignedCores == null)
            {
                assignedCores = ServerStore.LicenseManager.GetCoresToAssign(nodeInfo.NumberOfCores);
            }

            Debug.Assert(assignedCores <= nodeInfo.NumberOfCores);

            ServerStore.LicenseManager.AssertCanAddNode(nodeUrl, assignedCores.Value);

            if (ServerStore.IsLeader())
            {
                using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
                {
                    string          topologyId;
                    ClusterTopology clusterTopology;
                    using (ctx.OpenReadTransaction())
                    {
                        clusterTopology = ServerStore.GetClusterTopology(ctx);
                        topologyId      = clusterTopology.TopologyId;
                    }

                    var possibleNode = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
                    if (possibleNode.HasUrl)
                    {
                        throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because this url is already used by node {possibleNode.NodeTag}");
                    }

                    if (nodeInfo.ServerId == ServerStore.GetServerId())
                    {
                        throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster because it's a synonym of the current node URL:{ServerStore.GetNodeHttpServerUrl()}");
                    }

                    if (nodeInfo.TopologyId != null)
                    {
                        if (topologyId != nodeInfo.TopologyId)
                        {
                            throw new TopologyMismatchException(
                                      $"Adding a new node to cluster failed. The new node is already in another cluster. " +
                                      $"Expected topology id: {topologyId}, but we get {nodeInfo.TopologyId}");
                        }

                        if (nodeInfo.CurrentState != RachisState.Passive)
                        {
                            throw new InvalidOperationException($"Can't add a new node on {nodeUrl} to cluster " +
                                                                $"because it's already in the cluster under tag :{nodeInfo.NodeTag} " +
                                                                $"and URL: {clusterTopology.GetUrlFromTag(nodeInfo.NodeTag)}");
                        }
                    }

                    var nodeTag = nodeInfo.NodeTag == RachisConsensus.InitialTag ? null : nodeInfo.NodeTag;
                    CertificateDefinition oldServerCert = null;
                    X509Certificate2      certificate   = null;

                    if (remoteIsHttps)
                    {
                        if (nodeInfo.Certificate == null)
                        {
                            throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because it has no certificate while trying to use HTTPS");
                        }

                        certificate = new X509Certificate2(Convert.FromBase64String(nodeInfo.Certificate), (string)null, X509KeyStorageFlags.MachineKeySet);

                        var now = DateTime.UtcNow;
                        if (certificate.NotBefore.ToUniversalTime() > now)
                        {
                            // Because of time zone and time drift issues, we can't assume that the certificate generation will be
                            // proper. Because of that, we allow tolerance of the NotBefore to be a bit earlier / later than the
                            // current time. Clients may still fail to work with our certificate because of timing issues,
                            // but the admin needs to setup time sync properly and there isn't much we can do at that point
                            if ((certificate.NotBefore.ToUniversalTime() - now).TotalDays > 1)
                            {
                                throw new InvalidOperationException(
                                          $"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' is not yet valid. It starts on {certificate.NotBefore}");
                            }
                        }

                        if (certificate.NotAfter.ToUniversalTime() < now)
                        {
                            throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate '{certificate.FriendlyName}' expired on {certificate.NotAfter}");
                        }

                        var expected = GetStringQueryString("expectedThumbprint", required: false);
                        if (expected != null)
                        {
                            if (certificate.Thumbprint != expected)
                            {
                                throw new InvalidOperationException($"Cannot add node {nodeTag} with url {nodeUrl} to cluster because its certificate thumbprint '{certificate.Thumbprint}' doesn't match the expected thumbprint '{expected}'.");
                            }
                        }

                        using (ctx.OpenReadTransaction())
                        {
                            var key      = Constants.Certificates.Prefix + certificate.Thumbprint;
                            var readCert = ServerStore.Cluster.Read(ctx, key);
                            if (readCert != null)
                            {
                                oldServerCert = JsonDeserializationServer.CertificateDefinition(readCert);
                            }
                        }

                        if (oldServerCert == null)
                        {
                            var certificateDefinition = new CertificateDefinition
                            {
                                Certificate       = nodeInfo.Certificate,
                                Thumbprint        = certificate.Thumbprint,
                                NotAfter          = certificate.NotAfter,
                                Name              = "Server Certificate for " + nodeUrl,
                                SecurityClearance = SecurityClearance.ClusterNode
                            };

                            var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(Constants.Certificates.Prefix + certificate.Thumbprint, certificateDefinition));

                            await ServerStore.Cluster.WaitForIndexNotification(res.Index);
                        }
                    }

                    await ServerStore.AddNodeToClusterAsync(nodeUrl, nodeTag, validateNotInTopology : false, asWatcher : watcher ?? false);

                    using (ctx.OpenReadTransaction())
                    {
                        clusterTopology = ServerStore.GetClusterTopology(ctx);
                        possibleNode    = clusterTopology.TryGetNodeTagByUrl(nodeUrl);
                        nodeTag         = possibleNode.HasUrl ? possibleNode.NodeTag : null;

                        if (certificate != null)
                        {
                            var key = Constants.Certificates.Prefix + certificate.Thumbprint;

                            var modifiedServerCert = JsonDeserializationServer.CertificateDefinition(ServerStore.Cluster.Read(ctx, key));

                            if (modifiedServerCert == null)
                            {
                                throw new ConcurrencyException("After adding the certificate, it was removed, shouldn't happen unless another admin removed it midway through.");
                            }

                            if (oldServerCert == null)
                            {
                                modifiedServerCert.Name = "Server certificate for Node " + nodeTag;
                            }
                            else
                            {
                                var value = "Node " + nodeTag;
                                if (modifiedServerCert.Name.Contains(value) == false)
                                {
                                    modifiedServerCert.Name += ", " + value;
                                }
                            }

                            var res = await ServerStore.PutValueInClusterAsync(new PutCertificateCommand(key, modifiedServerCert));

                            await ServerStore.Cluster.WaitForIndexNotification(res.Index);
                        }

                        var nodeDetails = new NodeDetails
                        {
                            NodeTag             = nodeTag,
                            AssignedCores       = assignedCores.Value,
                            NumberOfCores       = nodeInfo.NumberOfCores,
                            InstalledMemoryInGb = nodeInfo.InstalledMemoryInGb,
                            UsableMemoryInGb    = nodeInfo.UsableMemoryInGb,
                            BuildInfo           = nodeInfo.BuildInfo
                        };
                        await ServerStore.LicenseManager.CalculateLicenseLimits(nodeDetails, forceFetchingNodeInfo : true, waitToUpdate : true);
                    }

                    NoContentStatus();
                    return;
                }
            }
            RedirectToLeader();
        }
예제 #22
0
        public Task GetOngoingTaskInfo()
        {
            if (ResourceNameValidator.IsValidResourceName(Database.Name, ServerStore.Configuration.Core.DataDirectory.FullPath, out string errorMessage) == false)
            {
                throw new BadRequestException(errorMessage);
            }

            var key     = GetLongQueryString("key");
            var typeStr = GetQueryStringValueAndAssertIfSingleAndNotEmpty("type");

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                using (context.OpenReadTransaction())
                {
                    var clusterTopology = ServerStore.GetClusterTopology(context);
                    var record          = ServerStore.Cluster.ReadDatabase(context, Database.Name);
                    var dbTopology      = record?.Topology;

                    if (Enum.TryParse <OngoingTaskType>(typeStr, true, out var type) == false)
                    {
                        throw new ArgumentException($"Unknown task type: {type}", "type");
                    }

                    string tag;

                    switch (type)
                    {
                    case OngoingTaskType.Replication:

                        var watcher = record?.ExternalReplication.Find(x => x.TaskId == key);
                        if (watcher == null)
                        {
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                            break;
                        }
                        var taskInfo = GetExternalReplicationInfo(dbTopology, clusterTopology, watcher);

                        WriteResult(context, taskInfo);

                        break;

                    case OngoingTaskType.Backup:

                        var backupConfiguration = record?.PeriodicBackups?.Find(x => x.TaskId == key);
                        if (backupConfiguration == null)
                        {
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                            break;
                        }

                        tag = dbTopology?.WhoseTaskIsIt(backupConfiguration, ServerStore.Engine.CurrentState);
                        var backupDestinations = GetBackupDestinations(backupConfiguration);
                        var backupStatus       = Database.PeriodicBackupRunner.GetBackupStatus(key);
                        var nextBackup         = Database.PeriodicBackupRunner.GetNextBackupDetails(record, backupConfiguration, backupStatus);

                        var backupTaskInfo = new OngoingTaskBackup
                        {
                            TaskId          = backupConfiguration.TaskId,
                            BackupType      = backupConfiguration.BackupType,
                            TaskName        = backupConfiguration.Name,
                            TaskState       = backupConfiguration.Disabled ? OngoingTaskState.Disabled : OngoingTaskState.Enabled,
                            ResponsibleNode = new NodeId
                            {
                                NodeTag = tag,
                                NodeUrl = clusterTopology.GetUrlFromTag(tag)
                            },
                            BackupDestinations    = backupDestinations,
                            LastFullBackup        = backupStatus.LastFullBackup,
                            LastIncrementalBackup = backupStatus.LastIncrementalBackup,
                            NextBackup            = nextBackup
                        };

                        WriteResult(context, backupTaskInfo);
                        break;

                    case OngoingTaskType.SqlEtl:

                        var sqlEtl = record?.SqlEtls?.Find(x => x.TaskId == key);
                        if (sqlEtl == null)
                        {
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                            break;
                        }

                        WriteResult(context, new OngoingTaskSqlEtlDetails()
                        {
                            TaskId        = sqlEtl.TaskId,
                            TaskName      = sqlEtl.Name,
                            Configuration = sqlEtl,
                            TaskState     = GetEtlTaskState(sqlEtl)
                        });
                        break;

                    case OngoingTaskType.RavenEtl:

                        var ravenEtl = record?.RavenEtls?.Find(x => x.TaskId == key);
                        if (ravenEtl == null)
                        {
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                            break;
                        }

                        WriteResult(context, new OngoingTaskRavenEtlDetails()
                        {
                            TaskId        = ravenEtl.TaskId,
                            TaskName      = ravenEtl.Name,
                            Configuration = ravenEtl,
                            TaskState     = GetEtlTaskState(ravenEtl)
                        });
                        break;

                    case OngoingTaskType.Subscription:

                        var nameKey = GetQueryStringValueAndAssertIfSingleAndNotEmpty("taskName");
                        var itemKey = SubscriptionState.GenerateSubscriptionItemKeyName(record.DatabaseName, nameKey);
                        var doc     = ServerStore.Cluster.Read(context, itemKey);
                        if (doc == null)
                        {
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                            break;
                        }

                        var subscriptionState = JsonDeserializationClient.SubscriptionState(doc);
                        tag = dbTopology?.WhoseTaskIsIt(subscriptionState, ServerStore.Engine.CurrentState);

                        var subscriptionStateInfo = new SubscriptionStateWithNodeDetails
                        {
                            Query = subscriptionState.Query,
                            ChangeVectorForNextBatchStartingPoint = subscriptionState.ChangeVectorForNextBatchStartingPoint,
                            SubscriptionId           = subscriptionState.SubscriptionId,
                            SubscriptionName         = subscriptionState.SubscriptionName,
                            LastBatchAckTime         = subscriptionState.LastBatchAckTime,
                            Disabled                 = subscriptionState.Disabled,
                            LastClientConnectionTime = subscriptionState.LastClientConnectionTime,
                            MentorNode               = subscriptionState.MentorNode,
                            ResponsibleNode          = new NodeId
                            {
                                NodeTag = tag,
                                NodeUrl = clusterTopology.GetUrlFromTag(tag)
                            }
                        };

                        // Todo: here we'll need to talk with the running node? TaskConnectionStatus = subscriptionState.Disabled ? OngoingTaskConnectionStatus.NotActive : OngoingTaskConnectionStatus.Active,

                        WriteResult(context, subscriptionStateInfo.ToJson());
                        break;

                    default:
                        HttpContext.Response.StatusCode = (int)HttpStatusCode.NotFound;
                        break;
                    }
                }
            }

            return(Task.CompletedTask);
        }
예제 #23
0
        private void WriteDatabaseInfo(string databaseName, BlittableJsonReaderObject dbRecordBlittable,
                                       TransactionOperationContext context, BlittableJsonTextWriter writer)
        {
            var online = ServerStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out Task <DocumentDatabase> dbTask) &&
                         dbTask != null &&
                         dbTask.IsCompleted;

            // Check for exceptions
            if (dbTask != null && dbTask.IsFaulted)
            {
                WriteFaultedDatabaseInfo(context, writer, dbTask, databaseName);
                return;
            }

            var dbRecord = JsonDeserializationCluster.DatabaseRecord(dbRecordBlittable);
            var db       = online ? dbTask.Result : null;

            var indexingStatus = db?.IndexStore.Status ?? IndexRunningStatus.Running;

            // Looking for disabled indexing flag inside the database settings for offline database status
            if (dbRecord.Settings.TryGetValue(RavenConfiguration.GetKey(x => x.Indexing.Disabled), out var val) && val == "true")
            {
                indexingStatus = IndexRunningStatus.Disabled;
            }
            var disabled        = dbRecord.Disabled;
            var topology        = dbRecord.Topology;
            var clusterTopology = ServerStore.GetClusterTopology(context);

            var nodesTopology = new NodesTopology();

            if (topology != null)
            {
                foreach (var member in topology.Members)
                {
                    var url  = clusterTopology.GetUrlFromTag(member);
                    var node = new InternalReplication
                    {
                        Database = databaseName,
                        NodeTag  = member,
                        Url      = url
                    };
                    nodesTopology.Members.Add(GetNodeId(node));
                    SetNodeStatus(topology, member, nodesTopology);
                }

                foreach (var promotable in topology.Promotables)
                {
                    var node   = GetNode(databaseName, clusterTopology, promotable, out var promotableTask);
                    var mentor = topology.WhoseTaskIsIt(promotableTask, ServerStore.IsPassive());
                    nodesTopology.Promotables.Add(GetNodeId(node, mentor));
                    SetNodeStatus(topology, promotable, nodesTopology);
                }

                foreach (var rehab in topology.Rehabs)
                {
                    var node   = GetNode(databaseName, clusterTopology, rehab, out var promotableTask);
                    var mentor = topology.WhoseTaskIsIt(promotableTask, ServerStore.IsPassive());
                    nodesTopology.Rehabs.Add(GetNodeId(node, mentor));
                    SetNodeStatus(topology, rehab, nodesTopology);
                }
            }

            if (online == false)
            {
                // If state of database is found in the cache we can continue
                if (ServerStore.DatabaseInfoCache.TryWriteOfflineDatabaseStatusToRequest(
                        context, writer, databaseName, disabled, indexingStatus, nodesTopology))
                {
                    return;
                }
                // We won't find it if it is a new database or after a dirty shutdown, so just report empty values then
            }

            var size = new Size(GetTotalSize(db));

            var databaseInfo = new DatabaseInfo
            {
                Name      = databaseName,
                Disabled  = disabled,
                TotalSize = size,

                IsAdmin     = true, //TODO: implement me!
                IsEncrypted = dbRecord.Encrypted,
                UpTime      = online ? (TimeSpan?)GetUptime(db) : null,
                BackupInfo  = GetBackupInfo(db),

                Alerts         = db?.NotificationCenter.GetAlertCount() ?? 0,
                RejectClients  = false, //TODO: implement me!
                LoadError      = null,
                IndexingErrors = db?.IndexStore.GetIndexes().Sum(index => index.GetErrorCount()) ?? 0,

                DocumentsCount             = db?.DocumentsStorage.GetNumberOfDocuments() ?? 0,
                HasRevisionsConfiguration  = db?.DocumentsStorage.RevisionsStorage.Configuration != null,
                HasExpirationConfiguration = db?.ExpiredDocumentsCleaner != null,
                IndexesCount   = db?.IndexStore.GetIndexes().Count() ?? 0,
                IndexingStatus = indexingStatus,

                NodesTopology            = nodesTopology,
                ReplicationFactor        = topology?.ReplicationFactor ?? -1,
                DynamicNodesDistribution = topology?.DynamicNodesDistribution ?? false
            };

            var doc = databaseInfo.ToJson();

            context.Write(writer, doc);
        }
예제 #24
0
        public async Task GetTopology()
        {
            var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");
            var applicationIdentifier = GetStringQueryString("applicationIdentifier", required: false);

            if (applicationIdentifier != null)
            {
                AlertIfDocumentStoreCreationRateIsNotReasonable(applicationIdentifier, name);
            }

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                using (context.OpenReadTransaction())
                    using (var rawRecord = ServerStore.Cluster.ReadRawDatabaseRecord(context, name))
                    {
                        if (await CanAccessDatabaseAsync(name, requireAdmin: false) == false)
                        {
                            return;
                        }

                        if (rawRecord == null)
                        {
                            // here we return 503 so clients will try to failover to another server
                            // if this is a newly created db that we haven't been notified about it yet
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
                            HttpContext.Response.Headers["Database-Missing"] = name;
                            await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                            {
                                context.Write(writer,
                                              new DynamicJsonValue
                                {
                                    ["Type"]    = "Error",
                                    ["Message"] = "Database " + name + " wasn't found"
                                });
                            }
                            return;
                        }

                        var clusterTopology = ServerStore.GetClusterTopology(context);
                        if (ServerStore.IsPassive() && clusterTopology.TopologyId != null)
                        {
                            // we were kicked-out from the cluster
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
                            return;
                        }

                        if (rawRecord.Topology.Members.Count == 0 && rawRecord.Topology.Rehabs.Count == 0 && rawRecord.DeletionInProgress.Any())
                        {
                            // The database at deletion progress from all nodes
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
                            HttpContext.Response.Headers["Database-Missing"] = name;
                            await using (var writer = new AsyncBlittableJsonTextWriter(context, HttpContext.Response.Body))
                            {
                                context.Write(writer, new DynamicJsonValue
                                {
                                    ["Type"]    = "Error",
                                    ["Message"] = "Database " + name + " was deleted"
                                });
                            }

                            return;
                        }

                        clusterTopology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext);

                        await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
                        {
                            context.Write(writer, new DynamicJsonValue
                            {
                                [nameof(Topology.Nodes)] = new DynamicJsonArray(
                                    rawRecord.Topology.Members.Select(x => new DynamicJsonValue
                                {
                                    [nameof(ServerNode.Url)]        = GetUrl(x, clusterTopology),
                                    [nameof(ServerNode.ClusterTag)] = x,
                                    [nameof(ServerNode.ServerRole)] = ServerNode.Role.Member,
                                    [nameof(ServerNode.Database)]   = rawRecord.DatabaseName
                                })
                                    .Concat(rawRecord.Topology.Rehabs.Select(x => new DynamicJsonValue
                                {
                                    [nameof(ServerNode.Url)]        = GetUrl(x, clusterTopology),
                                    [nameof(ServerNode.ClusterTag)] = x,
                                    [nameof(ServerNode.Database)]   = rawRecord.DatabaseName,
                                    [nameof(ServerNode.ServerRole)] = ServerNode.Role.Rehab
                                })
                                            )
                                    ),
                                [nameof(Topology.Etag)] = rawRecord.Topology.Stamp?.Index ?? -1
                            });
                        }
                    }
            }
        }
        public async Task GetClusterWideInfoPackage()
        {
            var contentDisposition = $"attachment; filename={DateTime.UtcNow:yyyy-MM-dd H:mm:ss} Cluster Wide.zip";

            HttpContext.Response.Headers["Content-Disposition"] = contentDisposition;

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext transactionOperationContext))
                using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext jsonOperationContext))
                    using (transactionOperationContext.OpenReadTransaction())
                    {
                        using (var ms = new MemoryStream())
                        {
                            using (var archive = new ZipArchive(ms, ZipArchiveMode.Create, true))
                            {
                                var localEndpointClient = new LocalEndpointClient(Server);

                                using (var localMemoryStream = new MemoryStream())
                                {
                                    //assuming that if the name tag is empty
                                    var nodeName = $"Node - [{ServerStore.NodeTag ?? "Empty node tag"}]";

                                    using (var localArchive = new ZipArchive(localMemoryStream, ZipArchiveMode.Create, true))
                                    {
                                        await WriteServerWide(localArchive, jsonOperationContext, localEndpointClient, _serverWidePrefix);
                                        await WriteForAllLocalDatabases(localArchive, jsonOperationContext, localEndpointClient);
                                        await WriteLogFile(localArchive);
                                    }

                                    localMemoryStream.Position = 0;
                                    var entry = archive.CreateEntry($"{nodeName}.zip");
                                    entry.ExternalAttributes = ((int)(FilePermissions.S_IRUSR | FilePermissions.S_IWUSR)) << 16;

                                    using (var entryStream = entry.Open())
                                    {
                                        localMemoryStream.CopyTo(entryStream);
                                        entryStream.Flush();
                                    }
                                }
                                var databaseNames = ServerStore.Cluster.GetDatabaseNames(transactionOperationContext);
                                var topology      = ServerStore.GetClusterTopology(transactionOperationContext);

                                //this means no databases are defined in the cluster
                                //in this case just output server-wide endpoints from all cluster nodes
                                if (databaseNames.Count == 0)
                                {
                                    foreach (var tagWithUrl in topology.AllNodes)
                                    {
                                        if (tagWithUrl.Value.Contains(ServerStore.GetNodeHttpServerUrl()))
                                        {
                                            continue;
                                        }

                                        try
                                        {
                                            await WriteDebugInfoPackageForNodeAsync(
                                                jsonOperationContext,
                                                archive,
                                                tag : tagWithUrl.Key,
                                                url : tagWithUrl.Value,
                                                certificate : Server.Certificate.Certificate,
                                                databaseNames : null);
                                        }
                                        catch (Exception e)
                                        {
                                            var entryName = $"Node - [{tagWithUrl.Key}]";
                                            DebugInfoPackageUtils.WriteExceptionAsZipEntry(e, archive, entryName);
                                        }
                                    }
                                }
                                else
                                {
                                    var nodeUrlToDatabaseNames = CreateUrlToDatabaseNamesMapping(transactionOperationContext, databaseNames);
                                    foreach (var urlToDatabaseNamesMap in nodeUrlToDatabaseNames)
                                    {
                                        if (urlToDatabaseNamesMap.Key.Contains(ServerStore.GetNodeHttpServerUrl()))
                                        {
                                            continue; //skip writing local data, we do it separately
                                        }
                                        try
                                        {
                                            await WriteDebugInfoPackageForNodeAsync(
                                                jsonOperationContext,
                                                archive,
                                                tag : urlToDatabaseNamesMap.Value.Item2,
                                                url : urlToDatabaseNamesMap.Key,
                                                databaseNames : urlToDatabaseNamesMap.Value.Item1,
                                                certificate : Server.Certificate.Certificate);
                                        }
                                        catch (Exception e)
                                        {
                                            var entryName = $"Node - [{urlToDatabaseNamesMap.Value.Item2}]";
                                            DebugInfoPackageUtils.WriteExceptionAsZipEntry(e, archive, entryName);
                                        }
                                    }
                                }
                            }

                            ms.Position = 0;
                            await ms.CopyToAsync(ResponseBodyStream());
                        }
                    }
        }
예제 #26
0
        public async Task ApplyCommand()
        {
            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                if (ServerStore.IsLeader() == false)
                {
                    throw new NoLeaderException("Not a leader, cannot accept commands.");
                }

                HttpContext.Response.Headers["Reached-Leader"] = "true";

                var commandJson = await context.ReadForMemoryAsync(RequestBodyStream(), "external/rachis/command");

                CommandBase command;
                try
                {
                    command = CommandBase.CreateFrom(commandJson);
                }
                catch (InvalidOperationException e)
                {
                    RequestRouter.AssertClientVersion(HttpContext, e);
                    throw;
                }

                switch (command)
                {
                case AddDatabaseCommand addDatabase:
                    if (addDatabase.Record.Topology.Count == 0)
                    {
                        ServerStore.AssignNodesToDatabase(ServerStore.GetClusterTopology(), addDatabase.Record);
                    }
                    break;

                case AddOrUpdateCompareExchangeBatchCommand batchCmpExchange:
                    batchCmpExchange.ContextToWriteResult = context;
                    break;

                case CompareExchangeCommandBase cmpExchange:
                    cmpExchange.ContextToWriteResult = context;
                    break;
                }

                var isClusterAdmin = IsClusterAdmin();
                command.VerifyCanExecuteCommand(ServerStore, context, isClusterAdmin);

                var(etag, result) = await ServerStore.Engine.PutAsync(command);

                HttpContext.Response.StatusCode = (int)HttpStatusCode.OK;
                var ms = context.CheckoutMemoryStream();
                try
                {
                    using (var writer = new BlittableJsonTextWriter(context, ms))
                    {
                        context.Write(writer, new DynamicJsonValue
                        {
                            [nameof(ServerStore.PutRaftCommandResult.RaftCommandIndex)] = etag,
                            [nameof(ServerStore.PutRaftCommandResult.Data)]             = result
                        });
                        writer.Flush();
                    }
                    // now that we know that we properly serialized it
                    ms.Position = 0;
                    await ms.CopyToAsync(ResponseBodyStream());
                }
                finally
                {
                    context.ReturnMemoryStream(ms);
                }
            }
        }
예제 #27
0
        protected RestoreBackupTaskBase(ServerStore serverStore,
                                        RestoreBackupConfigurationBase restoreFromConfiguration,
                                        string nodeTag,
                                        OperationCancelToken operationCancelToken)
        {
            _serverStore             = serverStore;
            RestoreFromConfiguration = restoreFromConfiguration;
            _nodeTag = nodeTag;
            _operationCancelToken = operationCancelToken;

            var dataDirectoryThatWillBeUsed = string.IsNullOrWhiteSpace(RestoreFromConfiguration.DataDirectory) ?
                                              _serverStore.Configuration.Core.DataDirectory.FullPath :
                                              new PathSetting(RestoreFromConfiguration.DataDirectory, _serverStore.Configuration.Core.DataDirectory.FullPath).FullPath;

            if (ResourceNameValidator.IsValidResourceName(RestoreFromConfiguration.DatabaseName, dataDirectoryThatWillBeUsed, out string errorMessage) == false)
            {
                throw new InvalidOperationException(errorMessage);
            }

            _serverStore.EnsureNotPassive();

            ClusterTopology clusterTopology;

            using (_serverStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
                using (context.OpenReadTransaction())
                {
                    if (_serverStore.Cluster.DatabaseExists(context, RestoreFromConfiguration.DatabaseName))
                    {
                        throw new ArgumentException($"Cannot restore data to an existing database named {RestoreFromConfiguration.DatabaseName}");
                    }

                    clusterTopology = _serverStore.GetClusterTopology(context);
                }

            _hasEncryptionKey = string.IsNullOrWhiteSpace(RestoreFromConfiguration.EncryptionKey) == false;
            if (_hasEncryptionKey)
            {
                var key = Convert.FromBase64String(RestoreFromConfiguration.EncryptionKey);
                if (key.Length != 256 / 8)
                {
                    throw new InvalidOperationException($"The size of the key must be 256 bits, but was {key.Length * 8} bits.");
                }

                if (AdminDatabasesHandler.NotUsingHttps(clusterTopology.GetUrlFromTag(_serverStore.NodeTag)))
                {
                    throw new InvalidOperationException("Cannot restore an encrypted database to a node which doesn't support SSL!");
                }
            }

            var backupEncryptionSettings = RestoreFromConfiguration.BackupEncryptionSettings;

            if (backupEncryptionSettings != null)
            {
                if (backupEncryptionSettings.EncryptionMode == EncryptionMode.UseProvidedKey &&
                    backupEncryptionSettings.Key == null)
                {
                    throw new InvalidOperationException($"{nameof(BackupEncryptionSettings.EncryptionMode)} is set to {nameof(EncryptionMode.UseProvidedKey)} but an encryption key wasn't provided");
                }


                if (backupEncryptionSettings.EncryptionMode != EncryptionMode.UseProvidedKey &&
                    backupEncryptionSettings.Key != null)
                {
                    throw new InvalidOperationException($"{nameof(BackupEncryptionSettings.EncryptionMode)} is set to {backupEncryptionSettings.EncryptionMode} but an encryption key was provided");
                }
            }

            var hasRestoreDataDirectory = string.IsNullOrWhiteSpace(RestoreFromConfiguration.DataDirectory) == false;

            if (hasRestoreDataDirectory &&
                HasFilesOrDirectories(dataDirectoryThatWillBeUsed))
            {
                throw new ArgumentException("New data directory must be empty of any files or folders, " +
                                            $"path: {dataDirectoryThatWillBeUsed}");
            }

            if (hasRestoreDataDirectory == false)
            {
                RestoreFromConfiguration.DataDirectory = GetDataDirectory();
            }

            _restoringToDefaultDataDirectory = IsDefaultDataDirectory(RestoreFromConfiguration.DataDirectory, RestoreFromConfiguration.DatabaseName);
        }
예제 #28
0
        private void WriteDatabaseInfo(string databaseName, BlittableJsonReaderObject dbRecordBlittable,
                                       TransactionOperationContext context, AbstractBlittableJsonTextWriter writer)
        {
            try
            {
                var online = ServerStore.DatabasesLandlord.DatabasesCache.TryGetValue(databaseName, out Task <DocumentDatabase> dbTask) &&
                             dbTask != null &&
                             dbTask.IsCompleted;

                // Check for exceptions
                if (dbTask != null && dbTask.IsFaulted)
                {
                    var exception = dbTask.Exception.ExtractSingleInnerException();
                    WriteFaultedDatabaseInfo(databaseName, exception, context, writer);
                    return;
                }

                var dbRecord = JsonDeserializationCluster.DatabaseRecord(dbRecordBlittable);
                var db       = online ? dbTask.Result : null;

                var indexingStatus = db?.IndexStore?.Status ?? IndexRunningStatus.Running;
                // Looking for disabled indexing flag inside the database settings for offline database status
                if (dbRecord.Settings.TryGetValue(RavenConfiguration.GetKey(x => x.Indexing.Disabled), out var val) && bool.TryParse(val, out var indexingDisabled) && indexingDisabled)
                {
                    indexingStatus = IndexRunningStatus.Disabled;
                }

                var disabled        = dbRecord.Disabled;
                var topology        = dbRecord.Topology;
                var clusterTopology = ServerStore.GetClusterTopology(context);
                clusterTopology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext);

                var studioEnvironment = StudioConfiguration.StudioEnvironment.None;
                if (dbRecord.Studio != null && !dbRecord.Studio.Disabled)
                {
                    studioEnvironment = dbRecord.Studio.Environment;
                }

                var nodesTopology = new NodesTopology();

                var statuses = ServerStore.GetNodesStatuses();
                if (topology != null)
                {
                    foreach (var member in topology.Members)
                    {
                        var url  = clusterTopology.GetUrlFromTag(member);
                        var node = new InternalReplication
                        {
                            Database = databaseName,
                            NodeTag  = member,
                            Url      = url
                        };
                        nodesTopology.Members.Add(GetNodeId(node));
                        SetNodeStatus(topology, member, nodesTopology, statuses);
                    }

                    foreach (var promotable in topology.Promotables)
                    {
                        topology.PredefinedMentors.TryGetValue(promotable, out var mentorCandidate);
                        var node   = GetNode(databaseName, clusterTopology, promotable, mentorCandidate, out var promotableTask);
                        var mentor = topology.WhoseTaskIsIt(ServerStore.Engine.CurrentState, promotableTask, null);
                        nodesTopology.Promotables.Add(GetNodeId(node, mentor));
                        SetNodeStatus(topology, promotable, nodesTopology, statuses);
                    }

                    foreach (var rehab in topology.Rehabs)
                    {
                        var node   = GetNode(databaseName, clusterTopology, rehab, null, out var promotableTask);
                        var mentor = topology.WhoseTaskIsIt(ServerStore.Engine.CurrentState, promotableTask, null);
                        nodesTopology.Rehabs.Add(GetNodeId(node, mentor));
                        SetNodeStatus(topology, rehab, nodesTopology, statuses);
                    }
                }

                if (online == false)
                {
                    // if state of database is found in the cache we can continue
                    if (ServerStore.DatabaseInfoCache.TryGet(databaseName, databaseInfoJson =>
                    {
                        databaseInfoJson.Modifications = new DynamicJsonValue(databaseInfoJson)
                        {
                            [nameof(DatabaseInfo.Disabled)] = disabled,
                            [nameof(DatabaseInfo.IndexingStatus)] = indexingStatus.ToString(),
                            [nameof(DatabaseInfo.NodesTopology)] = nodesTopology.ToJson(),
                            [nameof(DatabaseInfo.DeletionInProgress)] = DynamicJsonValue.Convert(dbRecord.DeletionInProgress),
                            [nameof(DatabaseInfo.Environment)] = studioEnvironment
                        };

                        context.Write(writer, databaseInfoJson);
                    }))
                    {
                        return;
                    }

                    // we won't find it if it is a new database or after a dirty shutdown,
                    // so just report empty values then
                }

                var size = db?.GetSizeOnDisk() ?? (new Size(0), new Size(0));

                var databaseInfo = new DatabaseInfo
                {
                    Name            = databaseName,
                    Disabled        = disabled,
                    TotalSize       = size.Data,
                    TempBuffersSize = size.TempBuffers,

                    IsAdmin     = true,
                    IsEncrypted = dbRecord.Encrypted,
                    UpTime      = online ? (TimeSpan?)GetUptime(db) : null,
                    BackupInfo  = GetBackupInfo(db),

                    Alerts         = db?.NotificationCenter.GetAlertCount() ?? 0,
                    RejectClients  = false,
                    LoadError      = null,
                    IndexingErrors = db?.IndexStore?.GetIndexes()?.Sum(index => index.GetErrorCount()) ?? 0,

                    DocumentsCount             = db?.DocumentsStorage.GetNumberOfDocuments() ?? 0,
                    HasRevisionsConfiguration  = db?.DocumentsStorage.RevisionsStorage.Configuration != null,
                    HasExpirationConfiguration = db?.ExpiredDocumentsCleaner != null,
                    IndexesCount   = db?.IndexStore?.GetIndexes()?.Count() ?? 0,
                    IndexingStatus = indexingStatus,
                    Environment    = studioEnvironment,

                    NodesTopology            = nodesTopology,
                    ReplicationFactor        = topology?.ReplicationFactor ?? -1,
                    DynamicNodesDistribution = topology?.DynamicNodesDistribution ?? false,
                    DeletionInProgress       = dbRecord.DeletionInProgress
                };

                var doc = databaseInfo.ToJson();
                context.Write(writer, doc);
            }
            catch (Exception e)
            {
                if (Logger.IsInfoEnabled)
                {
                    Logger.Info($"Failed to get database info for: {databaseName}", e);
                }

                WriteFaultedDatabaseInfo(databaseName, e, context, writer);
            }
        }
예제 #29
0
        public Task GetTopology()
        {
            var name = GetQueryStringValueAndAssertIfSingleAndNotEmpty("name");

            using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
            {
                var dbId = Constants.Documents.Prefix + name;
                using (context.OpenReadTransaction())
                    using (var dbBlit = ServerStore.Cluster.Read(context, dbId, out long _))
                    {
                        if (TryGetAllowedDbs(name, out var _, requireAdmin: false) == false)
                        {
                            return(Task.CompletedTask);
                        }

                        if (dbBlit == null)
                        {
                            // here we return 503 so clients will try to failover to another server
                            // if this is a newly created db that we haven't been notified about it yet
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
                            HttpContext.Response.Headers["Database-Missing"] = name;
                            using (var writer = new BlittableJsonTextWriter(context, HttpContext.Response.Body))
                            {
                                context.Write(writer,
                                              new DynamicJsonValue
                                {
                                    ["Type"]    = "Error",
                                    ["Message"] = "Database " + name + " wasn't found"
                                });
                            }
                            return(Task.CompletedTask);
                        }

                        var clusterTopology = ServerStore.GetClusterTopology(context);
                        if (ServerStore.IsPassive() && clusterTopology.TopologyId != null)
                        {
                            // we were kicked-out from the cluster
                            HttpContext.Response.StatusCode = (int)HttpStatusCode.ServiceUnavailable;
                            return(Task.CompletedTask);
                        }

                        clusterTopology.ReplaceCurrentNodeUrlWithClientRequestedNodeUrlIfNecessary(ServerStore, HttpContext);

                        var dbRecord = JsonDeserializationCluster.DatabaseRecord(dbBlit);
                        using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
                        {
                            context.Write(writer, new DynamicJsonValue
                            {
                                [nameof(Topology.Nodes)] = new DynamicJsonArray(
                                    dbRecord.Topology.Members.Select(x => new DynamicJsonValue
                                {
                                    [nameof(ServerNode.Url)]        = GetUrl(x, clusterTopology),
                                    [nameof(ServerNode.ClusterTag)] = x,
                                    [nameof(ServerNode.ServerRole)] = ServerNode.Role.Member,
                                    [nameof(ServerNode.Database)]   = dbRecord.DatabaseName
                                })
                                    .Concat(dbRecord.Topology.Rehabs.Select(x => new DynamicJsonValue
                                {
                                    [nameof(ServerNode.Url)]        = GetUrl(x, clusterTopology),
                                    [nameof(ServerNode.ClusterTag)] = x,
                                    [nameof(ServerNode.Database)]   = dbRecord.DatabaseName,
                                    [nameof(ServerNode.ServerRole)] = ServerNode.Role.Rehab
                                })
                                            )
                                    ),
                                [nameof(Topology.Etag)] = dbRecord.Topology.Stamp?.Index ?? -1
                            });
                        }
                    }
            }
            return(Task.CompletedTask);
        }
예제 #30
0
 public override void Initialize(ServerStore serverStore, TransactionOperationContext context)
 {
     ClusterNodes = serverStore.GetClusterTopology(context).AllNodes.Keys.ToArray();
 }