public RavenEtl(Transformation transformation, RavenEtlConfiguration configuration, DocumentDatabase database, ServerStore serverStore) : base(transformation, configuration, database, serverStore, RavenEtlTag) { Metrics = new EtlMetricsCountersManager(); if (configuration.TestMode == false) { _requestExecutor = RequestExecutor.Create(configuration.Connection.TopologyDiscoveryUrls, configuration.Connection.Database, serverStore.Server.Certificate.Certificate, DocumentConventions.Default); } _script = new RavenEtlDocumentTransformer.ScriptInput(transformation); }
private static RequestExecutor CreateNewRequestExecutor(RavenEtlConfiguration configuration, ServerStore serverStore) { var certificate = serverStore.Server.Certificate.Certificate; if (certificate != null && configuration.UsingEncryptedCommunicationChannel() == false && configuration.AllowEtlOnNonEncryptedChannel) { // we're running on HTTPS but sending data to non encrypted server // let's not provide the server certificate so we won't fail on request executor's URL validation certificate = null; } return(RequestExecutor.Create(configuration.Connection.TopologyDiscoveryUrls, configuration.Connection.Database, certificate, DocumentConventions.Default)); }
private TcpConnectionInfo GetConnectionInfo(ReplicationNode node, bool external) { var shutdownInfo = new ConnectionShutdownInfo { Node = node, External = external }; _outgoingFailureInfo.TryAdd(node, shutdownInfo); try { if (node is ExternalReplication exNode) { using (var requestExecutor = RequestExecutor.Create(exNode.ConnectionString.TopologyDiscoveryUrls, exNode.ConnectionString.Database, _server.Server.Certificate.Certificate, DocumentConventions.Default)) using (_server.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx)) { var database = exNode.ConnectionString.Database; var cmd = new GetTcpInfoCommand("extrenal-replication", database); requestExecutor.Execute(cmd, ctx); node.Database = database; node.Url = requestExecutor.Url; return(cmd.Result); } } if (node is InternalReplication internalNode) { using (var cts = new CancellationTokenSource(_server.Engine.TcpConnectionTimeout)) { return(ReplicationUtils.GetTcpInfo(internalNode.Url, internalNode.Database, "Replication", _server.Server.Certificate.Certificate, cts.Token)); } } throw new InvalidOperationException( $"Unexpected replication node type, Expected to be '{typeof(ExternalReplication)}' or '{typeof(InternalReplication)}', but got '{node.GetType()}'"); } catch (Exception e) { // will try to fetch it again later if (_log.IsInfoEnabled) { _log.Info($"Failed to fetch tcp connection information for the destination '{node.FromString()}' , the connection will be retried later.", e); } _reconnectQueue.TryAdd(shutdownInfo); } return(null); }
public void Admin_databases_endpoint_should_fetch_document_with_etag_in_metadata_property() { using (var store = GetDocumentStore()) { TransactionOperationContext context; using (Server.ServerStore.ContextPool.AllocateOperationContext(out context)) { var command = new GetDatabaseDocumentTestCommand(); using (var requestExecutor = RequestExecutor.Create(store.Urls, store.Database, null, DocumentConventions.Default)) { requestExecutor.Execute(command, context); } var hasEtag = command.Result.TryGet("Etag", out long etag); Assert.True(hasEtag && etag != 0, $"{hasEtag} - {etag}"); } } }
public void Admin_databases_endpoint_should_refuse_document_with_lower_etag_with_concurrency_Exception() { using (var store = GetDocumentStore()) { using (var session = store.OpenSession()) { session.Load <object>("users/1");// just waiting for the db to load } TransactionOperationContext context; using (Server.ServerStore.ContextPool.AllocateOperationContext(out context)) { var getCommand = new GetDatabaseDocumentTestCommand(); using (var requestExecutor = RequestExecutor.Create(store.Urls, store.Database, null, DocumentConventions.Default)) { requestExecutor.Execute(getCommand, context); using (var putCommand = new PutDatabaseDocumentTestCommand(getCommand.Result)) { Assert.Throws <ConcurrencyException>(() => requestExecutor.Execute(putCommand, context)); } } } } }
public async Task Round_robin_load_balancing_with_failing_node_should_work() { var databaseName = "Round_robin_load_balancing_should_work" + Guid.NewGuid(); var leader = await CreateRaftClusterAndGetLeader(3); var followers = Servers.Where(x => x.ServerStore.IsLeader() == false).ToArray(); var conventionsForLoadBalancing = new DocumentConventions { ReadBalanceBehavior = ReadBalanceBehavior.RoundRobin }; using (var leaderStore = new DocumentStore { Urls = new[] { leader.WebUrl }, Database = databaseName, Conventions = conventionsForLoadBalancing }) using (var follower1 = new DocumentStore { Urls = new[] { followers[0].WebUrl }, Database = databaseName, Conventions = conventionsForLoadBalancing }) using (var follower2 = new DocumentStore { Urls = new[] { followers[1].WebUrl }, Database = databaseName, Conventions = conventionsForLoadBalancing }) using (var context = JsonOperationContext.ShortTermSingleUse()) { leaderStore.Initialize(); follower1.Initialize(); follower2.Initialize(); var(index, _) = await CreateDatabaseInCluster(databaseName, 3, leader.WebUrl); await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(30)); var leaderRequestExecutor = leaderStore.GetRequestExecutor(); //wait until all nodes in database cluster are members (and not promotables) //GetTopologyCommand -> does not retrieve promotables var topology = new Topology(); while (topology.Nodes?.Count != 3) { var topologyGetCommand = new GetTopologyCommand(); await leaderRequestExecutor.ExecuteAsync(topologyGetCommand, context); topology = topologyGetCommand.Result; Thread.Sleep(50); } foreach (var server in Servers) { await server.ServerStore.Cluster.WaitForIndexNotification(index); } using (var session = leaderStore.OpenSession()) { session.Store(new User { Name = "John Dow" }); session.Store(new User { Name = "Jack Dow" }); session.Store(new User { Name = "Jane Dow" }); session.Store(new User { Name = "FooBar" }, "marker"); session.SaveChanges(); await WaitForDocumentInClusterAsync <User>(session as DocumentSession, "marker", x => true, leader.ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan); } var requestExecutor = RequestExecutor.Create(follower1.Urls, databaseName, null, follower1.Conventions); do //make sure there are three nodes in the topology { await Task.Delay(100); } while (requestExecutor.TopologyNodes == null); DisposeServerAndWaitForFinishOfDisposal(leader); var failedRequests = new HashSet <(string, Exception)>(); requestExecutor.FailedRequest += (url, e) => failedRequests.Add((url, e)); using (var tmpContext = JsonOperationContext.ShortTermSingleUse()) { for (var sessionId = 0; sessionId < 5; sessionId++) { requestExecutor.Cache.Clear(); //make sure we do not use request cache await requestExecutor.ExecuteAsync(new GetStatisticsCommand(), tmpContext, CancellationToken.None, new SessionInfo(sessionId, false)); } } } }
private static RequestExecutor CreateNewRequestExecutor(RavenEtlConfiguration configuration, ServerStore serverStore) { return(RequestExecutor.Create(configuration.Connection.TopologyDiscoveryUrls, configuration.Connection.Database, serverStore.Server.Certificate.Certificate, DocumentConventions.Default)); }