public RavenEtl(Transformation transformation, RavenEtlConfiguration configuration, DocumentDatabase database, ServerStore serverStore) : base(transformation, configuration, database, serverStore, RavenEtlTag)
{
Metrics = new EtlMetricsCountersManager();
if (configuration.TestMode == false)
{
_requestExecutor = RequestExecutor.Create(configuration.Connection.TopologyDiscoveryUrls, configuration.Connection.Database, serverStore.Server.Certificate.Certificate, DocumentConventions.Default);
}
_script = new RavenEtlDocumentTransformer.ScriptInput(transformation);
}
private static RequestExecutor CreateNewRequestExecutor(RavenEtlConfiguration configuration, ServerStore serverStore)
{
var certificate = serverStore.Server.Certificate.Certificate;
if (certificate != null && configuration.UsingEncryptedCommunicationChannel() == false && configuration.AllowEtlOnNonEncryptedChannel)
{
// we're running on HTTPS but sending data to non encrypted server
// let's not provide the server certificate so we won't fail on request executor's URL validation
certificate = null;
}
return(RequestExecutor.Create(configuration.Connection.TopologyDiscoveryUrls, configuration.Connection.Database, certificate, DocumentConventions.Default));
}
private TcpConnectionInfo GetConnectionInfo(ReplicationNode node, bool external)
{
var shutdownInfo = new ConnectionShutdownInfo
{
Node = node,
External = external
};
_outgoingFailureInfo.TryAdd(node, shutdownInfo);
try
{
if (node is ExternalReplication exNode)
{
using (var requestExecutor = RequestExecutor.Create(exNode.ConnectionString.TopologyDiscoveryUrls, exNode.ConnectionString.Database,
_server.Server.Certificate.Certificate,
DocumentConventions.Default))
using (_server.ContextPool.AllocateOperationContext(out TransactionOperationContext ctx))
{
var database = exNode.ConnectionString.Database;
var cmd = new GetTcpInfoCommand("extrenal-replication", database);
requestExecutor.Execute(cmd, ctx);
node.Database = database;
node.Url = requestExecutor.Url;
return(cmd.Result);
}
}
if (node is InternalReplication internalNode)
{
using (var cts = new CancellationTokenSource(_server.Engine.TcpConnectionTimeout))
{
return(ReplicationUtils.GetTcpInfo(internalNode.Url, internalNode.Database, "Replication", _server.Server.Certificate.Certificate, cts.Token));
}
}
throw new InvalidOperationException(
$"Unexpected replication node type, Expected to be '{typeof(ExternalReplication)}' or '{typeof(InternalReplication)}', but got '{node.GetType()}'");
}
catch (Exception e)
{
// will try to fetch it again later
if (_log.IsInfoEnabled)
{
_log.Info($"Failed to fetch tcp connection information for the destination '{node.FromString()}' , the connection will be retried later.", e);
}
_reconnectQueue.TryAdd(shutdownInfo);
}
return(null);
}
public void Admin_databases_endpoint_should_fetch_document_with_etag_in_metadata_property()
{
using (var store = GetDocumentStore())
{
TransactionOperationContext context;
using (Server.ServerStore.ContextPool.AllocateOperationContext(out context))
{
var command = new GetDatabaseDocumentTestCommand();
using (var requestExecutor = RequestExecutor.Create(store.Urls, store.Database, null, DocumentConventions.Default))
{
requestExecutor.Execute(command, context);
}
var hasEtag = command.Result.TryGet("Etag", out long etag);
Assert.True(hasEtag && etag != 0, $"{hasEtag} - {etag}");
}
}
}
public void Admin_databases_endpoint_should_refuse_document_with_lower_etag_with_concurrency_Exception()
{
using (var store = GetDocumentStore())
{
using (var session = store.OpenSession())
{
session.Load <object>("users/1");// just waiting for the db to load
}
TransactionOperationContext context;
using (Server.ServerStore.ContextPool.AllocateOperationContext(out context))
{
var getCommand = new GetDatabaseDocumentTestCommand();
using (var requestExecutor = RequestExecutor.Create(store.Urls, store.Database, null, DocumentConventions.Default))
{
requestExecutor.Execute(getCommand, context);
using (var putCommand = new PutDatabaseDocumentTestCommand(getCommand.Result))
{
Assert.Throws <ConcurrencyException>(() => requestExecutor.Execute(putCommand, context));
}
}
}
}
}
public async Task Round_robin_load_balancing_with_failing_node_should_work()
{
var databaseName = "Round_robin_load_balancing_should_work" + Guid.NewGuid();
var leader = await CreateRaftClusterAndGetLeader(3);
var followers = Servers.Where(x => x.ServerStore.IsLeader() == false).ToArray();
var conventionsForLoadBalancing = new DocumentConventions
{
ReadBalanceBehavior = ReadBalanceBehavior.RoundRobin
};
using (var leaderStore = new DocumentStore
{
Urls = new[] { leader.WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower1 = new DocumentStore
{
Urls = new[] { followers[0].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var follower2 = new DocumentStore
{
Urls = new[] { followers[1].WebUrl },
Database = databaseName,
Conventions = conventionsForLoadBalancing
})
using (var context = JsonOperationContext.ShortTermSingleUse())
{
leaderStore.Initialize();
follower1.Initialize();
follower2.Initialize();
var(index, _) = await CreateDatabaseInCluster(databaseName, 3, leader.WebUrl);
await WaitForRaftIndexToBeAppliedInCluster(index, TimeSpan.FromSeconds(30));
var leaderRequestExecutor = leaderStore.GetRequestExecutor();
//wait until all nodes in database cluster are members (and not promotables)
//GetTopologyCommand -> does not retrieve promotables
var topology = new Topology();
while (topology.Nodes?.Count != 3)
{
var topologyGetCommand = new GetTopologyCommand();
await leaderRequestExecutor.ExecuteAsync(topologyGetCommand, context);
topology = topologyGetCommand.Result;
Thread.Sleep(50);
}
foreach (var server in Servers)
{
await server.ServerStore.Cluster.WaitForIndexNotification(index);
}
using (var session = leaderStore.OpenSession())
{
session.Store(new User {
Name = "John Dow"
});
session.Store(new User {
Name = "Jack Dow"
});
session.Store(new User {
Name = "Jane Dow"
});
session.Store(new User {
Name = "FooBar"
}, "marker");
session.SaveChanges();
await WaitForDocumentInClusterAsync <User>(session as DocumentSession,
"marker", x => true,
leader.ServerStore.Configuration.Cluster.OperationTimeout.AsTimeSpan);
}
var requestExecutor = RequestExecutor.Create(follower1.Urls, databaseName, null, follower1.Conventions);
do //make sure there are three nodes in the topology
{
await Task.Delay(100);
} while (requestExecutor.TopologyNodes == null);
DisposeServerAndWaitForFinishOfDisposal(leader);
var failedRequests = new HashSet <(string, Exception)>();
requestExecutor.FailedRequest += (url, e) => failedRequests.Add((url, e));
using (var tmpContext = JsonOperationContext.ShortTermSingleUse())
{
for (var sessionId = 0; sessionId < 5; sessionId++)
{
requestExecutor.Cache.Clear(); //make sure we do not use request cache
await requestExecutor.ExecuteAsync(new GetStatisticsCommand(), tmpContext, CancellationToken.None, new SessionInfo(sessionId, false));
}
}
}
}
private static RequestExecutor CreateNewRequestExecutor(RavenEtlConfiguration configuration, ServerStore serverStore)
{
return(RequestExecutor.Create(configuration.Connection.TopologyDiscoveryUrls, configuration.Connection.Database, serverStore.Server.Certificate.Certificate, DocumentConventions.Default));
}
