public StreamApiRequestSender(
[NotNull] IClusterProvider clusterProvider,
[NotNull] ILog log,
[NotNull] BufferPool bufferPool,
[CanBeNull] ClusterClientSetup additionalSetup)
{
this.log = log;
this.bufferPool = bufferPool;
client = ClusterClientFactory.Create(
clusterProvider,
log,
Constants.ServiceNames.StreamApi,
config =>
{
config.SetupUniversalTransport(
new UniversalTransportSettings
{
BufferFactory = bufferPool.Rent
});
config.AddResponseTransform(TryDecompress);
additionalSetup?.Invoke(config);
});
responseAnalyzer = new ResponseAnalyzer(ResponseAnalysisContext.Stream);
compressionEnabled = LZ4Helper.Enabled;
}
public RequestContextTuner(IRequestContext requestContext)
{
fallbackClusterProvider = requestContext.ClusterProvider;
fallbackReplicaOrdering = requestContext.ReplicaOrdering;
fallbackConnectionAttempts = requestContext.ConnectionAttempts;
fallbackRequestStrategy = requestContext.Parameters.Strategy;
}
public void TestSetup()
{
replica1 = new Uri("http://replica1");
replica2 = new Uri("http://replica2");
response1 = new Response(ResponseCode.Ok);
response2 = new Response(ResponseCode.Ok);
selectedResponse = new Response(ResponseCode.Ok);
result1 = new ReplicaResult(replica1, response1, ResponseVerdict.DontKnow, TimeSpan.Zero);
result2 = new ReplicaResult(replica2, response2, ResponseVerdict.DontKnow, TimeSpan.Zero);
var parameters = RequestParameters.Empty
.WithStrategy(Substitute.For <IRequestStrategy>());
clusterProvider = Substitute.For <IClusterProvider>();
clusterProvider.GetCluster().Returns(new[] { replica1, replica2 });
replicaOrdering = Substitute.For <IReplicaOrdering>();
replicaOrdering.Order(null, null, null, null).ReturnsForAnyArgs(info => info.Arg <IList <Uri> >().Reverse());
context = new RequestContext(
Request.Get("foo/bar"),
parameters,
Budget.Infinite,
new SilentLog(),
clusterProvider,
replicaOrdering,
transport: default,
public RequestContext(
Request request,
RequestParameters parameters,
IRequestTimeBudget budget,
ILog log,
IClusterProvider clusterProvider,
IReplicaOrdering replicaOrdering,
ITransport transport,
int maximumReplicasToUse,
int connectionAttempts,
string clientApplicationName = null,
CancellationToken cancellationToken = default)
{
Request = request;
Budget = budget;
Log = log;
ClusterProvider = clusterProvider;
ReplicaOrdering = replicaOrdering;
Transport = transport;
Parameters = parameters;
CancellationToken = cancellationToken;
MaximumReplicasToUse = maximumReplicasToUse;
ConnectionAttempts = connectionAttempts;
ClientApplicationName = clientApplicationName;
ResetReplicaResults();
}
public StreamBinaryWriterSettings(
[NotNull] Func <string> apiKeyProvider,
[NotNull] IClusterProvider gateCluster)
{
ApiKeyProvider = apiKeyProvider ?? throw new ArgumentNullException(nameof(apiKeyProvider));
GateCluster = gateCluster ?? throw new ArgumentNullException(nameof(gateCluster));
}
public static void AddApiKey(string adminApiKey, string apiKey, IClusterProvider managementApiTopology, ILog log)
{
var client = new ClusterClient(
log,
configuration =>
{
configuration.ClusterProvider = managementApiTopology;
configuration.Transport = new UniversalTransport(log);
});
var task = client.SendAsync(
Request.Post("/rules/set")
.WithHeader("apiKey", adminApiKey)
.WithAdditionalQueryParameter("key", apiKey)
.WithAdditionalQueryParameter("pattern", "*")
.WithAdditionalQueryParameter("rights", "rwm"),
15.Seconds()
);
var result = task.GetAwaiter().GetResult();
if (result.Status != ClusterResultStatus.Success || result.Response.Code != ResponseCode.Ok)
{
throw new Exception($"Failed to add api key '{apiKey}'. Server responded with {result.Response.Code}: '{result.Response.Content}'.");
}
}
public void SetUp()
{
replica1 = new Uri("http://replica1");
replica2 = new Uri("http://replica2");
response1 = new Response(ResponseCode.Ok);
response2 = new Response(ResponseCode.Ok);
selectedResponse = new Response(ResponseCode.Ok);
result1 = new ReplicaResult(replica1, response1, ResponseVerdict.DontKnow, TimeSpan.Zero);
result2 = new ReplicaResult(replica2, response2, ResponseVerdict.DontKnow, TimeSpan.Zero);
var log = new ConsoleLog();
context = new RequestContext(Request.Get("foo/bar"), Substitute.For <IRequestStrategy>(), Budget.Infinite, log, CancellationToken.None, null, int.MaxValue);
// ReSharper disable AssignNullToNotNullAttribute
context.Strategy.SendAsync(null, null, null, null, 0, default(CancellationToken))
// ReSharper restore AssignNullToNotNullAttribute
.ReturnsForAnyArgs(
async info =>
{
var replicas = info.Arg <IEnumerable <Uri> >();
var sender = info.Arg <IRequestSender>();
foreach (var replica in replicas)
{
await sender.SendToReplicaAsync(replica, context.Request, TimeSpan.Zero, CancellationToken.None);
}
});
clusterProvider = Substitute.For <IClusterProvider>();
clusterProvider.GetCluster().Returns(new[] { replica1, replica2 });
requestSender = Substitute.For <IRequestSender>();
requestSender.SendToReplicaAsync(replica1, Arg.Any <Request>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).ReturnsTask(_ => result1);
requestSender.SendToReplicaAsync(replica2, Arg.Any <Request>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).ReturnsTask(_ => result2);
replicaOrdering = Substitute.For <IReplicaOrdering>();
// ReSharper disable AssignNullToNotNullAttribute
replicaOrdering.Order(null, null, null).ReturnsForAnyArgs(info => info.Arg <IList <Uri> >().Reverse());
// ReSharper restore AssignNullToNotNullAttribute
responseSelector = Substitute.For <IResponseSelector>();
// ReSharper disable once AssignNullToNotNullAttribute
responseSelector.Select(null).ReturnsForAnyArgs(_ => selectedResponse);
resultStatusSelector = Substitute.For <IClusterResultStatusSelector>();
// ReSharper disable AssignNullToNotNullAttribute
resultStatusSelector.Select(null, null).ReturnsForAnyArgs(ClusterResultStatus.Success);
// ReSharper restore AssignNullToNotNullAttribute
storageProvider = Substitute.For <IReplicaStorageProvider>();
module = new RequestExecutionModule(
clusterProvider,
replicaOrdering,
responseSelector,
storageProvider,
requestSender,
resultStatusSelector);
}
public static void Bootstrap(IFrameworkProvider frameworkProvider, IClusterProvider clusterProvider, string serviceName, string version)
{
_frameworkProvider = frameworkProvider;
var uri = _frameworkProvider.Start(serviceName, version);
var serviceId = serviceName + Guid.NewGuid();
_clusterProvider = clusterProvider;
_clusterProvider.RegisterServiceAsync(serviceName, serviceId, version, uri);
}
public IVostokHerculesSinkBuilder SetClusterProvider(IClusterProvider clusterProvider)
{
instance = null;
clusterProviderBuilder = ClusterProviderBuilder.FromValue(clusterProvider ?? throw new ArgumentNullException(nameof(clusterProvider)));
return(this);
}
public static void Bootstrap(IFrameworkProvider frameworkProvider, IClusterProvider clusterProvider, string serviceName, string version)
{
_frameworkProvider = frameworkProvider;
var uri = _frameworkProvider.Start(serviceName, version);
var serviceId = serviceName + Guid.NewGuid();
_clusterProvider = clusterProvider;
_clusterProvider.RegisterServiceAsync(serviceName, serviceId, version, uri).Wait();
}
public BucketProvider(IClusterProvider clusterProvider)
{
if (clusterProvider == null)
{
throw new ArgumentNullException(nameof(clusterProvider));
}
_clusterProvider = clusterProvider;
}
public NLogInitializer(IClusterProvider clusterProvider, IBucketProvider bucketProvider,
IOptions <CouchBaseLogging> setupAction, IConfiguration configuration,
ILogConfigAppService logConfigAppService)
{
_clusterProvider = clusterProvider;
_bucketProvider = bucketProvider;
_configuration = configuration;
_logConfigAppService = logConfigAppService;
_setupAction = setupAction.Value;
}
public void TestSetup()
{
replica1 = new Uri("http://replica1");
replica2 = new Uri("http://replica2");
baseProvider = Substitute.For <IClusterProvider>();
baseProvider.GetCluster().Returns(new[] { replica1, replica2 });
provider = new RepeatingClusterProvider(baseProvider, 3);
}
public HomeController(IBucketProvider bucketProvider,
IClusterProvider clusterProvider,
ITravelSample travelSample,
IMembucket membucket)
{
_bucketProvider = bucketProvider;
_clusterProvider = clusterProvider;
_travelSample = travelSample;
_membucket = membucket;
}
public DatabaseService(
IClusterProvider clusterProvider,
IBucketProvider bucketProvider,
IOptions <CouchbaseConfig> options,
ILogger <DatabaseService> logger)
{
_clusterProvider = clusterProvider;
_bucketProvider = bucketProvider;
_couchbaseConfig = options.Value;
_logger = logger;
}
public async static Task <ServiceInformation> FindServiceInstanceAsync(this IClusterProvider self, string serviceName)
{
var res = await self.FindServiceInstancesAsync(serviceName);
if (res.Length == 0)
{
throw new Exception("Could not find service");
}
return(res[ThreadLocalRandom.Current.Next(0, res.Length - 1)]);
}
private static ClusterConfig GetClusterConfig(
IClusterProvider clusterProvider,
IIdentityLookup identityLookup
)
{
var helloProps = Props.FromProducer(() => new WorkerActor());
return(ClusterConfig
.Setup("mycluster", clusterProvider, identityLookup)
.WithClusterKind("hello", helloProps));
}
public void TestSetup()
{
replica1 = new Uri("http://replica1");
replica2 = new Uri("http://replica2");
response1 = new Response(ResponseCode.Ok);
response2 = new Response(ResponseCode.Ok);
selectedResponse = new Response(ResponseCode.Ok);
result1 = new ReplicaResult(replica1, response1, ResponseVerdict.DontKnow, TimeSpan.Zero);
result2 = new ReplicaResult(replica2, response2, ResponseVerdict.DontKnow, TimeSpan.Zero);
var parameters = RequestParameters.Empty
.WithStrategy(Substitute.For <IRequestStrategy>());
context = new RequestContext(Request.Get("foo/bar"), parameters, Budget.Infinite, new SilentLog(), null, int.MaxValue);
context.Parameters.Strategy.SendAsync(null, null, null, null, null, 0, default)
.ReturnsForAnyArgs(
async info =>
{
var replicas = info.Arg <IEnumerable <Uri> >();
var sender = info.Arg <IRequestSender>();
foreach (var replica in replicas)
{
await sender.SendToReplicaAsync(replica, context.Request, null, TimeSpan.Zero, CancellationToken.None);
}
});
clusterProvider = Substitute.For <IClusterProvider>();
clusterProvider.GetCluster().Returns(new[] { replica1, replica2 });
requestSender = Substitute.For <IRequestSenderInternal>();
requestSender.SendToReplicaAsync(Arg.Any <ITransport>(), replica1, Arg.Any <Request>(), Arg.Any <TimeSpan?>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).ReturnsTask(_ => result1);
requestSender.SendToReplicaAsync(Arg.Any <ITransport>(), replica2, Arg.Any <Request>(), Arg.Any <TimeSpan?>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).ReturnsTask(_ => result2);
replicaOrdering = Substitute.For <IReplicaOrdering>();
replicaOrdering.Order(null, null, null, null).ReturnsForAnyArgs(info => info.Arg <IList <Uri> >().Reverse());
responseSelector = Substitute.For <IResponseSelector>();
responseSelector.Select(null, null, null).ReturnsForAnyArgs(_ => selectedResponse);
resultStatusSelector = Substitute.For <IClusterResultStatusSelector>();
resultStatusSelector.Select(null, null).ReturnsForAnyArgs(ClusterResultStatus.Success);
storageProvider = Substitute.For <IReplicaStorageProvider>();
module = new RequestExecutionModule(
clusterProvider,
replicaOrdering,
responseSelector,
storageProvider,
requestSender,
resultStatusSelector);
}
public ProfileController(
IClusterProvider clusterProvider,
IBucketProvider bucketProvider,
IOptions <CouchbaseConfig> options,
ILogger <ProfileController> logger)
{
_clusterProvider = clusterProvider;
_bucketProvider = bucketProvider;
_logger = logger;
_couchbaseConfig = options.Value;
}
public ClusterConfig(string name, string address, int port, IClusterProvider cp)
{
Name = name ?? throw new ArgumentNullException(nameof(name));
Address = address ?? throw new ArgumentNullException(nameof(address));
Port = port;
ClusterProvider = cp ?? throw new ArgumentNullException(nameof(cp));
RemoteConfig = new RemoteConfig();
TimeoutTimespan = TimeSpan.FromSeconds(5);
MemberStatusValueSerializer = new NullMemberStatusValueSerializer();
MemberStrategyBuilder = kind => new SimpleMemberStrategy();
}
public static void Start(string clusterName, string address, IClusterProvider provider)
{
var(h, p) = ParseAddress(address);
var kinds = RemotingSystem.GetKnownKinds();
SubscribePartitionKindsToEventStream();
SpawnPidCacheActor();
SpawnMembershipActor();
SubscribeMembershipActorToEventStream();
provider.RegisterMember(clusterName, h, p, kinds);
provider.MonitorMemberStatusChanges();
}
public GateRequestSender(
[NotNull] IClusterProvider clusterProvider,
[NotNull] ILog log,
[NotNull] BufferPool bufferPool,
[CanBeNull] ClusterClientSetup additionalSetup)
{
this.log = log;
this.bufferPool = bufferPool;
client = ClusterClientFactory.Create(clusterProvider, log, Constants.ServiceNames.Gate, additionalSetup);
responseAnalyzer = new ResponseAnalyzer(ResponseAnalysisContext.Stream);
compressionEnabled = LZ4Helper.Enabled;
}
public RepeatingClusterProvider(IClusterProvider provider, int repeatCount)
{
if (provider == null)
{
throw new ArgumentNullException(nameof(provider));
}
if (repeatCount <= 0)
{
throw new ArgumentOutOfRangeException(nameof(repeatCount), "Repeat count must be positive.");
}
cache = new CachingTransform <IList <Uri>, IList <Uri> >(x => Repeat(x, repeatCount), provider.GetCluster);
}
public static void Start(string clusterName, IClusterProvider provider)
{
_logger.LogInformation("Starting Proto.Actor cluster");
var(h, p) = ParseAddress(ProcessRegistry.Instance.Address);
var kinds = Remote.Remote.GetKnownKinds();
Partition.SpawnPartitionActors(kinds);
Partition.SubscribeToEventStream();
PidCache.Spawn();
MemberList.Spawn();
MemberList.SubscribeToEventStream();
provider.RegisterMemberAsync(clusterName, h, p, kinds).Wait();
provider.MonitorMemberStatusChanges();
_logger.LogInformation("Cluster started");
}
public static void Bootstrap(IFrameworkProvider frameworkProvider, IClusterProvider clusterProvider, string serviceName, string version)
{
try
{
_frameworkProvider = frameworkProvider;
var uri = _frameworkProvider.Start(serviceName, version);
var serviceId = serviceName + Guid.NewGuid();
_clusterProvider = clusterProvider;
_clusterProvider.RegisterServiceAsync(serviceName, serviceId, version, uri).Wait();
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
public RepeatingClusterProvider(IClusterProvider provider, int repeatCount)
{
if (provider == null)
{
throw new ArgumentNullException(nameof(provider));
}
if (repeatCount <= 0)
{
throw new ArgumentOutOfRangeException(nameof(repeatCount), "Repeat count must be positive.");
}
this.provider = provider;
this.repeatCount = repeatCount;
}
public RequestExecutionModule(
IClusterProvider clusterProvider,
IReplicaOrdering replicaOrdering,
IResponseSelector responseSelector,
IReplicaStorageProvider storageProvider,
IRequestSender requestSender,
IClusterResultStatusSelector resultStatusSelector)
{
this.clusterProvider = clusterProvider;
this.replicaOrdering = replicaOrdering;
this.responseSelector = responseSelector;
this.storageProvider = storageProvider;
this.requestSender = requestSender;
this.resultStatusSelector = resultStatusSelector;
}
public BatchesStreamConsumerSettings(
[NotNull] string streamName,
[NotNull] Func <string> apiKeyProvider,
[NotNull] IClusterProvider streamApiCluster,
[NotNull] Func <IBinaryBufferReader, IHerculesEventBuilder <T> > eventBuilderProvider,
[NotNull] IStreamCoordinatesStorage coordinatesStorage,
[NotNull] Func <StreamShardingSettings> shardingSettingsProvider)
{
StreamName = streamName ?? throw new ArgumentNullException(nameof(streamName));
ApiKeyProvider = apiKeyProvider ?? throw new ArgumentNullException(nameof(apiKeyProvider));
StreamApiCluster = streamApiCluster ?? throw new ArgumentNullException(nameof(streamApiCluster));
EventBuilderProvider = eventBuilderProvider ?? throw new ArgumentNullException(nameof(eventBuilderProvider));
CoordinatesStorage = coordinatesStorage ?? throw new ArgumentNullException(nameof(coordinatesStorage));
ShardingSettingsProvider = shardingSettingsProvider ?? throw new ArgumentNullException(nameof(shardingSettingsProvider));
}
public ClusterModule(IClusterProvider provider)
{
_provider = provider;
Post["/Align/Cluster/Match"] = _ =>
{
var data = (Match[])NancyExtensionToPost.GetData <Match[]>(Request.Query);
return(Response.AsJson(_provider.GetMatches(data)));
};
Post["/Align/Cluster/IsReverse"] = _ =>
{
var data = (Match[])NancyExtensionToPost.GetData <Match[]>(Request.Query);
return(Response.AsJson(_provider.IsReverseQueryDirection(data)));
};
}
private HerculesSinkSettings BuildHerculesSettings(IClusterProvider cluster, BuildContext context)
{
// NOTE(tsup): Do not inline this statements to get rid of closure of BuildContext
var formattedServiceName = context.ApplicationIdentity.FormatServiceName();
var tracer = context.Tracer;
// Note(kungurtsev): allow null api key provider, streams can be configured later.
return(new HerculesSinkSettings(cluster, apiKeyProvider ?? (() => null))
{
AdditionalSetup = setup =>
{
setup.ClientApplicationName = formattedServiceName;
setup.SetupDistributedTracing(tracer);
}
});
}
public ProtoClusterRxBroker(IClusterProvider clusterProvider)
{
Props props = Actor.FromProducer(() => new BrokerMasterActor());
Remote.RegisterKnownKind(nameof(BrokerMasterActor), props);
Serialization.RegisterFileDescriptor(BrokerReflection.Descriptor);
Remote.Start("127.0.0.1", PortUtils.FindAvailablePort());
//Cluster.Start("MyCluster", clusterProvider);
//PID brokerMasterPid = Cluster.GetAsync("Master", nameof(BrokerMasterActor)).Result;
//Cluster.GetAsync("Master2", nameof(BrokerMasterActor)).Wait();
//Cluster.GetAsync("Master" + Guid.NewGuid(), nameof(BrokerMasterActor)).Wait();
//props = Actor.FromProducer(() => new BrokerActor(brokerMasterPid));
//_brokerPid = Actor.SpawnNamed(props, "Broker");
}
public static void BootstrapClient(IClusterProvider clusterProvider)
{
_clusterProvider = clusterProvider;
_clusterProvider.BootstrapClientAsync().Wait();
}
