/// <inheritdoc />
public void Modify(Uri replica, IList <Uri> allReplicas, IReplicaStorageProvider storageProvider, Request request, RequestParameters parameters, ref double weight)
{
if (storageProvider.Obtain <THealth>(storageKey).TryGetValue(replica, out var currentHealth))
{
implementation.ModifyWeight(currentHealth, ref weight);
}
}
public void SetUp()
{
settings = new RelativeWeightSettings()
{
WeightUpdatePeriod = 200.Milliseconds(),
PenaltyMultiplier = 100,
InitialWeight = 1,
StatisticSmoothingConstant = 100.Milliseconds(),
WeightsDownSmoothingConstant = 100.Milliseconds(),
WeightsRaiseSmoothingConstant = 100.Milliseconds(),
WeightsTTL = 5.Minutes(),
MinWeight = 0.005,
Sensitivity = 3
};
clusterState = new ClusterState(
timeProvider: Substitute.For <ITimeProvider>(),
rawClusterStatistic: Substitute.For <IRawClusterStatistic>(),
statisticHistory: Substitute.For <IStatisticHistory>(),
relativeWeightCalculator: Substitute.For <IRelativeWeightCalculator>(),
weightsNormalizer: Substitute.For <IWeightsNormalizer>(),
weights: Substitute.For <IWeights>());
replicaStorageProvider = Substitute.For <IReplicaStorageProvider>();
storageProvider = Substitute.For <IGlobalStorageProvider>();
storageProvider.ObtainGlobalValue(Arg.Any <string>(), Arg.Any <Func <ClusterState> >())
.Returns(info => clusterState);
relativeWeightModifier = new RelativeWeightModifier(settings, "srv", "env", 0, 1, storageProvider);
}
/// <inheritdoc />
public void Learn(ReplicaResult result, IReplicaStorageProvider storageProvider)
{
foreach (var modifier in modifiers)
{
modifier.Learn(result, storageProvider);
}
}
public void SetUp()
{
request = Request.Get("foo/bar");
replicas = new[]
{
replica1 = new Uri("http://replica-1"),
replica2 = new Uri("http://replica-2"),
replica3 = new Uri("http://replica-3"),
replica4 = new Uri("http://replica-4"),
replica5 = new Uri("http://replica-5")
};
modifiers = new List <IReplicaWeightModifier>
{
Substitute.For <IReplicaWeightModifier>(),
Substitute.For <IReplicaWeightModifier>(),
Substitute.For <IReplicaWeightModifier>()
};
storageProvider = Substitute.For <IReplicaStorageProvider>();
weightCalculator = Substitute.For <IReplicaWeightCalculator>();
// ReSharper disable AssignNullToNotNullAttribute
weightCalculator.GetWeight(null, null, null, null).ReturnsForAnyArgs(1.0);
// ReSharper restore AssignNullToNotNullAttribute
ordering = new WeighedReplicaOrdering(modifiers, weightCalculator);
}
public void SetUp()
{
replica1 = new Uri("http://replica1");
replica2 = new Uri("http://replica2");
response1 = new Response(ResponseCode.Ok);
response2 = new Response(ResponseCode.Ok);
selectedResponse = new Response(ResponseCode.Ok);
result1 = new ReplicaResult(replica1, response1, ResponseVerdict.DontKnow, TimeSpan.Zero);
result2 = new ReplicaResult(replica2, response2, ResponseVerdict.DontKnow, TimeSpan.Zero);
var log = new ConsoleLog();
context = new RequestContext(Request.Get("foo/bar"), Substitute.For <IRequestStrategy>(), Budget.Infinite, log, CancellationToken.None, null, int.MaxValue);
// ReSharper disable AssignNullToNotNullAttribute
context.Strategy.SendAsync(null, null, null, null, 0, default(CancellationToken))
// ReSharper restore AssignNullToNotNullAttribute
.ReturnsForAnyArgs(
async info =>
{
var replicas = info.Arg <IEnumerable <Uri> >();
var sender = info.Arg <IRequestSender>();
foreach (var replica in replicas)
{
await sender.SendToReplicaAsync(replica, context.Request, TimeSpan.Zero, CancellationToken.None);
}
});
clusterProvider = Substitute.For <IClusterProvider>();
clusterProvider.GetCluster().Returns(new[] { replica1, replica2 });
requestSender = Substitute.For <IRequestSender>();
requestSender.SendToReplicaAsync(replica1, Arg.Any <Request>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).ReturnsTask(_ => result1);
requestSender.SendToReplicaAsync(replica2, Arg.Any <Request>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).ReturnsTask(_ => result2);
replicaOrdering = Substitute.For <IReplicaOrdering>();
// ReSharper disable AssignNullToNotNullAttribute
replicaOrdering.Order(null, null, null).ReturnsForAnyArgs(info => info.Arg <IList <Uri> >().Reverse());
// ReSharper restore AssignNullToNotNullAttribute
responseSelector = Substitute.For <IResponseSelector>();
// ReSharper disable once AssignNullToNotNullAttribute
responseSelector.Select(null).ReturnsForAnyArgs(_ => selectedResponse);
resultStatusSelector = Substitute.For <IClusterResultStatusSelector>();
// ReSharper disable AssignNullToNotNullAttribute
resultStatusSelector.Select(null, null).ReturnsForAnyArgs(ClusterResultStatus.Success);
// ReSharper restore AssignNullToNotNullAttribute
storageProvider = Substitute.For <IReplicaStorageProvider>();
module = new RequestExecutionModule(
clusterProvider,
replicaOrdering,
responseSelector,
storageProvider,
requestSender,
resultStatusSelector);
}
public void Modify(Uri replica, IList <Uri> allReplicas, IReplicaStorageProvider storageProvider, Request request, ref double weight)
{
DateTime lastGrayTimestamp;
var storage = storageProvider.Obtain <DateTime>(storageKey);
if (!storage.TryGetValue(replica, out lastGrayTimestamp))
{
return;
}
var currentTime = timeProvider.GetCurrentTime();
var grayPeriod = grayPeriodProvider.GetGrayPeriod();
if (lastGrayTimestamp + grayPeriod >= currentTime)
{
weight = 0.0;
}
else
{
if (storage.Remove(replica, lastGrayTimestamp))
{
LogReplicaIsNoLongerGray(replica);
}
}
}
public double GetWeight(Uri replica, IList <Uri> allReplicas, IReplicaStorageProvider storageProvider, Request request, RequestParameters parameters)
{
var weight = initialWeight;
foreach (var modifier in modifiers)
{
modifier.Modify(replica, allReplicas, storageProvider, request, parameters, ref weight);
if (weight < minimumWeight)
{
weight = minimumWeight;
}
if (weight > maximumWeight)
{
weight = maximumWeight;
}
}
if (double.IsNaN(weight))
{
weight = minimumWeight;
}
return(weight);
}
/// <inheritdoc />
public void Learn(ReplicaResult result, IReplicaStorageProvider storageProvider)
{
if (result.Verdict != ResponseVerdict.Reject)
{
return;
}
if (result.Response.Code == ResponseCode.StreamReuseFailure ||
result.Response.Code == ResponseCode.StreamInputFailure ||
result.Response.Code == ResponseCode.ContentInputFailure ||
result.Response.Code == ResponseCode.ContentReuseFailure)
{
return;
}
var storage = storageProvider.Obtain <DateTime>(StorageKey);
var wasGray = storage.ContainsKey(result.Replica);
storage[result.Replica] = getCurrentTime();
if (!wasGray)
{
LogReplicaIsGrayNow(result.Replica);
}
}
public void Learn(ReplicaResult result, IReplicaStorageProvider storageProvider)
{
for (var index = 0; index < modifiers.Count; index++)
{
modifiers[index].Learn(result, storageProvider);
}
}
public void TestSetup()
{
request = Request.Get("foo/bar");
parameters = RequestParameters.Empty;
replicas = new[]
{
replica1 = new Uri("http://replica-1"),
replica2 = new Uri("http://replica-2"),
replica3 = new Uri("http://replica-3"),
replica4 = new Uri("http://replica-4"),
replica5 = new Uri("http://replica-5")
};
modifiers = new List <IReplicaWeightModifier>
{
Substitute.For <IReplicaWeightModifier>(),
Substitute.For <IReplicaWeightModifier>(),
Substitute.For <IReplicaWeightModifier>()
};
storageProvider = Substitute.For <IReplicaStorageProvider>();
weightCalculator = Substitute.For <IReplicaWeightCalculator>();
weightCalculator.GetWeight(null, null, null, null, null).ReturnsForAnyArgs(1.0);
ordering = new WeighedReplicaOrdering(modifiers, weightCalculator);
}
public void SetUp()
{
replica = new Uri("http://replica/");
relativeRequest = Request.Get("foo/bar");
absoluteRequest = Request.Get("http://replica/foo/bar");
response = new Response(ResponseCode.Ok);
timeout = 5.Seconds();
log = Substitute.For <ILog>();
log
.When(l => l.Log(Arg.Any <LogEvent>()))
.Do(info => new ConsoleLog().Log(info.Arg <LogEvent>()));
configuration = Substitute.For <IClusterClientConfiguration>();
configuration.ResponseCriteria.Returns(new List <IResponseCriterion> {
Substitute.For <IResponseCriterion>()
});
configuration.LogReplicaRequests.Returns(true);
configuration.LogReplicaResults.Returns(true);
configuration.ReplicaOrdering.Returns(Substitute.For <IReplicaOrdering>());
configuration.Log.Returns(log);
storageProvider = Substitute.For <IReplicaStorageProvider>();
responseClassifier = Substitute.For <IResponseClassifier>();
responseClassifier.Decide(Arg.Any <Response>(), Arg.Any <IList <IResponseCriterion> >()).Returns(ResponseVerdict.Accept);
requestConverter = Substitute.For <IRequestConverter>();
requestConverter.TryConvertToAbsolute(relativeRequest, replica).Returns(_ => absoluteRequest);
transport = Substitute.For <ITransport>();
transport.SendAsync(Arg.Any <Request>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).Returns(_ => response);
sender = new RequestSender(configuration, storageProvider, responseClassifier, requestConverter, transport);
}
/// <inheritdoc />
public void Learn(ReplicaResult result, IReplicaStorageProvider storageProvider)
{
var storage = storageProvider.Obtain <bool>(StorageKey);
var wasLeader = IsLeader(result.Replica, storage, out var hadStoredStatus);
var isLeader = resultDetector.IsLeaderResult(result);
if (isLeader == wasLeader)
{
return;
}
var updatedStatus = hadStoredStatus
? storage.TryUpdate(result.Replica, isLeader, wasLeader)
: storage.TryAdd(result.Replica, isLeader);
if (updatedStatus)
{
if (isLeader)
{
LogLeaderDetected(result.Replica);
}
else
{
LogLeaderFailed(result.Replica);
}
}
}
/// <inheritdoc />
public void Modify(Uri replica, IList <Uri> allReplicas, IReplicaStorageProvider storageProvider, Request request, RequestParameters parameters, ref double weight)
{
if (!IsLeader(replica, storageProvider.Obtain <bool>(StorageKey)))
{
weight = 0.0;
}
}
private IEnumerable <Uri> OrderUsingPooledArray(IList <Uri> replicas, IReplicaStorageProvider storageProvider, Request request, RequestParameters parameters)
{
using (TreeArrays.Acquire(out var treeArray))
foreach (var replica in OrderInternal(replicas, storageProvider, request, parameters, treeArray))
{
yield return(replica);
}
}
/// <inheritdoc />
public void Modify(Uri replica, IList <Uri> allReplicas, IReplicaStorageProvider storageProvider, Request request, RequestParameters parameters, ref double weight)
{
var clusterState = globalStorageProvider.ObtainGlobalValue(storageKey, ClusterStateFactory);
ModifyClusterWeightsIfNeed(clusterState);
weight = ModifyAndApplyLimits(weight, clusterState.Weights.Get(replica, settings.WeightsTTL));
}
private IEnumerable <Uri> OrderUsingPooledArray(IList <Uri> replicas, IReplicaStorageProvider storageProvider, Request request)
{
using (var treeArray = treeArrays.AcquireHandle())
{
foreach (var replica in OrderInternal(replicas, storageProvider, request, treeArray))
{
yield return(replica);
}
}
}
public void Modify(Uri replica, IList <Uri> allReplicas, IReplicaStorageProvider storageProvider, Request request, RequestParameters parameters, ref double weight)
{
var weights = transform.Get(serviceLocator.Locate(environment, application));
if (weights == null || !weights.TryGetValue(replica, out var snitchWeight))
{
return;
}
weight *= snitchWeight.Value;
}
public RequestExecutionModule(
IResponseSelector responseSelector,
IReplicaStorageProvider storageProvider,
IRequestSenderInternal requestSender,
IClusterResultStatusSelector resultStatusSelector)
{
this.responseSelector = responseSelector;
this.storageProvider = storageProvider;
this.requestSender = requestSender;
this.resultStatusSelector = resultStatusSelector;
}
public void TestSetup()
{
replica1 = new Uri("http://replica1");
replica2 = new Uri("http://replica2");
response1 = new Response(ResponseCode.Ok);
response2 = new Response(ResponseCode.Ok);
selectedResponse = new Response(ResponseCode.Ok);
result1 = new ReplicaResult(replica1, response1, ResponseVerdict.DontKnow, TimeSpan.Zero);
result2 = new ReplicaResult(replica2, response2, ResponseVerdict.DontKnow, TimeSpan.Zero);
var parameters = RequestParameters.Empty
.WithStrategy(Substitute.For <IRequestStrategy>());
context = new RequestContext(Request.Get("foo/bar"), parameters, Budget.Infinite, new SilentLog(), null, int.MaxValue);
context.Parameters.Strategy.SendAsync(null, null, null, null, null, 0, default)
.ReturnsForAnyArgs(
async info =>
{
var replicas = info.Arg <IEnumerable <Uri> >();
var sender = info.Arg <IRequestSender>();
foreach (var replica in replicas)
{
await sender.SendToReplicaAsync(replica, context.Request, null, TimeSpan.Zero, CancellationToken.None);
}
});
clusterProvider = Substitute.For <IClusterProvider>();
clusterProvider.GetCluster().Returns(new[] { replica1, replica2 });
requestSender = Substitute.For <IRequestSenderInternal>();
requestSender.SendToReplicaAsync(Arg.Any <ITransport>(), replica1, Arg.Any <Request>(), Arg.Any <TimeSpan?>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).ReturnsTask(_ => result1);
requestSender.SendToReplicaAsync(Arg.Any <ITransport>(), replica2, Arg.Any <Request>(), Arg.Any <TimeSpan?>(), Arg.Any <TimeSpan>(), Arg.Any <CancellationToken>()).ReturnsTask(_ => result2);
replicaOrdering = Substitute.For <IReplicaOrdering>();
replicaOrdering.Order(null, null, null, null).ReturnsForAnyArgs(info => info.Arg <IList <Uri> >().Reverse());
responseSelector = Substitute.For <IResponseSelector>();
responseSelector.Select(null, null, null).ReturnsForAnyArgs(_ => selectedResponse);
resultStatusSelector = Substitute.For <IClusterResultStatusSelector>();
resultStatusSelector.Select(null, null).ReturnsForAnyArgs(ClusterResultStatus.Success);
storageProvider = Substitute.For <IReplicaStorageProvider>();
module = new RequestExecutionModule(
clusterProvider,
replicaOrdering,
responseSelector,
storageProvider,
requestSender,
resultStatusSelector);
}
public void SetUp()
{
replica = new Uri("http://replica");
replicas = new List <Uri> {
replica
};
request = Request.Get("foo/bar");
modifiers = new List <IReplicaWeightModifier>();
storageProvider = Substitute.For <IReplicaStorageProvider>();
calculator = new ReplicaWeightCalculator(modifiers, minWeight, maxWeight, initialWeight);
}
public RequestSender(
IClusterClientConfiguration configuration,
IReplicaStorageProvider storageProvider,
IResponseClassifier responseClassifier,
IRequestConverter requestConverter)
{
this.configuration = configuration;
this.storageProvider = storageProvider;
this.responseClassifier = responseClassifier;
this.requestConverter = requestConverter;
}
private void BuildTree(
TreeNode[] tree,
IList <Uri> replicas,
IReplicaStorageProvider storageProvider,
Request request,
RequestParameters parameters,
ref List <Uri> replicasWithInfiniteWeight,
ref List <Uri> replicasWithZeroWeight)
{
for (var i = 0; i < replicas.Count; i++)
{
var replica = replicas[i];
var weight = weightCalculator.GetWeight(replica, replicas, storageProvider, request, parameters);
if (weight < 0.0)
{
throw new BugcheckException($"A negative weight has been calculated for replica '{replica}': {weight}.");
}
// (iloktionov): Бесконечности портят расчёты на дереве, поэтому они обрабатываются отдельно и не вставляются в него:
if (double.IsPositiveInfinity(weight))
{
(replicasWithInfiniteWeight ?? (replicasWithInfiniteWeight = new List <Uri>())).Add(replica);
continue;
}
// (iloktionov): Чтобы избежать детерминированного упорядочивания реплик с нулевым весом, их тоже придётся рассмотреть отдельно:
if (weight < double.Epsilon)
{
(replicasWithZeroWeight ?? (replicasWithZeroWeight = new List <Uri>())).Add(replica);
continue;
}
var index = replicas.Count + i;
// (iloktionov): Заполняем листовую ноду дерева:
tree[index] = new TreeNode
{
Exists = true,
Weight = weight,
Replica = replica
};
// (iloktionov): И обновляем частичные суммы в промежуточных нодах вплоть до корня:
while (index > 0)
{
index = GetParentIndex(index);
tree[index].Exists = true;
tree[index].Weight += weight;
}
}
}
public void TestSetup()
{
replica = new Uri("http://replica");
replicas = new List <Uri> {
replica
};
request = Request.Get("foo/bar");
parameters = RequestParameters.Empty;
modifiers = new List <IReplicaWeightModifier>();
storageProvider = Substitute.For <IReplicaStorageProvider>();
calculator = new ReplicaWeightCalculator(modifiers, MinWeight, MaxWeight, InitialWeight);
}
private void BuildTree(
TreeNode[] tree,
IList <Uri> replicas,
IReplicaStorageProvider storageProvider,
Request request,
RequestParameters parameters,
ref List <Uri> replicasWithInfiniteWeight,
ref List <Uri> replicasWithZeroWeight)
{
var firstLeafIndex = FillLeaves(tree, replicas, storageProvider, request, parameters,
ref replicasWithInfiniteWeight, ref replicasWithZeroWeight);
GetWeight(tree, 0, firstLeafIndex);
}
public void Modify(Uri replica, IList <Uri> allReplicas, IReplicaStorageProvider storageProvider, Request request, RequestParameters parameters, ref double weight)
{
if (weight < minimumWeightForBoostingProvider())
{
return;
}
var localDatacenter = datacenters.GetLocalDatacenter();
var replicaDatacenter = datacenters.GetDatacenterWeak(replica.Host);
if (string.Equals(localDatacenter, replicaDatacenter, StringComparison.OrdinalIgnoreCase))
{
weight *= boostMultiplierProvider();
}
}
public RequestExecutionModule(
IClusterProvider clusterProvider,
IReplicaOrdering replicaOrdering,
IResponseSelector responseSelector,
IReplicaStorageProvider storageProvider,
IRequestSender requestSender,
IClusterResultStatusSelector resultStatusSelector)
{
this.clusterProvider = clusterProvider;
this.replicaOrdering = replicaOrdering;
this.responseSelector = responseSelector;
this.storageProvider = storageProvider;
this.requestSender = requestSender;
this.resultStatusSelector = resultStatusSelector;
}
/// <inheritdoc />
public IEnumerable <Uri> Order(IList <Uri> replicas, IReplicaStorageProvider storageProvider, Request request, RequestParameters parameters)
{
if (replicas.Count < 2)
{
return(replicas);
}
var requiredCapacity = replicas.Count * 2;
if (requiredCapacity > PooledArraySize)
{
return(OrderInternal(replicas, storageProvider, request, parameters, new TreeNode[requiredCapacity]));
}
return(OrderUsingPooledArray(replicas, storageProvider, request, parameters));
}
public void SetUp()
{
replica1 = new Uri("http://replica1");
replica2 = new Uri("http://replica2");
replicas = new List <Uri> {
replica1, replica2
};
request = Request.Get("foo/bar");
storageProvider = Substitute.For <IReplicaStorageProvider>();
storageProvider.Obtain <int>(Arg.Any <string>()).Returns(storage = new ConcurrentDictionary <Uri, int>());
implementation = Substitute.For <IAdaptiveHealthImplementation <int> >();
tuningPolicy = Substitute.For <IAdaptiveHealthTuningPolicy>();
modifier = new AdaptiveHealthModifier <int>(implementation, tuningPolicy, log = new ConsoleLog());
}
public void Learn(ReplicaResult result, IReplicaStorageProvider storageProvider)
{
if (result.Verdict != ResponseVerdict.Reject)
{
return;
}
var storage = storageProvider.Obtain <DateTime>(storageKey);
var wasGray = storage.ContainsKey(result.Replica);
storage[result.Replica] = timeProvider.GetCurrentTime();
if (!wasGray)
{
LogReplicaIsGrayNow(result.Replica);
}
}
public void Learn(ReplicaResult result, IReplicaStorageProvider storageProvider)
{
var storage = storageProvider.Obtain <THealth>(storageKey);
while (true)
{
THealth currentHealth;
THealth newHealth;
bool foundHealth;
if (!(foundHealth = storage.TryGetValue(result.Replica, out currentHealth)))
{
currentHealth = implementation.CreateDefaultHealth();
}
switch (tuningPolicy.SelectAction(result))
{
case AdaptiveHealthAction.Increase:
newHealth = implementation.IncreaseHealth(currentHealth);
break;
case AdaptiveHealthAction.Decrease:
newHealth = implementation.DecreaseHealth(currentHealth);
break;
default:
newHealth = currentHealth;
break;
}
if (implementation.AreEqual(currentHealth, newHealth))
{
break;
}
var updatedHealth = foundHealth
? storage.TryUpdate(result.Replica, newHealth, currentHealth)
: storage.TryAdd(result.Replica, newHealth);
if (updatedHealth)
{
implementation.LogHealthChange(result.Replica, currentHealth, newHealth, log);
break;
}
}
}
