public void DisableWaitsForInFlightQueriesTest()
{
const int replicaInstance = 0;
var replicaSet = GraphClusterLowLevel.GetGraphReplicaSetLowLevel("published");
Parallel.For(0, 100, async(i, state) =>
{
TestOutputHelper.WriteLine($"Iteration {i} on thread #{Thread.CurrentThread.ManagedThreadId}");
if (i == 5)
{
TestOutputHelper.WriteLine($"Disabling replica #{replicaInstance}. Iteration #{i}. Thread #{Thread.CurrentThread.ManagedThreadId}");
TestOutputHelper.WriteLine("----------------------------->>>-------------------------------");
replicaSet.Disable(replicaInstance);
TestOutputHelper.WriteLine("-----------------------------<<<-------------------------------");
TestOutputHelper.WriteLine($"Disabled replica #{replicaInstance}. Iteration #{i}. Thread #{Thread.CurrentThread.ManagedThreadId}");
}
else
{
await replicaSet.Run(Query);
}
});
Trace(replicaSet);
Thread.Sleep(2000);
// what can we check?, we can't assume all Run()'s before the disabled were started
// and we can't assume Run()'s from later iteration's weren't started
// int enabledInstanceCount = replicaSet.EnabledInstanceCount;
// Assert.Equal(NumberOfReplicasConfiguredForPublishedSet, enabledInstanceCount);
Assert.True(true);
}
public void EnableReplicaTest()
{
const int totalIterations = 100;
const int disableIteration = 5;
const int reenableIteration = 30;
const int reenableReplicaInstance = 0;
const string disableReplicaInstanceGraphName = "neo4j";
const string untouchedReplicaInstanceGraphName = "published1";
var replicaSet = GraphClusterLowLevel.GetGraphReplicaSetLowLevel("published");
Parallel.For(0, totalIterations, async(i, state) =>
{
switch (i)
{
case disableIteration:
replicaSet.Disable(reenableReplicaInstance);
break;
case reenableIteration:
replicaSet.Enable(reenableReplicaInstance);
break;
default:
await replicaSet.Run(Query);
break;
}
});
Logger.LogTrace(replicaSet.ToTraceString());
List <LogEntry> log = Logger.Entries.ToList();
(int?enabledLogEntryIndex, _) = GetLogEntry(log, LogId.GraphEnabled);
// checking replica was re-enabled
Assert.NotNull(enabledLogEntryIndex);
ulong jobsStartedOnReEnabledReplica = (ulong)log
.Skip(enabledLogEntryIndex !.Value + 1)
.Count(l => IsLog(l, IntegrationTestLogId.RunQueryStarted,
kv => kv.Key == "DatabaseName" && (string)kv.Value == disableReplicaInstanceGraphName));
ulong jobsStartedOnUntouchedReplica = (ulong)log
.Skip(enabledLogEntryIndex !.Value + 1)
.Count(l => IsLog(l, IntegrationTestLogId.RunQueryStarted,
kv => kv.Key == "DatabaseName" && (string)kv.Value == untouchedReplicaInstanceGraphName));
Logger.LogInformation(
$"Jobs started on replicas after reenable: reenabled={jobsStartedOnReEnabledReplica}, untouched={jobsStartedOnUntouchedReplica}.");
Assert.True(jobsStartedOnReEnabledReplica > 0);
Assert.True(jobsStartedOnUntouchedReplica > 0);
const int allowableDifferenceBetweenJobStarts = 3;
Assert.True(Math.Abs((int)jobsStartedOnReEnabledReplica - (int)jobsStartedOnUntouchedReplica) <= allowableDifferenceBetweenJobStarts,
"Jobs started after replica reenable are roughly equal between the 2 replicas.");
}
public void DisableAndQuiesceReplicaTest()
{
const int disableReplicaInstance = 0;
const string disableReplicaInstanceGraphName = "neo4j";
var replicaSet = GraphClusterLowLevel.GetGraphReplicaSetLowLevel("published");
Parallel.For(0, 100, async(i, state) =>
{
if (i == 5)
{
replicaSet.Disable(disableReplicaInstance);
}
else
{
await replicaSet.Run(Query);
}
});
Logger.LogTrace(replicaSet.ToTraceString());
List <LogEntry> log = Logger.Entries.ToList();
(int?disabledLogEntryIndex, _) = GetLogEntry(log, LogId.GraphDisabled);
// checking replica was disabled
Assert.NotNull(disabledLogEntryIndex);
Assert.True(log
.Skip(disabledLogEntryIndex !.Value + 1)
.All(l => !IsLog(l, IntegrationTestLogId.RunQueryStarted,
kv => kv.Key == "DatabaseName" && (string)kv.Value == disableReplicaInstanceGraphName)),
"No jobs were started on the disabled replica.");
(int?quiescingLogEntryIndex, LogEntry? quiescingLogEntry) = GetLogEntry(log, LogId.QuiescingGraph);
// checking replica started quiescing
Assert.NotNull(quiescingLogEntryIndex);
ulong inFlightCount = Get <ulong>(quiescingLogEntry !, "InFlightCount");
ulong jobsFinishedOnDisabledReplicaAfterReplicaDisabled = (ulong)log
.Skip(disabledLogEntryIndex !.Value + 1)
.Count(l => IsLog(l, IntegrationTestLogId.RunQueryFinished,
kv => kv.Key == "DatabaseName" && (string)kv.Value == disableReplicaInstanceGraphName));
// check right number of In flight jobs finished after replica was disabled
Assert.Equal(inFlightCount, jobsFinishedOnDisabledReplicaAfterReplicaDisabled);
}
protected void ReferenceCountTest(int parallelLoops)
{
const int replicaInstance = 0;
var replicaSet = GraphClusterLowLevel.GetGraphReplicaSetLowLevel("published");
Parallel.For(0, parallelLoops, (i, state) =>
{
TestOutputHelper.WriteLine($"Thread id: {Thread.CurrentThread.ManagedThreadId}");
replicaSet.Disable(replicaInstance);
replicaSet.Enable(replicaInstance);
});
int enabledInstanceCount = replicaSet.EnabledInstanceCount;
Assert.Equal(NumberOfReplicasConfiguredForPublishedSet, enabledInstanceCount);
Assert.True(replicaSet.IsEnabled(replicaInstance));
}