/// <summary>
/// Changes <c>Auth</c> for the given session.
/// </summary>
/// <param name="backend">Backend associated with the session</param>
/// <param name="session">Session for which the <c>Auth</c> must be changed</param>
/// <param name="authDigest">Digest string</param>
private static void SetAuth(RingMasterBackendCore backend, ClientSession session, string authDigest)
{
backend.ProcessMessage(
new RequestSetAuth(authDigest, null),
session,
(response, ex) => response.ResultCode.Should().Be((int)Code.Ok));
}
private static RingMasterBackendCore CreateBackend(InMemoryFactory persistedDataFactory)
{
var backendStarted = new ManualResetEventSlim();
RingMasterBackendCore.GetSettingFunction = GetSetting;
RingMasterBackendCore backend = null;
try
{
backend = new RingMasterBackendCore(persistedDataFactory);
backend.StartService = (p1, p2) => { backendStarted.Set(); };
backend.Start(CancellationToken.None);
backend.OnBecomePrimary();
backendStarted.Wait();
RingMasterBackendCore backendReturn = backend;
backend = null;
return(backendReturn);
}
finally
{
if (backend != null)
{
backend.Dispose();
}
}
}
/// <inheritdoc />
public void Dispose()
{
if (!this.disposed)
{
this.disposed = true;
if (this.factory != null)
{
this.factory.Dispose();
this.factory = null;
}
if (this.backend != null)
{
this.backend.Dispose();
this.backend = null;
}
if (this.ringMasterServer != null)
{
this.ringMasterServer.Dispose();
this.ringMasterServer = null;
}
}
}
/// <summary>
/// Creates and initializes a new session
/// </summary>
/// <param name="backend">Backend to initialize the session for</param>
/// <param name="authDigest">Optional authentication digest</param>
/// <returns>The session</returns>
private static ClientSession CreateSession(RingMasterBackendCore backend, string authDigest = null)
{
var session = new ClientSession((requestCall, clientSession, responseAction) =>
{
backend.ProcessMessage(requestCall.Request, clientSession, responseAction);
});
backend.ProcessSessionInitialization(
new RequestCall
{
CallId = 0,
Request = new RequestInit(0, Guid.NewGuid().ToString(), null, true, RedirectionPolicy.RedirectPreferred)
},
session).ResultCode.Should().Be((int)Code.Ok);
if (authDigest != null)
{
backend.ProcessMessage(
new RequestSetAuth(authDigest, null),
session,
(response, ex) => response.ResultCode.Should().Be((int)Code.Ok));
}
return(session);
}
private static void TestDelete(
Action <string> log,
int numberOfThreads,
RingMasterBackendCore backendCore,
ClientSession clientSession)
{
var sw = new Stopwatch();
var data = new byte[PayloadLength];
var countPerThread = TotalNodeCount / numberOfThreads;
var threads = Enumerable.Range(0, numberOfThreads)
.Select(n => new Thread(() =>
{
for (int i = 0; i < countPerThread; i++)
{
var delReq = new RequestDelete($"/test/{n}/{i}", null, -1, null, true);
backendCore.ProcessMessage(delReq, clientSession, null);
}
}))
.ToArray();
int gen0 = GC.CollectionCount(0), gen1 = GC.CollectionCount(1), gen2 = GC.CollectionCount(2);
sw.Start();
Parallel.ForEach(threads, t => t.Start());
Parallel.ForEach(threads, t => t.Join());
sw.Stop();
var rate = 10 * countPerThread * numberOfThreads / sw.Elapsed.TotalSeconds;
log($"Delete nodes: {sw.Elapsed} QPS={rate:G3}");
log($" Gen0={GC.CollectionCount(0) - gen0} Gen1={GC.CollectionCount(1) - gen1} Gen2={GC.CollectionCount(2) - gen2}\n");
}
/// <summary>
/// Creates a new backend with an in-memory store
/// </summary>
/// <returns>Backend instance</returns>
private static RingMasterBackendCore CreateBackend()
{
RingMasterBackendCore backend = null;
try
{
var backendStarted = new ManualResetEventSlim();
backend = new RingMasterBackendCore(inMemoryFactory);
backend.StartService = (p1, p2) => { backendStarted.Set(); };
backend.Start(CancellationToken.None);
backend.OnBecomePrimary();
Assert.IsTrue(backendStarted.Wait(30000));
var backendToReturn = backend;
backend = null;
return(backendToReturn);
}
finally
{
if (backend != null)
{
backend.Dispose();
}
}
}
/// <summary>
/// Creates a new backend with an in-memory store
/// </summary>
/// <returns>Backend instance</returns>
private RingMasterBackendCore CreateBackend()
{
RingMasterBackendCore backend = null;
try
{
var backendStarted = new ManualResetEventSlim();
Trace.TraceInformation("CreateBackend");
RingMasterBackendCore.GetSettingFunction = GetSetting;
backend = new RingMasterBackendCore(this.factory);
backend.StartService = (p1, p2) => { backendStarted.Set(); };
backend.Start(CancellationToken.None);
backend.OnBecomePrimary();
Assert.IsTrue(backendStarted.Wait(30000));
var backendToReturn = backend;
backend = null;
return(backendToReturn);
}
finally
{
if (backend != null)
{
backend.Dispose();
}
}
}
/// <summary>
/// Activates the specified argBackend.
/// </summary>
/// <param name="argBackend">The argBackend.</param>
/// <param name="client">The client.</param>
public void Activate(RingMasterBackendCore argBackend, IPersistedDataFactoryClient client)
{
if (client != null)
{
while (!client.CanBecomePrimary())
{
Thread.Sleep(100);
}
client.OnBecomePrimary();
}
}
/// <summary>
/// Creates a new backend with an in-memory store
/// </summary>
/// <param name="auditConsumer">Interface to an object that can consume audit events</param>
/// <returns>Backend instance</returns>
private static RingMasterBackendCore CreateBackend(IRingMasterAudit auditConsumer)
{
var backendStarted = new ManualResetEventSlim();
RingMasterBackendCore.GetSettingFunction = settingName => null;
var backend = new RingMasterBackendCore(new InMemoryFactory(), auditConsumer);
backend.StartService = (p1, p2) => { backendStarted.Set(); };
backend.Start(CancellationToken.None);
backend.OnBecomePrimary();
backendStarted.Wait(30000).Should().BeTrue();
return(backend);
}
/// <summary>
/// Processes a request in the backend
/// </summary>
/// <param name="backend">Backend for the operation</param>
/// <param name="session">Session for the operation</param>
/// <param name="request">Request to execute</param>
/// <param name="expectedResponseCode">Expected response code for the request</param>
private static void ProcessRequest(RingMasterBackendCore backend, ClientSession session, IRingMasterBackendRequest request, Code expectedResponseCode)
{
var evt = ManualResetEventPool.InstancePool.GetOne();
backend.ProcessMessage(
request,
session,
(response, ex) =>
{
response.ResultCode.Should().Be((int)expectedResponseCode);
evt.Set();
});
ManualResetEventPool.InstancePool.WaitOneAndReturn(ref evt);
}
/// <summary>
/// Creates a new backend with an in-memory store
/// </summary>
/// <param name="getSettingFunction">Function to get settings</param>
/// <returns>Backend instance</returns>
private static RingMasterBackendCore CreateBackend(RingMasterBackendCore.getSettingFunctionDelegate getSettingFunction)
{
var backendStarted = new ManualResetEventSlim();
RingMasterBackendCore.GetSettingFunction = getSettingFunction;
var backend = new RingMasterBackendCore(new InMemoryFactory());
backend.StartService = (p1, p2) => { backendStarted.Set(); };
backend.Start(CancellationToken.None);
backend.OnBecomePrimary();
backendStarted.Wait(30000).Should().BeTrue();
return(backend);
}
/// <summary>
/// Initializes a new instance of the <see cref="RingMasterService"/> class.
/// </summary>
/// <param name="context">Service context</param>
/// <param name="ringMasterMetricsFactory">Metrics factory for MDM</param>
/// <param name="persistenceMetricsFactory">Metrics factory for persistence</param>
public RingMasterService(StatefulServiceContext context, IMetricsFactory ringMasterMetricsFactory, IMetricsFactory persistenceMetricsFactory)
: base(context, PersistedDataFactory.CreateStateManager(context))
{
var path = System.Reflection.Assembly.GetExecutingAssembly().Location;
var builder = new ConfigurationBuilder().SetBasePath(Path.GetDirectoryName(path)).AddJsonFile("appSettings.json");
appSettings = builder.Build();
RingMasterBackendCore.GetSettingFunction = GetSetting;
string factoryName = $"{this.Context.ServiceTypeName}-{this.Context.ReplicaId}-{this.Context.NodeContext.NodeName}";
this.zooKeeperServerInstrumentation = new ZooKeeperServerInstrumentation(ringMasterMetricsFactory);
this.ringMasterServerInstrumentation = new RingMasterServerInstrumentation(ringMasterMetricsFactory);
var ringMasterInstrumentation = new RingMasterBackendInstrumentation(ringMasterMetricsFactory);
var persistenceInstrumentation = new ServiceFabricPersistenceInstrumentation(persistenceMetricsFactory);
RingMasterBackendCoreInstrumentation.Instance = ringMasterInstrumentation;
bool needFixStatDuringLoad = bool.TryParse(GetSetting("WinFabPersistence.FixStatDuringLoad"), out needFixStatDuringLoad) && needFixStatDuringLoad;
var persistenceConfiguration = new PersistedDataFactory.Configuration
{
EnableActiveSecondary = true,
FixStatDuringLoad = needFixStatDuringLoad,
};
bool useInMemoryPersistence;
if (bool.TryParse(GetSetting("InMemoryPersistence"), out useInMemoryPersistence) && useInMemoryPersistence)
{
this.factory = new InMemoryFactory();
}
else
{
this.factory = new PersistedDataFactory(
this.StateManager,
factoryName,
persistenceConfiguration,
persistenceInstrumentation,
this.cancellationSource.Token);
}
this.backend = new RingMasterBackendCore(this.factory);
this.factory.SetBackend(this.backend);
this.executor = this.backend;
}
/// <summary>
/// Creates a new node with random characters in the node path
/// </summary>
/// <param name="backend">The backend to create the node in</param>
/// <param name="session">The session to use for the create</param>
/// <param name="segments">Amount of path segments</param>
/// <param name="segmentLength">Length of each path segment</param>
/// <param name="expectedResponseCode">Expected response code for the create request</param>
/// <param name="endingSegmentLength">If non-zero, will add an extra segment on the end with the specified length</param>
/// <param name="createMode">Mode for creating the node</param>
private static void CreateNode(RingMasterBackendCore backend, ClientSession session, int segments, int segmentLength, Code expectedResponseCode, int endingSegmentLength = 0, CreateMode createMode = CreateMode.Persistent)
{
var stringBuilder = new StringBuilder();
for (int i = 0; i < segments; ++i)
{
stringBuilder.AppendFormat("/{0}", CreateRandomString(segmentLength));
}
if (endingSegmentLength > 0)
{
stringBuilder.AppendFormat("/{0}", CreateRandomString(endingSegmentLength));
}
CreateNode(backend, session, stringBuilder.ToString(), null, expectedResponseCode, createMode);
}
/// <summary>
/// Creates a new backend with an in-memory store
/// </summary>
/// <param name="auditConsumer">Interface to an object that can consume audit events</param>
/// <returns>Backend instance</returns>
protected static RingMasterBackendCore CreateBackend(IRingMasterAudit auditConsumer)
{
var backendStarted = new ManualResetEventSlim();
Trace.TraceInformation("RingMasterBackendCoreUnitTest.CreateBackend");
RingMasterBackendCore.GetSettingFunction = GetSetting;
var backend = new RingMasterBackendCore(new InMemoryFactory(), auditConsumer);
backend.StartService = (p1, p2) => { backendStarted.Set(); };
backend.Start(CancellationToken.None);
backend.OnBecomePrimary();
backendStarted.Wait(30000).Should().BeTrue();
Trace.TraceInformation("RingMasterBackendCoreUnitTest.BackendStarted");
return(backend);
}
/// <summary>
/// Initializes a new instance of the <see cref="RingMasterInstance"/> class.
/// </summary>
/// <param name="id">Unique Id of the instance</param>
/// <param name="isPrimary">If <c>true</c> this instance is the primary</param>
public RingMasterInstance(int id, bool isPrimary)
{
this.Id = id;
Trace.TraceInformation($"RingMasterInstance id={this.Id}");
this.persistedDataFactory = new InMemoryFactory(isPrimary, null, CancellationToken.None);
this.backend = new RingMasterBackendCore(this.persistedDataFactory);
this.persistedDataFactory.SetBackend(this.backend);
this.backend.StartService = this.OnStartService;
this.persistedDataFactory.OnFatalError = (message, exception) =>
{
Assert.Fail($"RingMasterInstance.FatalError: instanceId={id} message={message}, exception={exception}");
};
this.persistedDataFactory.OnChangeListCommitted = this.OnChangeListCommitted;
}
public static void Setup(TestContext context)
{
var path = System.Reflection.Assembly.GetExecutingAssembly().Location;
var builder = new ConfigurationBuilder().SetBasePath(Path.GetDirectoryName(path)).AddJsonFile("appSettings.json");
IConfiguration appSettings = builder.Build();
Helpers.SetupTraceLog(Path.Combine(appSettings["LogFolder"], "VegaInMemoryPerf.LogPath"));
log = s => Trace.TraceInformation(s);
// If a parameter is specified as follows:
// te.exe VegaInMemoryPerf.dll /p:ServerAddress=127.0.0.1:99
if (!context.Properties.ContainsKey("ServerAddress"))
{
backendCore = CreateBackend();
backendServer = new RingMasterServer(protocol, null, CancellationToken.None);
var transportConfig = new SecureTransport.Configuration
{
UseSecureConnection = false,
IsClientCertificateRequired = false,
CommunicationProtocolVersion = RingMasterCommunicationProtocol.MaximumSupportedVersion,
};
serverTransport = new SecureTransport(transportConfig);
backendServer.RegisterTransport(serverTransport);
backendServer.OnInitSession = initRequest =>
{
return(new CoreRequestHandler(backendCore, initRequest));
};
serverTransport.StartServer(10009);
serverAddress = "127.0.0.1:10009";
}
else
{
serverAddress = context.Properties["ServerAddress"] as string;
}
// Read the app settings
minPayloadSize = int.Parse(appSettings["MinPayloadSize"]);
maxPayloadSize = int.Parse(appSettings["MaxPayloadSize"]);
threadCount = int.Parse(appSettings["ThreadCount"]);
}
/// <summary>
/// Initializes a new instance of the <see cref="RingMasterClientUnitTest"/> class.
/// </summary>
public RingMasterClientUnitTest()
{
this.serverTransport = new SimpleTransport();
this.backend = this.CreateBackend();
this.ringMasterServer = new RingMasterServer(this.protocol, null, this.cancellationTokenSource.Token);
this.ringMasterServer.RegisterTransport(this.serverTransport);
this.ringMasterServer.OnInitSession = initRequest =>
{
return(new CoreRequestHandler(this.backend, initRequest));
};
foreach (var source in EventSource.GetSources())
{
if (source.Guid == this.ringMasterClientEventSourceGuid)
{
Trace.TraceInformation($"Enabling EventSource {source.Name}");
this.listener.EnableEvents(source, EventLevel.Verbose);
}
}
}
/// <summary>
/// Creates a new backend with an in-memory store
/// </summary>
private void CreateBackend()
{
var backendStarted = new ManualResetEventSlim();
Trace.TraceInformation("CreateBackend");
var path = System.Reflection.Assembly.GetExecutingAssembly().Location;
var builder = new ConfigurationBuilder().SetBasePath(Path.GetDirectoryName(path)).AddJsonFile("appSettings.json");
appSettings = builder.Build();
RingMasterBackendCore.GetSettingFunction = GetSetting;
this.factory = new InMemoryFactory();
this.backend = new RingMasterBackendCore(this.factory);
this.backend.StartService = (p1, p2) => { backendStarted.Set(); };
this.backend.Start(CancellationToken.None);
this.backend.OnBecomePrimary();
if (!backendStarted.Wait(30000))
{
throw new RingMasterBackendException("Backend failed to start");
}
}
/// <summary>
/// Creates a new node in the backend
/// </summary>
/// <param name="backend">Backend to create the node in</param>
/// <param name="session">Session to use for creating the node</param>
/// <param name="nodePath">Path of the node</param>
/// <param name="data">Data for the node</param>
/// <param name="expectedResponseCode">Expected response for the operation</param>
/// <param name="createMode">Creation mode</param>
/// <param name="acls">ACLs for the node</param>
private static void CreateNode(RingMasterBackendCore backend, ClientSession session, string nodePath, byte[] data, Code expectedResponseCode, CreateMode createMode = CreateMode.Persistent, List <Acl> acls = null)
{
ProcessRequest(backend, session, new RequestCreate(nodePath, null, data, acls, createMode, null), expectedResponseCode);
}
/// <summary>
/// Deletes a node in the backend
/// </summary>
/// <param name="backend">Backend to delete the node from</param>
/// <param name="session">Session to use for the deletion</param>
/// <param name="nodePath">Path of the node</param>
private static void DeleteNode(RingMasterBackendCore backend, ClientSession session, string nodePath)
{
ProcessRequest(backend, session, new RequestDelete(nodePath, null, -1, null), Code.Ok);
}
/// <summary>
/// Sets data on a node in the backend
/// </summary>
/// <param name="backend">Backend for the operation</param>
/// <param name="session">Session for the operation</param>
/// <param name="nodePath">Path of the node</param>
/// <param name="data">Data to set</param>
/// <param name="expectedResponseCode">Expected result code of the operation</param>
private static void SetData(RingMasterBackendCore backend, ClientSession session, string nodePath, byte[] data, Code expectedResponseCode)
{
ProcessRequest(backend, session, new RequestSetData(nodePath, null, data, -1, null), expectedResponseCode);
}
protected RingMasterBackendCoreUnitTest()
{
this.backend = CreateBackend(null);
}
/// <summary>
/// Sets an ACL on a node in the backend
/// </summary>
/// <param name="backend">Backend for the operation</param>
/// <param name="session">Session for the operation</param>
/// <param name="nodePath">Path of the node</param>
/// <param name="acls">Data to set</param>
/// <param name="expectedResponseCode">Expected result code of the operation</param>
private static void SetAcl(RingMasterBackendCore backend, ClientSession session, string nodePath, List <Acl> acls, Code expectedResponseCode)
{
ProcessRequest(backend, session, new RequestSetAcl(nodePath, null, acls, -1, null), expectedResponseCode);
}
