public static BasicStateMachine CreateSM(int instance, bool designatedPrimary, TestConfig cfg, TestTracer tracer)
{
ManagedRSLNode[] nodes = new ManagedRSLNode[cfg.Ports.Length / 2];
for (int i = 0; i < nodes.Length; i++)
{
nodes[i] = BasicStateMachine.CreateNode(cfg.Ports[2 * i], cfg.Ports[(2 * i) + 1]);
}
BasicStateMachine sm = new BasicStateMachine(tracer, nodes[instance].RslPort, nodes[instance].RslLearnPort);
try
{
sm.CanBP = designatedPrimary;
using (ManagedRSLConfigParam rslCfg = SetupConfiguration(cfg.RslData))
{
Stopwatch w = Stopwatch.StartNew();
while (true)
{
if (sm.Initialize(rslCfg, sm.SelfNode, ManagedRSLProtocolVersion.ManagedRSLProtocolVersion_5, false))
{
break;
}
if (w.ElapsedMilliseconds > 10000)
{
throw new InvalidOperationException("couldn't initialize the SM");
}
Thread.Sleep(100);
}
sm.InitiateBootstrap(nodes, 10);
return(sm);
}
}
catch (Exception e)
{
try
{
sm.Log("Exception during InitializeStateMachine {0}", e);
}
catch (Exception)
{
// ignore
}
sm.Dispose();
throw;
}
}
void Run(ManagedRSLNode[] nodes, ManagedRSLConfigParam cfgParam, ManagedRSLNode selfNode)
{
m_score = 0;
m_outstanding = 0;
m_size = 10 * 1024 * 1024;
m_state = new byte[m_size];
m_clientEnd = false;
m_isPrimary = false;
bool res = Initialize(
cfgParam,
nodes,
selfNode,
ManagedRSLProtocolVersion.ManagedRSLProtocolVersion_3,
false);
Debug.Assert(res);
for (int timer = 0; !m_clientEnd; timer++)
{
if (timer >= 1000 / SLEEP_TIME)
{
timer = 0;
if (m_isPrimary)
{
ChangeConfigurationIfNeeded();
}
}
if (m_isPrimary && m_outstanding < 1)
{
byte[] inc = BitConverter.GetBytes((int)1);
SendRequest(inc, this, UInt64.MaxValue);
}
Thread.Sleep(SLEEP_TIME);
}
}
/// <summary>
/// Setups the configuration.
/// </summary>
/// <param name="rslPath">the path to rsl</param>
/// <returns>ManagedRSLConfigParam.</returns>
private static ManagedRSLConfigParam SetupConfiguration(string rslPath)
{
ManagedRSLConfigParam cfg = new ManagedRSLConfigParam();
try
{
// the following settings affect only how the local replica will handle its own logs.
// bear in mind that codexes are sent to the primary and other replicas at times, so
// - more frequent codexes (shorter logfiles) will cause longer times to catch up since replicas
// will need to use codexes instead of the log entries they missed.
// - larger logfiles enable better recoverability (a node can be down for longer and catch up
// without requiring a codex) but it delays the restart time of replicas, since the logfile
// must be replayed.
// - Keeping a lot of logfiles or codex files will enable a more precise "time travel", but will
// consume more disk space on the replica
cfg.WorkingDir = rslPath;
cfg.MaxLogLenMB = 200;
cfg.MaxVotesInLog = 100000;
cfg.MaxCheckpoints = 4;
cfg.MaxCheckpointIntervalSec = (int)TimeSpan.FromMinutes(10).TotalSeconds;
cfg.MaxLogs = 10;
cfg.LogLenRandomize = 15;
// When a node is primary, it sends heartbeats to the rest.
// - The frequency of those (HeartBeatIntervalSec) determines the fault detection time.
// - The grace period is the time a replica needs to start suspecting from the primary if no HN is seen.
// - After a replica suspects from a primary, it will wait for
// "ElectionDelaySec" seconds before initiating another primary election. The longer this value,
// the fewer unnecessary reconfigurations due to partial failures, but the longer fault recovery
// on real failures.
// - The randomization times reduces the chances for two replicas to try at the same time to become
// primary when they suspect from the primary (often they will suspect nearly at the same time).
cfg.HeartBeatIntervalSec = 1;
cfg.NewLeaderGracePeriodSec = 5;
cfg.ElectionDelaySec = 2;
cfg.ElectionRandomize = 2;
cfg.MaxElectionRandomizeSec = 1;
cfg.AllowPrimaryPromotionWhileCatchingUp = false;
// Retry limits and intervals are intended to cover for partial, transient, short term failures.
// the larger these values, the more chances a transient failure will be cover will exist. However,
// larger values will geopardize the fault detection time when a real failure happened.
cfg.InitializeRetryIntervalSec = 1;
cfg.PrepareRetryIntervalSec = 1;
cfg.VoteRetryIntervalSec = 3;
cfg.VoteMaxOutstandingIntervalSec = 480;
cfg.CPQueryRetryIntervalSec = 5;
// Longer timeouts and larger message size limit allow for bigger transactions, but they also
// geopardize fault detection time, and unresponsiveness of the primary during replication
cfg.SendTimeoutSec = 5;
cfg.ReceiveTimeoutSec = 5;
cfg.MaxMessageSizeMB = 400;
cfg.MaxOutstandingPerReplica = 10;
cfg.MaxCacheLengthMB = 50;
cfg.FastReadSeqThreshold = 0;
return(cfg);
}
catch (Exception)
{
cfg.Dispose();
throw;
}
}
static void Main(string[] args)
{
if (args.Length != 1)
{
PrintUsage(null);
Exit(1);
}
int id = Int32.Parse(args[0]);
ManagedRSLStateMachine.Init(@".\debuglogs\" + id);
ManagedRSLConfigParam cfgParam = new ManagedRSLConfigParam();
// Initialize the default config param
cfgParam.NewLeaderGracePeriodSec = 15;
cfgParam.HeartBeatIntervalSec = 2;
cfgParam.ElectionDelaySec = 10;
cfgParam.MaxElectionRandomizeSec = 1;
cfgParam.InitializeRetryIntervalSec = 1;
cfgParam.PrepareRetryIntervalSec = 1;
cfgParam.VoteRetryIntervalSec = 3;
cfgParam.CPQueryRetryIntervalSec = 5;
cfgParam.MaxCheckpointIntervalSec = 0;
cfgParam.JoinMessagesIntervalSec = 1;
cfgParam.MaxLogLenMB = 1;
cfgParam.SendTimeoutSec = 5;
cfgParam.ReceiveTimeoutSec = 5;
cfgParam.MaxCacheLengthMB = 50;
cfgParam.MaxVotesInLog = 10000;
cfgParam.MaxOutstandingPerReplica = 10;
cfgParam.MaxCheckpoints = 4;
cfgParam.MaxLogs = 10;
cfgParam.LogLenRandomize = 20;
cfgParam.ElectionRandomize = 10;
cfgParam.FastReadSeqThreshold = 5;
cfgParam.InMemoryExecutionQueueSizeMB = 10;
cfgParam.NumReaderThreads = 3;
cfgParam.WorkingDir = @".\data";
int configNumber = 0;
int numReplicas = 0;
int[] memberIds = null;
bool res = ReadConfigurationFromFile(ref configNumber, ref numReplicas, ref memberIds);
Debug.Assert(res);
// Populate member set
if (id <= 0)
{
PrintUsage("invalid member id!");
Exit(1);
}
int bufSize = 100 * 1024;
g_buf = new byte[bufSize];
for (int i = 0; i < bufSize; i++)
{
g_buf[i] = (byte)('a' + (i % 26));
}
ManagedRSLNode[] nodes = new ManagedRSLNode[numReplicas];
for (int i = 0; i < numReplicas; i++)
{
nodes[i] = new ManagedRSLNode();
PopulateNode(nodes[i], (uint)memberIds[i]);
}
ManagedRSLNode selfNode = new ManagedRSLNode();
PopulateNode(selfNode, (uint)id);
// Start replica
Console.WriteLine("Starting member #" + selfNode.MemberId);
TestRSLStateMachineProcessor sm = new TestRSLStateMachineProcessor();
sm.Run(nodes, cfgParam, selfNode);
}
/// <summary>
/// Starts the configured number of replicas, this function will create
/// the necessery files that are needed for the aplicaton to run
/// </summary>
/// <param name="numberOfReplicas">numberOfReplicas</param>
/// <returns></returns>
public int StartUp(int numberOfReplicas)
{
cfg = new ManagedRSLConfigParam();
cfg.NewLeaderGracePeriodSec = 10;
cfg.HeartBeatIntervalSec = 2;
cfg.ElectionDelaySec = 5;
cfg.MaxElectionRandomizeSec = 1;
cfg.InitializeRetryIntervalSec = 1;
cfg.PrepareRetryIntervalSec = 1;
cfg.VoteRetryIntervalSec = 1;
cfg.CPQueryRetryIntervalSec = 5;
cfg.MaxCheckpointIntervalSec = 0;
cfg.MaxLogLenMB = 100;
cfg.SendTimeoutSec = 10;
cfg.ReceiveTimeoutSec = 10;
cfg.MaxCacheLengthMB = 50;
cfg.MaxVotesInLog = 1000;
cfg.MaxOutstandingPerReplica = 10;
cfg.MaxCheckpoints = 2;
cfg.MaxLogs = 5;
cfg.LogLenRandomize = 20;
cfg.ElectionRandomize = 10;
cfg.FastReadSeqThreshold = 5;
cfg.InMemoryExecutionQueueSizeMB = 10;
cfg.NumReaderThreads = 5;
cfg.MaxMessageSizeMB = 1;
cfg.WorkingDir = RSLWorkingDir;
ManagedRSLNode[] nodes = new ManagedRSLNode[numberOfReplicas];
for (int i = 0; i < nodes.Length; i++)
{
nodes[i] = new ManagedRSLNode();
nodes[i].MemberId = (i + 1).ToString();
nodes[i].RslPort = (ushort)(5000 + (100 * (i + 1)));
nodes[i].HostName = "127.0.0.1";
nodes[i].Ip = new IPAddress(0x0100007F); // 127.0.0.1
}
int startedReplicas = 0;
if (_replicaSet == null)
{
_replicaSet = new List <BasicRSLTestMachine>();
ManagedRSLStateMachine.NotificationsCallback = OnNotification;
ManagedRSLStateMachine.Init(".\\" + RSLDebugLogsDir);
for (int i = 0; i < nodes.Length; i++)
{
BasicRSLTestMachine replica = new BasicRSLTestMachine(nodes[i]);
try
{
bool res = replica.Initialize(
cfg,
nodes[i],
ManagedRSLProtocolVersion.ManagedRSLProtocolVersion_4,
false);
Debug.Assert(res);
startedReplicas++;
_replicaSet.Add(replica);
_allReplicaSet.Add(replica);
}
catch (Exception e)
{
if (e is NullReferenceException || e is System.Runtime.InteropServices.SEHException)
{
throw;
}
} //catch
} // for
Console.WriteLine("Bootstrapping");
byte[] buf = new byte[] { 1 };
ManagedRSLMemberSet memberSet = new ManagedRSLMemberSet(nodes, buf, 0, 1);
foreach (BasicRSLTestMachine replica in _allReplicaSet)
{
RSLResponse resp = replica.Bootstrap(memberSet, 10);
Console.WriteLine(resp);
}
}
else
{
startedReplicas = _replicaSet.Count;
}
return(startedReplicas);
} //StartUp
static void Main(string[] args)
{
uint lastRequested = 0;
Hashtable arguments = ParseArguments(args);
imPrimary = false;
int id = (int)arguments["id"];
int targetState = (int)arguments["cases"];
int numReplicas = (int)arguments["numreplicas"];
//Create the output stream
try
{
//_writer = new System.IO.StreamWriter((String)arguments["out"], true);
_writer = Console.Out;
}
catch
{
System.Environment.Exit(-1);
}
//Start up the common infraestructure
ManagedRSLConfigParam cfg = new ManagedRSLConfigParam();
cfg.NewLeaderGracePeriodSec = 10;
cfg.HeartBeatIntervalSec = 2;
cfg.ElectionDelaySec = 5;
cfg.MaxElectionRandomizeSec = 1;
cfg.InitializeRetryIntervalSec = 1;
cfg.PrepareRetryIntervalSec = 1;
cfg.VoteRetryIntervalSec = 1;
cfg.CPQueryRetryIntervalSec = 5;
cfg.MaxCheckpointIntervalSec = 0;
cfg.JoinMessagesIntervalSec = 1;
cfg.MaxLogLenMB = 10;
cfg.SendTimeoutSec = 10;
cfg.ReceiveTimeoutSec = 10;
cfg.MaxCacheLengthMB = 50;
cfg.MaxVotesInLog = 1000;
cfg.MaxOutstandingPerReplica = 10;
cfg.MaxCheckpoints = 2;
cfg.MaxLogs = 5;
cfg.LogLenRandomize = 20;
cfg.ElectionRandomize = 10;
cfg.FastReadSeqThreshold = 5;
cfg.InMemoryExecutionQueueSizeMB = 10;
cfg.NumReaderThreads = 5;
cfg.MaxMessageSizeMB = 1;
cfg.WorkingDir = ".\\rslib" + id;
ManagedRSLNode[] nodes = new ManagedRSLNode[numReplicas];
for (int i = 0; i < nodes.Length; i++)
{
nodes[i] = new ManagedRSLNode();
nodes[i].MemberId = (i + 1).ToString();
nodes[i].RslPort = (ushort)(20000 + (1000 * (i + 1)));
nodes[i].HostName = "127.0.0.1";
nodes[i].Ip = new IPAddress(0x0100007F); // 127.0.0.1
}
ManagedRSLNode selfNode = new ManagedRSLNode();
selfNode.MemberId = id.ToString();
selfNode.RslPort = (ushort)(20000 + (1000 * id));
selfNode.HostName = "127.0.0.1";
selfNode.Ip = new IPAddress(0x0100007F); // 127.0.0.1
ManagedRSLStateMachine.Init(".\\debuglogs" + id);
//Create and initialize the state machine
_stateMachine = new StateMachine((int)arguments["reqsize"]);
if (_stateMachine.Initialize(cfg, nodes, selfNode, ManagedRSLProtocolVersion.ManagedRSLProtocolVersion_4, false) == false)
{
System.Environment.Exit(-1);
}
//Wait 3 seconds for the pending request to be executed
System.Threading.Thread.Sleep(3000);
_writer.WriteLine("{1} Initial state: {0}", _stateMachine.State, DateTime.Now);
lastRequested = _stateMachine.State;
//Execute the test scenario
while (_stateMachine.State < targetState)
{
if (imPrimary)
{
if (_stateMachine.State == lastRequested)
{
if (_stateMachine.WriteToState((UInt32)_stateMachine.State + 1) == true)
{
lastRequested = _stateMachine.State + 1;
lock (_writer)
{
_writer.WriteLine("{1} Request: {0}", lastRequested, DateTime.Now);
}
}
}
}
else
{
lastRequested = _stateMachine.State;
}
_stateMachine.ReadFromState();
_writer.Flush();
System.Threading.Thread.Sleep(1);
}
_stateMachine.DumpState();
_writer.Flush();
try
{
_writerFlag = new System.IO.StreamWriter(String.Format("{0}.done", arguments["out"]), true);
_writerFlag.WriteLine("DONE");
_writerFlag.Close();
}
catch
{
System.Environment.Exit(-1);
}
_writer.Close();
System.Threading.Thread.Sleep(int.MaxValue);
}
