private async Task<bool> OnStartCore()
{
_current = this;
// Set up logging
LoggingConfiguration logConfig = new LoggingConfiguration();
var fileTarget = new FileTarget()
{
Layout = "[${logger}](${date}) ${message}",
FileName = Path.Combine(RoleEnvironment.GetLocalResource("ConsulLogs").RootPath, "Consulate.log")
};
var traceTarget = new TraceTarget()
{
Layout = "[${logger}] ${message}"
};
logConfig.AddTarget("file", fileTarget);
logConfig.AddTarget("trace", traceTarget);
logConfig.LoggingRules.Add(new LoggingRule("*", NLog.LogLevel.Trace, fileTarget));
logConfig.LoggingRules.Add(new LoggingRule("*", NLog.LogLevel.Trace, traceTarget));
LogManager.Configuration = logConfig;
// Create config
var config = new ConsulConfig()
{
NodeName = RoleEnvironment.CurrentRoleInstance.Id,
DataDir = RoleEnvironment.GetLocalResource("ConsulData").RootPath,
BindAddr = GetIP("Consul.Rpc"),
ClientAddr = GetIP("Consul.Rpc"),
Ports = new ConsulPorts()
{
Dns = GetPort("Consul.Dns"),
Http = GetPort("Consul.Http"),
Rpc = GetPort("Consul.Rpc"),
SerfLan = GetPort("Consul.SerfLan"),
SerfWan = GetPort("Consul.SerfWan"),
Server = GetPort("Consul.Server")
}
};
var clients = GetClients();
// Step 1 - Poll for an existing cluster
Log.Info("Searching for cluster...");
var existingCluster = await FindExistingCluster(clients);
if (!existingCluster.Any())
{
Log.Info("No cluster found, attempting to bootstrap one!");
_agent = await Bootstrap(clients, config);
}
else
{
Log.Info("Found a cluster! Joining it");
_agent = await Join(config, existingCluster);
}
return true;
}
public ConsulAgent(string consulPath, ConsulConfig config, Process consulProcess)
{
_consulPath = consulPath;
_config = config;
_logger = LogManager.GetLogger(typeof(ConsulAgent).FullName + ":PID" + consulProcess.Id);
_consulProcess = consulProcess;
_consulProcess.Exited += (_, __) =>
{
_logger.Info("Shutdown");
};
_consulProcess.ErrorDataReceived += (s, a) =>
{
_logger.Error(a.Data);
};
_consulProcess.OutputDataReceived += (s2, a2) =>
{
_logger.Info(a2.Data);
};
_consulProcess.BeginErrorReadLine();
_consulProcess.BeginOutputReadLine();
}
private async Task<ConsulAgent> Bootstrap(IEnumerable<ConsulClient> clients, ConsulConfig config)
{
// Try to lock the bootstrapper blob
var acct = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("StorageAccount"));
var container = acct.CreateCloudBlobClient().GetContainerReference("consulate");
await container.CreateIfNotExistsAsync();
var blob = container.GetBlockBlobReference("bootstrap.lock");
try
{
await blob.UploadTextAsync("Bootstrap Lock. Only used for leasing", null, AccessCondition.GenerateIfNoneMatchCondition("*"), new BlobRequestOptions(), new OperationContext());
}
catch (StorageException ex)
{
if (ex.RequestInformation == null || ex.RequestInformation.HttpStatusCode != 409)
{
// Expected a conflict, but this was not a conflict :(
throw;
}
}
// Attempt to lease the blob
string leaseId = null;
try
{
bool leased = false;
while (!leased)
{
try
{
Log.Info("Acquiring one-minute bootstrapper lease.");
leaseId = await blob.AcquireLeaseAsync(TimeSpan.FromMinutes(1), null);
leased = true;
}
catch (StorageException ex)
{
if (ex.RequestInformation == null || ex.RequestInformation.ExtendedErrorInformation == null || ex.RequestInformation.ExtendedErrorInformation.ErrorCode != BlobErrorCodeStrings.LeaseAlreadyPresent)
{
throw;
}
else
{
// Already leased
leased = false;
}
}
// We may or may not have a lease, but either way, check for clusters again
Log.Info("Checking for clusters again");
var cluster = await FindExistingCluster(clients);
if (cluster.Any())
{
Log.Info("Found a cluster!");
if (leased)
{
Log.Info("Releasing Bootstrap lease.");
await blob.ReleaseLeaseAsync(AccessCondition.GenerateLeaseCondition(leaseId));
}
// Join the existing cluster
return await Join(config, cluster);
}
}
Log.Info("Bootstreap lease acquired, launching agent in bootstrap mode");
var agent = ConsulAgent.LaunchServer(bootstrap: true, config: config);
var client = new ConsulClient(config.HttpApiUri);
IList<string> nodes = null;
int quorum = (RoleEnvironment.CurrentRoleInstance.Role.Instances.Count / 2) + 1;
Log.Info("Waiting for quorum of {0} nodes to join...", quorum);
while (nodes == null || nodes.Count < quorum)
{
Log.Info("Checking how many nodes have connected", quorum);
nodes = (await client.Peers()).ToList();
if (nodes.Count < quorum)
{
Log.Info("{0} nodes have joined, no quorum yet. Sleeping for one second", nodes.Count);
await Task.Delay(1000);
}
}
Log.Info("Quorum reached! Leaving and rejoining");
agent.Shutdown();
agent.WaitForExit();
Log.Info("Agent has shut down, rejoining as non-bootstrapper...");
return ConsulAgent.LaunchServer(bootstrap: false, config: config, join: nodes.Select(s => IPAddress.Parse(s.Substring(0, s.IndexOf(":")))));
}
finally
{
if (leaseId != null)
{
try
{
Log.Warn("Releasing lease in finally block!");
blob.ReleaseLease(AccessCondition.GenerateLeaseCondition(leaseId));
}
catch (Exception) { }
}
}
}
private Task<ConsulAgent> Join(ConsulConfig config, IEnumerable<string> existingCluster)
{
return Task.FromResult(ConsulAgent.LaunchServer(
bootstrap: false,
config: config,
join: RoleEnvironment
.CurrentRoleInstance
.Role
.Instances
.Where(i => i != RoleEnvironment.CurrentRoleInstance)
.Select(i => i.InstanceEndpoints["Consul.SerfLan"].IPEndpoint.Address)));
}
/// <summary>
/// Starts the consul agent in server mode and returns a ConsulServer to manage it
/// </summary>
public static ConsulAgent LaunchServer(bool bootstrap, ConsulConfig config)
{
return LaunchServer(DefaultConsulPath, bootstrap, config, Enumerable.Empty<IPAddress>());
}
private static ConsulAgent Launch(string consulExePath, ConsulConfig config, string args)
{
ConsulAgentTrace.Info("Launching: {0} {1}", consulExePath, args);
return new ConsulAgent(consulExePath, config, Process.Start(CreateStartInfo(consulExePath, args)));
}
/// <summary>
/// Starts the consul agent in server mode and returns a ConsulServer to manage it
/// </summary>
public static ConsulAgent LaunchServer(string consulExePath, bool bootstrap, ConsulConfig config, IEnumerable<IPAddress> join)
{
// Write the config file
var configFile = Path.Combine(Path.GetTempPath(), "consul-config.json");
File.WriteAllText(configFile, config.ToJson());
// Build args
string args = "agent -server -config-file \"" + configFile + "\"";
if (bootstrap)
{
args += " -bootstrap";
}
if (join.Any())
{
args += String.Join(" ", join.Select(a => " -join " + a.ToString()));
}
// Launch!
return Launch(consulExePath, config, args);
}
/// <summary>
/// Starts the consul agent in server mode and returns a ConsulServer to manage it
/// </summary>
public static ConsulAgent LaunchServer(bool bootstrap, ConsulConfig config, IEnumerable<IPAddress> join)
{
return LaunchServer(DefaultConsulPath, bootstrap, config, join);
}