public void StopLocalNode()
{
// AP service manager waits for ShutdownTimeoutInMilliseconds (by default 5s) before forcefully terminating the service host.
// For now we are not waiting for Fabric host service to stop on stopping path.
ServiceController fabricInstallerService = this.GetInstalledService(Constants.FabricInstallerServiceName);
if (fabricInstallerService != null)
{
if (fabricInstallerService.Status != ServiceControllerStatus.StopPending && fabricInstallerService.Status != ServiceControllerStatus.Stopped)
{
TextLogger.LogInfo("Stopping Fabric installer serivce.");
fabricInstallerService.Stop();
}
}
ServiceController fabricHostService = this.GetInstalledService(Constants.FabricHostServiceName);
if (fabricHostService != null)
{
if (fabricHostService.Status != ServiceControllerStatus.StopPending && fabricHostService.Status != ServiceControllerStatus.Stopped)
{
TextLogger.LogInfo("Stopping Fabric host serivce.");
fabricHostService.Stop();
}
}
}
public bool ConfigureLocalNode(ClusterTopology topologyToDeploy)
{
try
{
TextLogger.LogInfo("Configuring local Service Fabric node based on bootstrap cluster manifest.");
if (!this.InvokeFabricDeployer(topologyToDeploy, DeploymentType.Create, false))
{
TextLogger.LogError("Failed to configure local Service Fabric node based on bootstrap cluster manifest.");
return(false);
}
Utilities.SetWindowsFabricNodeConfigurationCompleted(false);
TextLogger.LogInfo("Successfully configured local Service Fabric node based on bootstrap cluster manifest.");
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to configure local Service Fabric node based on bootstrap cluster manifest : {0}", e);
return(false);
}
}
public bool InstallProduct()
{
TextLogger.LogInfo("Installing Service Fabric product.");
try
{
string installedServiceFabricProductVersion = Utilities.GetCurrentWindowsFabricVersion();
if (installedServiceFabricProductVersion == null)
{
/*
* For Service Fabric on Autopilot, xcopyable deployment is the only supported mechanism for installing Service Fabric product.
* MSI is not supported.
*/
string serviceFabricCabFile = this.GetServiceFabricCabFile();
if (File.Exists(serviceFabricCabFile))
{
TextLogger.LogInfo("Installing Service Fabric product with cab file {0}.", serviceFabricCabFile);
string localInstallationPath = Environment.ExpandEnvironmentVariables(Constants.FabricRootPath);
if (!Directory.Exists(localInstallationPath))
{
Directory.CreateDirectory(localInstallationPath);
}
Utilities.UnCabFile(serviceFabricCabFile, localInstallationPath);
Utilities.SetFabricRoot(localInstallationPath);
Utilities.SetFabricCodePath(Path.Combine(localInstallationPath, Constants.PathToFabricCode));
// TODO: registry key not available yet
installedServiceFabricProductVersion = Utilities.GetCurrentWindowsFabricVersion();
TextLogger.LogInfo("Successfully installed Service Fabric product version {0} to local path {1} and set Service Fabric registry keys.", installedServiceFabricProductVersion, localInstallationPath);
}
else
{
// TODO: [General] surface fatal errors outside the agent as AP health report.
TextLogger.LogError("Failed to install Service Fabric product : Service Fabric product cab file does not exist in agent application directory as {0}.", serviceFabricCabFile);
return(false);
}
}
else
{
TextLogger.LogInfo("Service Fabric product version {0} is already installed locally. Skip product installation.", installedServiceFabricProductVersion);
}
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to install Service Fabric product : {0}", e);
return(false);
}
}
private void ConsoleCancelEventHanlder(object sender, ConsoleCancelEventArgs args)
{
TextLogger.LogInfo("Cancel event (Ctrl + C / Ctrl + Break) received. Stopping Service Fabric Environment Sync.");
this.certificateManagerCancellationTokenSource.Cancel();
TextLogger.LogInfo("Service Fabric Environment Sync has stopped. Exiting the process.");
this.Exit(true);
}
public static void LogCurrentConfigurations()
{
TextLogger.LogInfo("Service Fabric Autopilot Agent configurations:");
TextLogger.LogInfo("BackgroundStateMachineRunIntervalSeconds = {0}", BackgroundStateMachineRunIntervalSeconds);
TextLogger.LogInfo("DeploymentRetryIntervalSeconds = {0}", DeploymentRetryIntervalSeconds);
TextLogger.LogInfo("DynamicTopologyUpdateMode = {0}", DynamicTopologyUpdateMode);
TextLogger.LogInfo("BootstrapClusterManifestLocation = {0}", BootstrapClusterManifestLocation);
TextLogger.LogInfo("DynamicPortRangeStart = {0}", DynamicPortRangeStart);
TextLogger.LogInfo("DynamicPortRangeEnd = {0}", DynamicPortRangeEnd);
TextLogger.LogInfo("PortFowardingRulesCount = {0}", PortFowardingRules.Count);
}
private bool ApplyPortForwardingRule(PortFowardingRule portForwardingRule)
{
if (string.Compare(portForwardingRule.MachineFunction, "*", StringComparison.OrdinalIgnoreCase) == 0 || string.Compare(portForwardingRule.MachineFunction, APRuntime.MachineFunction, StringComparison.OrdinalIgnoreCase) == 0)
{
TextLogger.LogInfo("Applying port forwarding rule {0} from port {1} to port {2}.", portForwardingRule.RuleName, portForwardingRule.ListenPort, portForwardingRule.ConnectPort);
return(this.CreatePortProxy(IPAddress.Any.ToString(), portForwardingRule.ListenPort, this.LocalMachineIpAddress, portForwardingRule.ConnectPort));
}
else
{
TextLogger.LogInfo("Port forwarding rule {0} does not apply to machine function {1}.", portForwardingRule.RuleName, APRuntime.MachineFunction);
return(true);
}
}
public void SetNextState(ServiceFabricAutopilotAgentState nextState, bool toIgnoreStateTransitionWhenStopping = true)
{
lock (this.StatusLock)
{
if (toIgnoreStateTransitionWhenStopping && this.State < ServiceFabricAutopilotAgentState.Started)
{
TextLogger.LogVerbose("Application is stopping. Ignore state transition request to {0}.", nextState);
return;
}
TextLogger.LogInfo("State transition: {0} -> {1}", this.State, nextState);
this.State = nextState;
}
}
/*
* Test mode allows testing of ServiceFabricAutopilotAgent outside an Autopilot hosting environment.
* In test mode, all dependencies on AP Runtime are replaced by test environment equivalences:
* Logging => text log
* Configuration => default test configurations
*/
public void Start(bool useTestMode = false)
{
string agentApplicationDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
if (!useTestMode)
{
APRuntime.Initialize(StringConstants.ApplicationConfigurationFileName);
}
TextLogger.InitializeLogging(StringConstants.CustomLogIdName, useTestMode, StringConstants.TestModeLogFileNameTemplate, StringConstants.TestModeLogDirectoryName, StringConstants.TestModeTraceListenerName);
TextLogger.LogInfo("Starting Service Fabric Autopilot Agent.");
if (!ConfigurationManager.GetCurrentConfigurations(useTestMode))
{
this.Exit(false);
return;
}
TextLogger.SetLogLevel(ConfigurationManager.LogLevel);
ConfigurationManager.LogCurrentConfigurations();
ServiceFabricLocalNodeManager localNodeManager;
if (!ServiceFabricLocalNodeManager.Create(agentApplicationDirectory, useTestMode, out localNodeManager))
{
this.Exit(false);
return;
}
this.localNodeManager = localNodeManager;
this.agentStatus.InitializeAgentStatus();
this.RegisterConsoleCancelEventNotifications();
this.backgroundStateMachineTask = Task.Factory.StartNew(
() => this.RunBackgroundStateMachine(this.backgroundStateMachineCancellationTokenSource.Token),
this.backgroundStateMachineCancellationTokenSource.Token,
TaskCreationOptions.LongRunning,
TaskScheduler.Default);
}
public static bool ReportMachineProperty(string machineName, MachinePropertyLevel propertyLevel, string property, string value)
{
if (useTestMode)
{
TextLogger.LogInfo("Test mode machine property report => machine : {0}, level : {1}, property : {2}, value : {3}.", machineName, propertyLevel, property, value);
return(true);
}
else
{
try
{
UpdateMachinePropertyRequest request = new UpdateMachinePropertyRequest
{
StatusFlag = "always",
Environment = APRuntime.EnvironmentName,
};
request.PropertyList.Add(
new DMMachineProperty
{
MachineName = machineName,
Level = propertyLevel,
Property = property,
Value = value
});
DMServerResponseCode responseCode = dmCommands.UpdateMachineProperty(request);
if (responseCode != DMServerResponseCode.Ok)
{
TextLogger.LogError("Failed to report {0} level machine property {1} against machine {2} with value {3}. Response code : {4}.", propertyLevel, property, machineName, value, responseCode);
return(false);
}
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to report {0} level machine property {1} against machine {2} with value {3} : {4}.", propertyLevel, property, machineName, value, e);
return(false);
}
}
}
private bool ComputeNextStateFromManagingNetworkConfigurationState(out ServiceFabricAutopilotAgentState nextState)
{
nextState = ServiceFabricAutopilotAgentState.None;
try
{
bool serviceFabricNodeConfigurationCompleted = Utilities.IsWindowsFabricNodeConfigurationCompleted(false);
TextLogger.LogInfo("Service Fabric node configuration completed locally : {0}.", serviceFabricNodeConfigurationCompleted);
if (!serviceFabricNodeConfigurationCompleted)
{
nextState = ServiceFabricAutopilotAgentState.InstallingProduct;
}
else
{
ServiceController fabricHostService = this.localNodeManager.GetInstalledService(Constants.FabricHostServiceName);
if (fabricHostService == null)
{
// TODO: consider assert instead of an error message
TextLogger.LogError("Service Fabric node configuration has completed locally while Fabric host service does not exist.");
return(false);
}
if (fabricHostService.Status != ServiceControllerStatus.Running)
{
nextState = ServiceFabricAutopilotAgentState.StartingNode;
}
else
{
nextState = ServiceFabricAutopilotAgentState.NodeStarted;
}
}
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to compute next state from {0} state : {1}", ServiceFabricAutopilotAgentState.ManagingNetworkConfiguration, e);
return(false);
}
}
private async Task RunCertificateManager(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
bool succeeded = this.certificateManager.EnsureEnvironmentCertificatesPresence();
if (succeeded)
{
await Task.Delay(TimeSpan.FromSeconds(ConfigurationManager.CertificateManagerRunIntervalSeconds), cancellationToken);
}
else
{
TextLogger.LogInfo("Certificate manager run failed. Retry after {0} seconds.", ConfigurationManager.CertificateManagerRetryIntervalSeconds);
await Task.Delay(TimeSpan.FromSeconds(ConfigurationManager.CertificateManagerRetryIntervalSeconds), cancellationToken);
}
}
}
private bool GetLocalMachineSku(out string localMachineSku)
{
localMachineSku = null;
if (this.useTestMode)
{
localMachineSku = "TestModeMachineSku";
return(true);
}
// When server pool is enabled, logical machine name would be used. Otherwise, physical machine name would be used.
// Note that in a server pool disabled environment, logical machine names would be the same as physical machine names.
string machineInfoFile = Path.Combine(APRuntime.DataDirectory, StringConstants.MachineInfoFileName);
string localMachineName = APRuntime.MachineName;
try
{
using (CsvReader reader = new CsvReader(machineInfoFile))
{
while (reader.Read())
{
if (string.Compare(reader["MachineName"], localMachineName, StringComparison.OrdinalIgnoreCase) == 0)
{
localMachineSku = reader["SKU"];
TextLogger.LogInfo("Local machine {0} is using machine SKU {1}.", localMachineName, localMachineSku);
return(true);
}
}
}
TextLogger.LogError("Local machine {0} is not in machine info file {1}.", localMachineName, machineInfoFile);
return(false);
}
catch (Exception e)
{
TextLogger.LogError("Failed to get machine SKU of local machine {0} from machine info file {1} : {2}", localMachineName, machineInfoFile, e);
return(false);
}
}
private void LogClusterTopology(ClusterManifestTypeInfrastructureWindowsAzureStaticTopology clusterTopology)
{
TextLogger.LogInfo("Cluster topology from static topology provider:");
TextLogger.LogInfo("NodeName, NodeType, IsSeedNode, UpgradeDomain, FaultDomain, IPAddressOrFQDN");
foreach (FabricNodeType node in clusterTopology.NodeList)
{
TextLogger.LogInfo(
"{0}, {1}, {2}, {3}, {4}, {5}",
node.NodeName,
node.NodeTypeRef,
node.IsSeedNode,
node.UpgradeDomain,
node.FaultDomain,
node.IPAddressOrFQDN
);
}
}
private bool SetDynamicPortRange(int startPort, int endPort)
{
int portCount = endPort - startPort + 1;
TextLogger.LogInfo("Setting dynamic port range. Start port : {0}. Port count : {1}.", startPort, portCount);
if (Socket.OSSupportsIPv4)
{
if (!SetDynamicPortRange(startPort, portCount, "ipv4", "tcp"))
{
TextLogger.LogError("Failed to set dynamic port range for ipv4:tcp. Start port : {0}. Port count : {1}.", startPort, portCount);
return(false);
}
if (!SetDynamicPortRange(startPort, portCount, "ipv4", "udp"))
{
TextLogger.LogError("Failed to set dynamic port range for ipv4:udp. Start port : {0}. Port count : {1}.", startPort, portCount);
return(false);
}
}
if (Socket.OSSupportsIPv6)
{
if (!SetDynamicPortRange(startPort, portCount, "ipv6", "tcp"))
{
TextLogger.LogError("Failed to set dynamic port range for ipv6:tcp. Start port : {0}. Port count : {1}.", startPort, portCount);
return(false);
}
if (!SetDynamicPortRange(startPort, portCount, "ipv6", "udp"))
{
TextLogger.LogError("Failed to set dynamic port range for ipv6:udp. Start port : {0}. Port count : {1}.", startPort, portCount);
return(false);
}
}
return(true);
}
private bool InvokeFabricDeployer(ClusterTopology topologyToDeploy, DeploymentType deploymentType, bool toUseCurrentClusterManifestForDeployment)
{
TextLogger.LogInfo("Invoking FabricDeployer with deployment type = {0} and toUseCurrentClusterManifestForDeployment = {1}.", deploymentType, toUseCurrentClusterManifestForDeployment);
string clusterManifestFileToDeploy;
if (toUseCurrentClusterManifestForDeployment)
{
string currentClusterManifestFile = this.GetCurrentClusterManifestFile(topologyToDeploy.CurrentNodeName);
if (File.Exists(currentClusterManifestFile))
{
TextLogger.LogInfo("Using current cluster manifest file {0} for deployment.", currentClusterManifestFile);
clusterManifestFileToDeploy = currentClusterManifestFile;
}
else
{
TextLogger.LogError("Current cluster manifest file does not exist as {0} when toUseCurrentClusterManifestForDeployment = true.", currentClusterManifestFile);
return(false);
}
}
else
{
clusterManifestFileToDeploy = this.GetBootstrapClusterManifestFile();
}
if (!this.FabricDeploy(deploymentType, clusterManifestFileToDeploy, this.DataRootDirectory, topologyToDeploy.ServiceFabricNodes, true))
{
TextLogger.LogError(
"Failed to invoke FabricDeployer with deployment type = {0}, cluster manifest file = {1}, data root directory = {2}.",
deploymentType,
clusterManifestFileToDeploy,
this.DataRootDirectory);
return(false);
}
return(true);
}
private void ConsoleCancelEventHanlder(object sender, ConsoleCancelEventArgs args)
{
TextLogger.LogInfo("Cancel event (Ctrl + C / Ctrl + Break) received. Stopping Service Fabric Autopilot Agent.");
if (this.agentStatus.GetCurrentState() < ServiceFabricAutopilotAgentState.Started)
{
return;
}
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.Stopping, false);
this.backgroundStateMachineCancellationTokenSource.Cancel();
this.localNodeManager.StopLocalNode();
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.Stopped, false);
TextLogger.LogInfo("Service Fabric Autopilot Agent has stopped. Exiting the process.");
this.Exit(true);
}
private string GetBootstrapClusterManifestFile()
{
string bootstrapClusterManifestFile;
if (ConfigurationManager.BootstrapClusterManifestLocation == BootstrapClusterManifestLocationType.Default || this.useTestMode)
{
bootstrapClusterManifestFile = Path.Combine(this.agentApplicationDirectory, StringConstants.DefaultBootstrapClusterManifestFileName);
}
else if (ConfigurationManager.BootstrapClusterManifestLocation == BootstrapClusterManifestLocationType.EnvironmentDefault)
{
bootstrapClusterManifestFile = Path.Combine(this.agentApplicationDirectory, StringConstants.EnvironmentDefaultConfigDirectoryName, APRuntime.ClusterName, APRuntime.EnvironmentName, StringConstants.DefaultBootstrapClusterManifestFileName);
}
else
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Bootstrap cluster manifest file is not available based on current BootstrapClusterManifestLocation {0}.", ConfigurationManager.BootstrapClusterManifestLocation));
}
TextLogger.LogInfo("Based on bootstrapClusterManifestLocation {0} and useTestMode {1}, bootstrapClusterManifestFile = {2}", ConfigurationManager.BootstrapClusterManifestLocation, this.useTestMode, bootstrapClusterManifestFile);
return(bootstrapClusterManifestFile);
}
public void Start()
{
string applicationDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
this.certificateManager = new EnvironmentCertificateManager(applicationDirectory);
APRuntime.Initialize(StringConstants.ApplicationConfigurationFileName);
TextLogger.InitializeLogging(StringConstants.CustomLogIdName);
TextLogger.LogInfo("Starting Service Fabric Environment Sync.");
if (!ConfigurationManager.GetCurrentConfigurations())
{
this.Exit(false);
return;
}
TextLogger.SetLogLevel(ConfigurationManager.LogLevel);
if (!this.certificateManager.GetEnvironmentCertificatesFromCurrentConfigurations())
{
this.Exit(false);
return;
}
this.RegisterConsoleCancelEventNotifications();
this.certificateManagerTask = Task.Factory.StartNew(
() => this.RunCertificateManager(this.certificateManagerCancellationTokenSource.Token),
this.certificateManagerCancellationTokenSource.Token,
TaskCreationOptions.LongRunning,
TaskScheduler.Default);
}
internal void LogTopology()
{
if (this.ServiceFabricNodes.Count == 0)
{
TextLogger.LogInfo("Static topology defined in cluster manifest + dynamic topology update disabled.");
}
else
{
TextLogger.LogInfo("Static topology defined in cluster manifest + dynamic topology update with {0} nodes", this.ServiceFabricNodes.Count);
TextLogger.LogInfo("NodeName, NodeType, UpgradeDomain, FaultDomain, IPAddressOrFQDN");
foreach (InfrastructureNodeType node in this.ServiceFabricNodes)
{
TextLogger.LogInfo(
"{0}, {1}, {2}, {3}, {4}",
node.NodeName,
node.NodeTypeRef,
node.UpgradeDomain,
node.FaultDomain,
node.IPAddressOrFQDN
);
}
}
}
public bool EnsureEnvironmentCertificatesPresence()
{
foreach (var storeEnvironmentCertificates in this.environmentCertificates)
{
StoreName storeName = storeEnvironmentCertificates.Key;
TextLogger.LogInfo("Ensuring all environment certificates for certificate store {0}\\{1} are present in local environment.", StoreLocation.LocalMachine, storeName);
int installedEnvironmentCertificates = 0;
X509Store store = null;
try
{
store = new X509Store(storeName, StoreLocation.LocalMachine);
store.Open(OpenFlags.OpenExistingOnly | OpenFlags.ReadWrite);
foreach (var environmentCertificateEntry in storeEnvironmentCertificates.Value)
{
bool environmentCertificateInstalled = false;
X509Certificate2 environmentCertificate = environmentCertificateEntry.Value.Certificate;
if (store.Certificates != null)
{
X509Certificate2Collection certificateCollection = store.Certificates.Find(X509FindType.FindByThumbprint, environmentCertificate.Thumbprint, false);
environmentCertificateInstalled = certificateCollection.Count > 0;
}
if (!environmentCertificateInstalled)
{
TextLogger.LogInfo(
"{0} certificate with thumbprint {1}, subject {2} is being installed in certificate store {3}\\{4}.",
environmentCertificate.HasPrivateKey ? "Private key" : "Public key",
environmentCertificate.Thumbprint,
environmentCertificate.Subject,
StoreLocation.LocalMachine,
storeName);
store.Add(environmentCertificate);
if (environmentCertificate.HasPrivateKey)
{
TextLogger.LogInfo(
"Private key certificate with thumbprint {0}, subject {1} is being ACLed.",
environmentCertificate.Thumbprint,
environmentCertificate.Subject);
// ACL private key part of a private key certificate to network service.
RSACryptoServiceProvider cryptoServiceProvider = environmentCertificate.PrivateKey as RSACryptoServiceProvider;
CspParameters cspParameters = new CspParameters(cryptoServiceProvider.CspKeyContainerInfo.ProviderType, cryptoServiceProvider.CspKeyContainerInfo.ProviderName, cryptoServiceProvider.CspKeyContainerInfo.KeyContainerName)
{
Flags = CspProviderFlags.UseExistingKey | CspProviderFlags.UseMachineKeyStore,
CryptoKeySecurity = cryptoServiceProvider.CspKeyContainerInfo.CryptoKeySecurity
};
cspParameters.CryptoKeySecurity.AddAccessRule(new CryptoKeyAccessRule(new SecurityIdentifier(WellKnownSidType.NetworkServiceSid, null), CryptoKeyRights.FullControl | CryptoKeyRights.GenericRead, AccessControlType.Allow));
using (RSACryptoServiceProvider updatedCryptoServiceProvider = new RSACryptoServiceProvider(cspParameters))
{
// Create a new RSACryptoServiceProvider with updated ACL rules to apply ACL changes.
}
}
installedEnvironmentCertificates++;
}
}
}
catch (Exception e)
{
TextLogger.LogError("Failed to ensure all environment certificates for certificate store {0}\\{1} are present in local environment : {2}", StoreLocation.LocalMachine, storeName, e);
return(false);
}
finally
{
if (store != null)
{
store.Close();
}
}
TextLogger.LogInfo("Certificate store {0}\\{1} : Successfully installed {2} environment certificates. All {3} environment certificates referred to in current configurations for the certificate store have been installed.", StoreLocation.LocalMachine, storeName, installedEnvironmentCertificates, storeEnvironmentCertificates.Value.Count);
}
return(true);
}
private void RunStateMachine()
{
bool toRunStateMachineCore = false;
lock (this.agentStatus.StatusLock)
{
if (this.agentStatus.State < ServiceFabricAutopilotAgentState.Started)
{
return;
}
TextLogger.LogVerbose("RunStateMachine : {0}.", this.agentStatus);
if (this.agentStatus.LastFailedDeploymentAction != DateTime.MinValue && (DateTime.UtcNow - this.agentStatus.LastFailedDeploymentAction).TotalSeconds < ConfigurationManager.DeploymentRetryIntervalSeconds)
{
return;
}
// Note that when a deployer operation (node configuration creation or update) fails, it would be retried based on the same cluster manifest (bootstrap or current).
// I.e a retry would happen even when a previous operation failed with the same cluster manifest.
if ((this.agentStatus.State > ServiceFabricAutopilotAgentState.Started && agentStatus.State < ServiceFabricAutopilotAgentState.DiscoveringTopology) ||
(this.agentStatus.State == ServiceFabricAutopilotAgentState.DiscoveringTopology && ConfigurationManager.BootstrapClusterManifestLocation != BootstrapClusterManifestLocationType.NotReady) ||
(this.agentStatus.State > ServiceFabricAutopilotAgentState.DiscoveringTopology && agentStatus.State < ServiceFabricAutopilotAgentState.NodeStarted) ||
(this.agentStatus.State == ServiceFabricAutopilotAgentState.NodeStarted && !this.agentStatus.ExpectedTopologyRunning))
{
TextLogger.LogInfo(
"RunStateMachine: current agent state necessitates RunStateMachineCore. Agent status = {0}, ExpectedTopologyRunning = {1}, LastFailedDeploymentAction = {2}, BootstrapClusterManifestLocation = {3}.",
this.agentStatus.State,
this.agentStatus.ExpectedTopologyRunning,
this.agentStatus.LastFailedDeploymentAction,
ConfigurationManager.BootstrapClusterManifestLocation);
toRunStateMachineCore = true;
}
}
if (toRunStateMachineCore)
{
bool coreStateMachineRunResult;
ServiceFabricAutopilotAgentState currentState;
do
{
currentState = this.agentStatus.GetCurrentState();
if (currentState < ServiceFabricAutopilotAgentState.Started)
{
return;
}
coreStateMachineRunResult = this.RunStateMachineCore(currentState);
TextLogger.LogInfo("RunStateMachine: RunStateMachineCore completed with result {0}. State prior to the run : {1}.", coreStateMachineRunResult, currentState);
if (!coreStateMachineRunResult)
{
lock (agentStatus.StatusLock)
{
this.agentStatus.LastFailedDeploymentAction = DateTime.UtcNow;
}
}
else
{
lock (agentStatus.StatusLock)
{
this.agentStatus.LastFailedDeploymentAction = DateTime.MinValue;
}
}
currentState = this.agentStatus.GetCurrentState();
}while (currentState > ServiceFabricAutopilotAgentState.Started && currentState < ServiceFabricAutopilotAgentState.NodeStarted && coreStateMachineRunResult);
}
}
public static int ExecuteCommand(string command, string arguments, string workingDirectory, TimeSpan timeout)
{
TextLogger.LogInfo("Executing command {0} with arguments = {1}, working directory = {2}, timeout = {3} seconds.", command, arguments, workingDirectory, timeout == TimeSpan.MaxValue ? "infinite" : timeout.TotalSeconds.ToString());
using (Process process = new Process())
{
process.StartInfo.FileName = command;
process.StartInfo.Arguments = arguments;
process.StartInfo.UseShellExecute = false;
process.StartInfo.CreateNoWindow = true;
process.StartInfo.WorkingDirectory = workingDirectory;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.RedirectStandardError = true;
ProcessStandardOutputHandler processStandardOutputHandler = new ProcessStandardOutputHandler();
process.OutputDataReceived += new DataReceivedEventHandler(processStandardOutputHandler.StandardOutputHandler);
process.ErrorDataReceived += new DataReceivedEventHandler(processStandardOutputHandler.StandardErrorHandler);
process.Start();
process.BeginOutputReadLine();
process.BeginErrorReadLine();
int exitCode;
if (timeout == TimeSpan.MaxValue)
{
process.WaitForExit();
TextLogger.LogInfo("Commond execution completed with exit code {0}.", process.ExitCode);
exitCode = process.ExitCode;
}
else
{
if (!process.WaitForExit((int)timeout.TotalMilliseconds))
{
process.Kill();
TextLogger.LogError("Execution of command {0} with arguments {1} timed out after {2} seconds. The process is terminated.", command, arguments, timeout.TotalSeconds);
exitCode = -1;
}
else
{
TextLogger.LogInfo("Commond execution completed with exit code {0}.", process.ExitCode);
exitCode = process.ExitCode;
}
}
if (!string.IsNullOrEmpty(processStandardOutputHandler.StandardOutput))
{
TextLogger.LogInfo("Standard output : {0}", processStandardOutputHandler.StandardOutput);
}
if (!string.IsNullOrEmpty(processStandardOutputHandler.StandardError))
{
TextLogger.LogInfo("Standard error : {0}", processStandardOutputHandler.StandardError);
}
return(exitCode);
}
}
private bool RunStateMachineCore(ServiceFabricAutopilotAgentState currentState)
{
TextLogger.LogInfo("Starting RunStateMachineCore. Current state : {0}.", currentState);
switch (currentState)
{
case ServiceFabricAutopilotAgentState.ManagingNetworkConfiguration:
{
if (!this.localNodeManager.ManageNetworkConfiguration())
{
return(false);
}
ServiceFabricAutopilotAgentState nextState;
if (!this.ComputeNextStateFromManagingNetworkConfigurationState(out nextState))
{
return(false);
}
this.agentStatus.SetNextState(nextState);
break;
}
case ServiceFabricAutopilotAgentState.InstallingProduct:
{
if (!this.localNodeManager.InstallProduct())
{
return(false);
}
// For now, we assume the topology is static.
// TODO: when adding dynamic topology support, change state machine transition here.
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.DiscoveringTopology);
break;
}
case ServiceFabricAutopilotAgentState.DiscoveringTopology:
{
ClusterTopology expectedTopology;
if (!this.localNodeManager.DiscoverTopology(out expectedTopology))
{
return(false);
}
lock (this.agentStatus.StatusLock)
{
this.agentStatus.ExpectedTopology = expectedTopology;
this.agentStatus.ExpectedTopologyRunning = false;
}
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.ConfiguringNode);
break;
}
case ServiceFabricAutopilotAgentState.ConfiguringNode:
{
ClusterTopology topologyToDeploy = this.agentStatus.GetTopologyToDeploy();
if (!this.localNodeManager.ConfigureLocalNode(topologyToDeploy))
{
return(false);
}
lock (this.agentStatus.StatusLock)
{
this.agentStatus.DeployedTopology = topologyToDeploy;
this.agentStatus.ExpectedTopologyRunning = !this.agentStatus.DeployedTopology.IsClusterTopologyChanged(this.agentStatus.ExpectedTopology);
}
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.StartingNode);
break;
}
case ServiceFabricAutopilotAgentState.StartingNode:
{
if (!this.localNodeManager.StartLocalNode())
{
return(false);
}
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.NodeStarted);
break;
}
case ServiceFabricAutopilotAgentState.NodeStarted:
{
// TODO: this only applies when DynamicTopologyUpdateMode == None or OnNodeConfigurationOnly.
// When we want to update topology whenever topology changes, this needs to change.
lock (this.agentStatus.StatusLock)
{
this.agentStatus.ExpectedTopologyRunning = true;
this.agentStatus.ExpectedTopology = ClusterTopology.GetClusterTopology();
this.agentStatus.DeployedTopology = this.agentStatus.ExpectedTopology;
}
break;
}
}
return(true);
}
public bool DiscoverTopology(out ClusterTopology expectedTopology)
{
expectedTopology = null;
try
{
TextLogger.LogInfo("Discovering cluster topology from bootstrap cluster manifest file.");
if (ConfigurationManager.BootstrapClusterManifestLocation == BootstrapClusterManifestLocationType.NotReady)
{
TextLogger.LogWarning("Bootstrap cluster manifest is not ready. Cluster topology would not be discovered until bootstrap cluster manifest is ready.");
return(false);
}
ClusterManifestType bootstrapClusterManifest;
if (!this.ReadBootstrapClusterManifestFile(out bootstrapClusterManifest))
{
return(false);
}
TopologyProviderType topologyProviderType = TopologyProvider.GetTopologyProviderTypeFromClusterManifest(bootstrapClusterManifest);
TextLogger.LogInfo("Topology provider type : {0}.", topologyProviderType);
if (topologyProviderType != TopologyProviderType.StaticTopologyProvider)
{
TextLogger.LogInfo("Topology provider type {0} is not supported to provide static topology.", topologyProviderType);
return(false);
}
else
{
/*
* Static topology provider defines the authoritative cluster topology in the cluster manifest.
* If the local machine (physical or virtual) is not part of the cluster topology specified in static topology provider section, a Service Fabric node will not be started locally.
* During scale-up, bootstrap cluster manifest has to be upgraded to allow new Service Fabric nodes to be started on the new machines and join the ring.
* A scale-up is normally a two-phase process with allocation of new machines (and their IPs) and generation of a new bootstrap cluster manifest with updated topology.
*/
ClusterManifestTypeInfrastructureWindowsAzureStaticTopology staticTopologyProviderElement = TopologyProvider.GetStaticTopologyProviderElementFromClusterManifest(bootstrapClusterManifest);
if (staticTopologyProviderElement.NodeList == null || staticTopologyProviderElement.NodeList.Length == 0)
{
TextLogger.LogError("Static topology provider section of bootstrap cluster manifest does not specify topology of the Service Fabric cluster.");
return(false);
}
LogClusterTopology(staticTopologyProviderElement);
ClusterTopology staticTopologyProviderClusterTopology = ClusterTopology.GetClusterTopology();
staticTopologyProviderClusterTopology.LogTopology();
expectedTopology = staticTopologyProviderClusterTopology;
TextLogger.LogInfo("Successfully discovered cluster topology.");
return(true);
}
}
catch (Exception e)
{
TextLogger.LogError("Failed to discover cluster topology : {0}", e);
return(false);
}
}
/*
* Private key certificate files (each private key certificate has an encrypted private key file (.pfx.encr) and an encrypted password file (.pfx.password.encr) ) as well as public key certificate files (i.e. cer files) are part of ServiceFabricEnvironmentSync application packages.
*
* All certificates to be installed by ServiceFabricEnvironmentSync need to be listed in the form of "CertificateFriendlyName=CertificateDataDirectory" in certificate section of ServiceFabricEnvironmentSync configuration file.
* 1) CertificateDataDirectory refers to a relative path from application package root directory.
* 2) For a public key certificate, CertificateFriendlyName.cer should be present in CertificateDataDirectory.
* 3) For a private key certificate, CertificateFriendlyName.pfx.encr + CertificateFriendlyName.password.encr should be present in CertificateDataDirectory.
*
* All certificates listed would be installed in LocalMachine\My or LocalMachine\CA certificate store.
*
* Sample certificate configuration sections
* [ServiceFabricEnvironmentSync.Certificate.LocalMachine.My]
* certA=data\certificates
* certB=data\certificates
*
* [ServiceFabricEnvironmentSync.Certificate.LocalMachine.CertificateAuthority]
* certC=data\certificates
* certD=data\certificates
*/
public bool GetEnvironmentCertificatesFromCurrentConfigurations(StoreName storeName)
{
try
{
if (storeName != StoreName.My && storeName != StoreName.CertificateAuthority)
{
TextLogger.LogError("Certificate store name {0} is not supported. Supported store names : {1}, {2}.", storeName, StoreName.My, StoreName.CertificateAuthority);
return(false);
}
string certificateListConfigurationSectionName = string.Format(CultureInfo.InvariantCulture, StringConstants.CertificateListConfigurationSectionNameTemplate, storeName);
IConfiguration configuration = APConfiguration.GetConfiguration();
if (configuration != null && configuration.SectionExists(certificateListConfigurationSectionName))
{
string[] certificateFriendlyNames = configuration.GetSectionKeys(certificateListConfigurationSectionName);
if (certificateFriendlyNames != null)
{
foreach (string certificateFriendlyName in certificateFriendlyNames)
{
string certificateDataDirectoryName = configuration.GetStringValue(certificateListConfigurationSectionName, certificateFriendlyName);
string certificateDataDirectory = Path.Combine(this.applicationDirectory, certificateDataDirectoryName);
string encryptedPrivateKeyFile = Path.Combine(
certificateDataDirectory,
string.Format(
CultureInfo.InvariantCulture,
StringConstants.EncryptedPrivateKeyFileNameTemplate,
certificateFriendlyName));
bool encryptedPrivateKeyFileExists = File.Exists(encryptedPrivateKeyFile);
string encryptedPasswordFile = Path.Combine(
certificateDataDirectory,
string.Format(
CultureInfo.InvariantCulture,
StringConstants.EncryptedPasswordFileNameTemplate,
certificateFriendlyName));
bool encryptedPasswordFileExists = File.Exists(encryptedPasswordFile);
string publicKeyFile = Path.Combine(
certificateDataDirectory,
string.Format(
CultureInfo.InvariantCulture,
StringConstants.PublicKeyFileNameTemplate,
certificateFriendlyName));
bool publicKeyFileExists = File.Exists(publicKeyFile);
// TODO: report health
if (!encryptedPrivateKeyFileExists && !encryptedPasswordFileExists && !publicKeyFileExists)
{
TextLogger.LogError("Neither encrypted private key & encrypted password files nor public key file exist for certificate {0} in certificate data directory {1}. A certificate to be installed needs to be either a public key certificate or a private key certificate.", certificateFriendlyName, certificateDataDirectory);
return(false);
}
if ((encryptedPrivateKeyFileExists || encryptedPasswordFileExists) && publicKeyFileExists)
{
TextLogger.LogError("Both encrypted private key & encrypted password files and public key file exist for certificate {0} in certificate data directory {1}. A certificate to be installed needs to be either a public key certificate or a private key certificate but not both.", certificateFriendlyName, certificateDataDirectory);
return(false);
}
if (encryptedPrivateKeyFileExists ^ encryptedPasswordFileExists)
{
TextLogger.LogError("Only one of encrypted private key file and encrypted password file exist for certificate {0} in certificate data directory {1}. Both are required to install a private key certificate.", certificateFriendlyName, certificateDataDirectory);
return(false);
}
if (publicKeyFileExists)
{
PublicKeyCertificate publicKeyCertificate;
if (!PublicKeyCertificate.TryLoadFromFile(publicKeyFile, out publicKeyCertificate))
{
return(false);
}
if (this.environmentCertificates[storeName].ContainsKey(publicKeyCertificate.Thumbprint) && this.environmentCertificates[storeName][publicKeyCertificate.Thumbprint].HasPrivateKey)
{
TextLogger.LogError("Certificate with thumbprint {0}, subject {1} are requested to be installed as both public key certificate and private key certificate in configuration under different entries.", publicKeyCertificate.Thumbprint, publicKeyCertificate.Certificate.Subject);
return(false);
}
this.environmentCertificates[storeName][publicKeyCertificate.Thumbprint] = publicKeyCertificate;
TextLogger.LogInfo("Public key certificate with thumbprint {0}, subject {1} needs to be present in certificate store {2}\\{3} in local environment based on current configurations.", publicKeyCertificate.Thumbprint, publicKeyCertificate.Certificate.Subject, StoreLocation.LocalMachine, storeName);
}
else
{
if (storeName == StoreName.CertificateAuthority)
{
TextLogger.LogError("Private key certificate {0} cannot be installed in certificate store with name {1}.", certificateFriendlyName, storeName);
return(false);
}
PrivateKeyCertificate privateKeyCertificate;
if (!PrivateKeyCertificate.TryLoadFromFile(encryptedPrivateKeyFile, encryptedPasswordFile, out privateKeyCertificate))
{
return(false);
}
if (this.environmentCertificates[storeName].ContainsKey(privateKeyCertificate.Thumbprint) && !this.environmentCertificates[storeName][privateKeyCertificate.Thumbprint].HasPrivateKey)
{
TextLogger.LogError("Certificate with thumbprint {0}, subject {1} are requested to be installed as both public key certificate and private key certificate in configuration under different entries.", privateKeyCertificate.Thumbprint, privateKeyCertificate.Certificate.Subject);
return(false);
}
this.environmentCertificates[storeName][privateKeyCertificate.Thumbprint] = privateKeyCertificate;
TextLogger.LogInfo("Private key certificate with thumbprint {0}, subject {1} needs to be present in certificate store {2}\\{3} in local environment based on current configurations.", privateKeyCertificate.Thumbprint, privateKeyCertificate.Certificate.Subject, StoreLocation.LocalMachine, storeName);
}
}
}
}
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to get environment certificates based on current configurations : {0}", e);
return(false);
}
}
public bool StartLocalNode()
{
try
{
TextLogger.LogInfo("Starting local Service Fabric node.");
// TODO: After current xcopyable issue is fixed, update the pattern here.
// TODO: [General] disposal of ServiceController
ServiceController fabricHostService = this.GetInstalledService(Constants.FabricHostServiceName);
if (fabricHostService != null)
{
if (fabricHostService.Status == ServiceControllerStatus.StartPending)
{
TextLogger.LogWarning("Fabric host service is starting.");
return(false);
}
else if (fabricHostService.Status == ServiceControllerStatus.Running)
{
TextLogger.LogInfo("Successfully started local Service Fabric node.");
return(true);
}
else if (!this.StartFabricHostService())
{
TextLogger.LogError("Failed to start local Service Fabric node ");
return(false);
}
}
else
{
ServiceController fabricInstallerService = this.GetInstalledService(Constants.FabricInstallerServiceName);
// Install and configure Fabric installer service if not installed.
if (fabricInstallerService == null)
{
TextLogger.LogInfo("Installing and configuring Fabric installer service.");
if (!this.InstallAndConfigureFabricInstallerService(this.agentApplicationDirectory))
{
TextLogger.LogError("Failed to install and configure Fabric installer service");
return(false);
}
fabricInstallerService = this.GetInstalledService(Constants.FabricInstallerServiceName);
}
// If Fabric installer service is not in stopped state, stop it.
if (fabricInstallerService.Status != ServiceControllerStatus.Stopped)
{
TextLogger.LogInfo("Fabric installer service is in {0} state. Stopping it.", fabricInstallerService.Status);
if (!this.StopFabricInstallerService())
{
TextLogger.LogError("Failed to stop Fabric installer service.");
return(false);
}
}
// Start Fabric installer service to start Fabric host service.
TextLogger.LogInfo("Starting Fabric installer service to start Fabric host service.");
if (!this.StartFabricInstallerService())
{
TextLogger.LogError("Failed to start Fabric installer service.");
return(false);
}
}
TextLogger.LogInfo("Successfully started local Service Fabric node.");
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to start local Service Fabric node : {0}", e);
return(false);
}
}
private bool GetAndCreateDataRootDirectory(out string dataRootDirectory)
{
dataRootDirectory = null;
try
{
// On AP machines, data root would depend on machine SKU (specifically the SKU's disk layout). For test mode testing, use C:\SFRoot.
if (this.useTestMode)
{
dataRootDirectory = Path.Combine(Path.GetPathRoot("C:\\"), StringConstants.ServiceFabricRootDirectoryName);
TextLogger.LogInfo("Using {0} as data root directory in test mode.", dataRootDirectory);
}
else
{
string diskLayoutConfigurationFile = Path.Combine(this.agentApplicationDirectory, StringConstants.DiskLayoutConfigurationFileName);
TextLogger.LogInfo("Data root directory would be determined based on local machine SKU {0} and disk layout configuration file {1}.", this.LocalMachineSku, diskLayoutConfigurationFile);
string dataRootHint = null;
using (CsvReader reader = new CsvReader(diskLayoutConfigurationFile))
{
while (reader.Read())
{
if (Regex.IsMatch(this.LocalMachineSku, reader["SKU"], RegexOptions.IgnoreCase | RegexOptions.CultureInvariant))
{
dataRootHint = Path.GetFullPath(Environment.ExpandEnvironmentVariables(reader["DataRootHint"]));
TextLogger.LogInfo(
"Local machine SKU {0} matched disk layout SKU rule '{1}' -> '{2}' which expands to data root '{3}'",
this.LocalMachineSku,
reader["SKU"],
reader["DataRootHint"],
dataRootHint);
break;
}
}
}
if (string.IsNullOrEmpty(dataRootHint))
{
TextLogger.LogError("Disk layout configuration file {0} does not contain a valid entry for local machine SKU {1}.", diskLayoutConfigurationFile, this.LocalMachineSku);
return(false);
}
dataRootDirectory = Path.Combine(Path.GetPathRoot(dataRootHint), StringConstants.ServiceFabricRootDirectoryName);
TextLogger.LogInfo("Using {0} as data root directory based on local machine SKU {1} and disk layout configuration file {2}.", dataRootDirectory, this.LocalMachineSku, diskLayoutConfigurationFile);
}
TextLogger.LogInfo("Creating data root directory {0}.", dataRootDirectory);
Directory.CreateDirectory(dataRootDirectory);
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to get and create data root directory : {0}", e);
return(false);
}
}