public static bool TryLoadFromFile(string publicKeyFile, out PublicKeyCertificate publicKeyCertificate)
{
publicKeyCertificate = null;
try
{
X509Certificate2 certificate = new X509Certificate2(publicKeyFile);
if (certificate.HasPrivateKey)
{
TextLogger.LogError("Certificate from public key file {0} should not present a private key.", publicKeyFile);
return(false);
}
publicKeyCertificate = new PublicKeyCertificate(certificate);
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to load public key certificate from public key file {0} : {1}", publicKeyFile, e);
return(false);
}
}
public void StopLocalNode()
{
// AP service manager waits for ShutdownTimeoutInMilliseconds (by default 5s) before forcefully terminating the service host.
// For now we are not waiting for Fabric host service to stop on stopping path.
ServiceController fabricInstallerService = this.GetInstalledService(Constants.FabricInstallerServiceName);
if (fabricInstallerService != null)
{
if (fabricInstallerService.Status != ServiceControllerStatus.StopPending && fabricInstallerService.Status != ServiceControllerStatus.Stopped)
{
TextLogger.LogInfo("Stopping Fabric installer serivce.");
fabricInstallerService.Stop();
}
}
ServiceController fabricHostService = this.GetInstalledService(Constants.FabricHostServiceName);
if (fabricHostService != null)
{
if (fabricHostService.Status != ServiceControllerStatus.StopPending && fabricHostService.Status != ServiceControllerStatus.Stopped)
{
TextLogger.LogInfo("Stopping Fabric host serivce.");
fabricHostService.Stop();
}
}
}
/*
* TODO: currently AP safe config deployment is not supported to update configurations.
* The only way to update the configurations is via an application upgrade.
*/
public static bool GetCurrentConfigurations()
{
IConfiguration configuration = APConfiguration.GetConfiguration();
CertificateManagerRunIntervalSeconds = configuration.GetInt32Value(StringConstants.CertificateManagerConfigurationSectionName, "CertificateManagerRunIntervalSeconds", 3600);
CertificateManagerRetryIntervalSeconds = configuration.GetInt32Value(StringConstants.CertificateManagerConfigurationSectionName, "CertificateManagerRetryIntervalSeconds", 300);
EnableSecretStorePersistentCaching = configuration.GetBoolValue(StringConstants.CertificateManagerConfigurationSectionName, "EnableSecretStoreClientSidePersistentCaching", true);
string logLevelString = configuration.GetStringValue(StringConstants.EnvironmentSyncConfigurationSectionName, "LogLevel", "Info");
LogLevel logLevel;
if (!Enum.TryParse <LogLevel>(logLevelString, out logLevel))
{
TextLogger.LogError("Log level is invalid : {0}.", logLevelString);
return(false);
}
LogLevel = logLevel;
return(true);
}
public bool ConfigureLocalNode(ClusterTopology topologyToDeploy)
{
try
{
TextLogger.LogInfo("Configuring local Service Fabric node based on bootstrap cluster manifest.");
if (!this.InvokeFabricDeployer(topologyToDeploy, DeploymentType.Create, false))
{
TextLogger.LogError("Failed to configure local Service Fabric node based on bootstrap cluster manifest.");
return(false);
}
Utilities.SetWindowsFabricNodeConfigurationCompleted(false);
TextLogger.LogInfo("Successfully configured local Service Fabric node based on bootstrap cluster manifest.");
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to configure local Service Fabric node based on bootstrap cluster manifest : {0}", e);
return(false);
}
}
public bool InstallProduct()
{
TextLogger.LogInfo("Installing Service Fabric product.");
try
{
string installedServiceFabricProductVersion = Utilities.GetCurrentWindowsFabricVersion();
if (installedServiceFabricProductVersion == null)
{
/*
* For Service Fabric on Autopilot, xcopyable deployment is the only supported mechanism for installing Service Fabric product.
* MSI is not supported.
*/
string serviceFabricCabFile = this.GetServiceFabricCabFile();
if (File.Exists(serviceFabricCabFile))
{
TextLogger.LogInfo("Installing Service Fabric product with cab file {0}.", serviceFabricCabFile);
string localInstallationPath = Environment.ExpandEnvironmentVariables(Constants.FabricRootPath);
if (!Directory.Exists(localInstallationPath))
{
Directory.CreateDirectory(localInstallationPath);
}
Utilities.UnCabFile(serviceFabricCabFile, localInstallationPath);
Utilities.SetFabricRoot(localInstallationPath);
Utilities.SetFabricCodePath(Path.Combine(localInstallationPath, Constants.PathToFabricCode));
// TODO: registry key not available yet
installedServiceFabricProductVersion = Utilities.GetCurrentWindowsFabricVersion();
TextLogger.LogInfo("Successfully installed Service Fabric product version {0} to local path {1} and set Service Fabric registry keys.", installedServiceFabricProductVersion, localInstallationPath);
}
else
{
// TODO: [General] surface fatal errors outside the agent as AP health report.
TextLogger.LogError("Failed to install Service Fabric product : Service Fabric product cab file does not exist in agent application directory as {0}.", serviceFabricCabFile);
return(false);
}
}
else
{
TextLogger.LogInfo("Service Fabric product version {0} is already installed locally. Skip product installation.", installedServiceFabricProductVersion);
}
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to install Service Fabric product : {0}", e);
return(false);
}
}
private void ConsoleCancelEventHanlder(object sender, ConsoleCancelEventArgs args)
{
TextLogger.LogInfo("Cancel event (Ctrl + C / Ctrl + Break) received. Stopping Service Fabric Environment Sync.");
this.certificateManagerCancellationTokenSource.Cancel();
TextLogger.LogInfo("Service Fabric Environment Sync has stopped. Exiting the process.");
this.Exit(true);
}
public static void LogCurrentConfigurations()
{
TextLogger.LogInfo("Service Fabric Autopilot Agent configurations:");
TextLogger.LogInfo("BackgroundStateMachineRunIntervalSeconds = {0}", BackgroundStateMachineRunIntervalSeconds);
TextLogger.LogInfo("DeploymentRetryIntervalSeconds = {0}", DeploymentRetryIntervalSeconds);
TextLogger.LogInfo("DynamicTopologyUpdateMode = {0}", DynamicTopologyUpdateMode);
TextLogger.LogInfo("BootstrapClusterManifestLocation = {0}", BootstrapClusterManifestLocation);
TextLogger.LogInfo("DynamicPortRangeStart = {0}", DynamicPortRangeStart);
TextLogger.LogInfo("DynamicPortRangeEnd = {0}", DynamicPortRangeEnd);
TextLogger.LogInfo("PortFowardingRulesCount = {0}", PortFowardingRules.Count);
}
public static bool TryLoadFromFile(string encryptedPrivateKeyFile, string encryptedPasswordFile, out PrivateKeyCertificate privateKeyCertificate)
{
privateKeyCertificate = null;
try
{
byte[] encryptedPrivateKeyBytes = File.ReadAllBytes(encryptedPrivateKeyFile);
byte[] encryptedPasswordBytes = File.ReadAllBytes(encryptedPasswordFile);
byte[] decryptedPrivateKeyBlob;
byte[] decryptedPasswordBlob;
using (ApSecretProtection protect = new ApSecretProtection(ConfigurationManager.EnableSecretStorePersistentCaching))
{
decryptedPrivateKeyBlob = protect.Decrypt(encryptedPrivateKeyBytes);
decryptedPasswordBlob = protect.Decrypt(encryptedPasswordBytes);
}
string decryptedPassword;
using (MemoryStream stream = new MemoryStream(decryptedPasswordBlob))
using (StreamReader reader = new StreamReader(stream))
{
decryptedPassword = reader.ReadToEnd().Trim();
}
X509Certificate2 certificate = new X509Certificate2(decryptedPrivateKeyBlob, decryptedPassword, X509KeyStorageFlags.PersistKeySet | X509KeyStorageFlags.MachineKeySet);
if (!certificate.HasPrivateKey)
{
TextLogger.LogError("Certificate from encrypted private key file {0} should present a private key.", encryptedPrivateKeyFile);
return(false);
}
if (!(certificate.PrivateKey is RSACryptoServiceProvider))
{
TextLogger.LogError("Private key certificate from encrypted private key file {0} does not use a RSA based asymmetric algorithm implementation.", encryptedPrivateKeyFile);
return(false);
}
privateKeyCertificate = new PrivateKeyCertificate(certificate);
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to load private key certificate from encrypted private key file {0} / encrypted password file {1} : {2}", encryptedPrivateKeyFile, encryptedPasswordFile, e);
return(false);
}
}
private bool ApplyPortForwardingRule(PortFowardingRule portForwardingRule)
{
if (string.Compare(portForwardingRule.MachineFunction, "*", StringComparison.OrdinalIgnoreCase) == 0 || string.Compare(portForwardingRule.MachineFunction, APRuntime.MachineFunction, StringComparison.OrdinalIgnoreCase) == 0)
{
TextLogger.LogInfo("Applying port forwarding rule {0} from port {1} to port {2}.", portForwardingRule.RuleName, portForwardingRule.ListenPort, portForwardingRule.ConnectPort);
return(this.CreatePortProxy(IPAddress.Any.ToString(), portForwardingRule.ListenPort, this.LocalMachineIpAddress, portForwardingRule.ConnectPort));
}
else
{
TextLogger.LogInfo("Port forwarding rule {0} does not apply to machine function {1}.", portForwardingRule.RuleName, APRuntime.MachineFunction);
return(true);
}
}
public void SetNextState(ServiceFabricAutopilotAgentState nextState, bool toIgnoreStateTransitionWhenStopping = true)
{
lock (this.StatusLock)
{
if (toIgnoreStateTransitionWhenStopping && this.State < ServiceFabricAutopilotAgentState.Started)
{
TextLogger.LogVerbose("Application is stopping. Ignore state transition request to {0}.", nextState);
return;
}
TextLogger.LogInfo("State transition: {0} -> {1}", this.State, nextState);
this.State = nextState;
}
}
/*
* Test mode allows testing of ServiceFabricAutopilotAgent outside an Autopilot hosting environment.
* In test mode, all dependencies on AP Runtime are replaced by test environment equivalences:
* Logging => text log
* Configuration => default test configurations
*/
public void Start(bool useTestMode = false)
{
string agentApplicationDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
if (!useTestMode)
{
APRuntime.Initialize(StringConstants.ApplicationConfigurationFileName);
}
TextLogger.InitializeLogging(StringConstants.CustomLogIdName, useTestMode, StringConstants.TestModeLogFileNameTemplate, StringConstants.TestModeLogDirectoryName, StringConstants.TestModeTraceListenerName);
TextLogger.LogInfo("Starting Service Fabric Autopilot Agent.");
if (!ConfigurationManager.GetCurrentConfigurations(useTestMode))
{
this.Exit(false);
return;
}
TextLogger.SetLogLevel(ConfigurationManager.LogLevel);
ConfigurationManager.LogCurrentConfigurations();
ServiceFabricLocalNodeManager localNodeManager;
if (!ServiceFabricLocalNodeManager.Create(agentApplicationDirectory, useTestMode, out localNodeManager))
{
this.Exit(false);
return;
}
this.localNodeManager = localNodeManager;
this.agentStatus.InitializeAgentStatus();
this.RegisterConsoleCancelEventNotifications();
this.backgroundStateMachineTask = Task.Factory.StartNew(
() => this.RunBackgroundStateMachine(this.backgroundStateMachineCancellationTokenSource.Token),
this.backgroundStateMachineCancellationTokenSource.Token,
TaskCreationOptions.LongRunning,
TaskScheduler.Default);
}
public static bool ReportMachineProperty(string machineName, MachinePropertyLevel propertyLevel, string property, string value)
{
if (useTestMode)
{
TextLogger.LogInfo("Test mode machine property report => machine : {0}, level : {1}, property : {2}, value : {3}.", machineName, propertyLevel, property, value);
return(true);
}
else
{
try
{
UpdateMachinePropertyRequest request = new UpdateMachinePropertyRequest
{
StatusFlag = "always",
Environment = APRuntime.EnvironmentName,
};
request.PropertyList.Add(
new DMMachineProperty
{
MachineName = machineName,
Level = propertyLevel,
Property = property,
Value = value
});
DMServerResponseCode responseCode = dmCommands.UpdateMachineProperty(request);
if (responseCode != DMServerResponseCode.Ok)
{
TextLogger.LogError("Failed to report {0} level machine property {1} against machine {2} with value {3}. Response code : {4}.", propertyLevel, property, machineName, value, responseCode);
return(false);
}
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to report {0} level machine property {1} against machine {2} with value {3} : {4}.", propertyLevel, property, machineName, value, e);
return(false);
}
}
}
private bool ComputeNextStateFromManagingNetworkConfigurationState(out ServiceFabricAutopilotAgentState nextState)
{
nextState = ServiceFabricAutopilotAgentState.None;
try
{
bool serviceFabricNodeConfigurationCompleted = Utilities.IsWindowsFabricNodeConfigurationCompleted(false);
TextLogger.LogInfo("Service Fabric node configuration completed locally : {0}.", serviceFabricNodeConfigurationCompleted);
if (!serviceFabricNodeConfigurationCompleted)
{
nextState = ServiceFabricAutopilotAgentState.InstallingProduct;
}
else
{
ServiceController fabricHostService = this.localNodeManager.GetInstalledService(Constants.FabricHostServiceName);
if (fabricHostService == null)
{
// TODO: consider assert instead of an error message
TextLogger.LogError("Service Fabric node configuration has completed locally while Fabric host service does not exist.");
return(false);
}
if (fabricHostService.Status != ServiceControllerStatus.Running)
{
nextState = ServiceFabricAutopilotAgentState.StartingNode;
}
else
{
nextState = ServiceFabricAutopilotAgentState.NodeStarted;
}
}
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to compute next state from {0} state : {1}", ServiceFabricAutopilotAgentState.ManagingNetworkConfiguration, e);
return(false);
}
}
private async Task RunCertificateManager(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
bool succeeded = this.certificateManager.EnsureEnvironmentCertificatesPresence();
if (succeeded)
{
await Task.Delay(TimeSpan.FromSeconds(ConfigurationManager.CertificateManagerRunIntervalSeconds), cancellationToken);
}
else
{
TextLogger.LogInfo("Certificate manager run failed. Retry after {0} seconds.", ConfigurationManager.CertificateManagerRetryIntervalSeconds);
await Task.Delay(TimeSpan.FromSeconds(ConfigurationManager.CertificateManagerRetryIntervalSeconds), cancellationToken);
}
}
}
private bool GetLocalMachineSku(out string localMachineSku)
{
localMachineSku = null;
if (this.useTestMode)
{
localMachineSku = "TestModeMachineSku";
return(true);
}
// When server pool is enabled, logical machine name would be used. Otherwise, physical machine name would be used.
// Note that in a server pool disabled environment, logical machine names would be the same as physical machine names.
string machineInfoFile = Path.Combine(APRuntime.DataDirectory, StringConstants.MachineInfoFileName);
string localMachineName = APRuntime.MachineName;
try
{
using (CsvReader reader = new CsvReader(machineInfoFile))
{
while (reader.Read())
{
if (string.Compare(reader["MachineName"], localMachineName, StringComparison.OrdinalIgnoreCase) == 0)
{
localMachineSku = reader["SKU"];
TextLogger.LogInfo("Local machine {0} is using machine SKU {1}.", localMachineName, localMachineSku);
return(true);
}
}
}
TextLogger.LogError("Local machine {0} is not in machine info file {1}.", localMachineName, machineInfoFile);
return(false);
}
catch (Exception e)
{
TextLogger.LogError("Failed to get machine SKU of local machine {0} from machine info file {1} : {2}", localMachineName, machineInfoFile, e);
return(false);
}
}
private void LogClusterTopology(ClusterManifestTypeInfrastructureWindowsAzureStaticTopology clusterTopology)
{
TextLogger.LogInfo("Cluster topology from static topology provider:");
TextLogger.LogInfo("NodeName, NodeType, IsSeedNode, UpgradeDomain, FaultDomain, IPAddressOrFQDN");
foreach (FabricNodeType node in clusterTopology.NodeList)
{
TextLogger.LogInfo(
"{0}, {1}, {2}, {3}, {4}, {5}",
node.NodeName,
node.NodeTypeRef,
node.IsSeedNode,
node.UpgradeDomain,
node.FaultDomain,
node.IPAddressOrFQDN
);
}
}
private bool ReadBootstrapClusterManifestFile(out ClusterManifestType bootstrapClusterManifest)
{
bootstrapClusterManifest = null;
string bootstrapClusterManifestFile = this.GetBootstrapClusterManifestFile();
try
{
if (!File.Exists(bootstrapClusterManifestFile))
{
TextLogger.LogError("Bootstrap cluster manifest does not exist in application directory as {0}.", bootstrapClusterManifestFile);
return(false);
}
/*
* As part of application directory, the bootstrap cluster manifest would only be updated via application upgrades where the agent host would be stopped prior to the update.
*/
using (FileStream fileStream = File.Open(bootstrapClusterManifestFile, FileMode.Open, FileAccess.Read, FileShare.Read))
{
// TODO: validation against schema
XmlReaderSettings xmlReaderSettings = new XmlReaderSettings();
xmlReaderSettings.ValidationType = ValidationType.None;
xmlReaderSettings.XmlResolver = null;
using (XmlReader xmlReader = XmlReader.Create(fileStream, xmlReaderSettings))
{
XmlSerializer serializer = new XmlSerializer(typeof(ClusterManifestType));
bootstrapClusterManifest = (ClusterManifestType)serializer.Deserialize(xmlReader);
}
}
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to read bootstrap cluster manifest file {0} : {1}", bootstrapClusterManifestFile, e);
return(false);
}
}
private bool SetDynamicPortRange(int startPort, int endPort)
{
int portCount = endPort - startPort + 1;
TextLogger.LogInfo("Setting dynamic port range. Start port : {0}. Port count : {1}.", startPort, portCount);
if (Socket.OSSupportsIPv4)
{
if (!SetDynamicPortRange(startPort, portCount, "ipv4", "tcp"))
{
TextLogger.LogError("Failed to set dynamic port range for ipv4:tcp. Start port : {0}. Port count : {1}.", startPort, portCount);
return(false);
}
if (!SetDynamicPortRange(startPort, portCount, "ipv4", "udp"))
{
TextLogger.LogError("Failed to set dynamic port range for ipv4:udp. Start port : {0}. Port count : {1}.", startPort, portCount);
return(false);
}
}
if (Socket.OSSupportsIPv6)
{
if (!SetDynamicPortRange(startPort, portCount, "ipv6", "tcp"))
{
TextLogger.LogError("Failed to set dynamic port range for ipv6:tcp. Start port : {0}. Port count : {1}.", startPort, portCount);
return(false);
}
if (!SetDynamicPortRange(startPort, portCount, "ipv6", "udp"))
{
TextLogger.LogError("Failed to set dynamic port range for ipv6:udp. Start port : {0}. Port count : {1}.", startPort, portCount);
return(false);
}
}
return(true);
}
private bool InvokeFabricDeployer(ClusterTopology topologyToDeploy, DeploymentType deploymentType, bool toUseCurrentClusterManifestForDeployment)
{
TextLogger.LogInfo("Invoking FabricDeployer with deployment type = {0} and toUseCurrentClusterManifestForDeployment = {1}.", deploymentType, toUseCurrentClusterManifestForDeployment);
string clusterManifestFileToDeploy;
if (toUseCurrentClusterManifestForDeployment)
{
string currentClusterManifestFile = this.GetCurrentClusterManifestFile(topologyToDeploy.CurrentNodeName);
if (File.Exists(currentClusterManifestFile))
{
TextLogger.LogInfo("Using current cluster manifest file {0} for deployment.", currentClusterManifestFile);
clusterManifestFileToDeploy = currentClusterManifestFile;
}
else
{
TextLogger.LogError("Current cluster manifest file does not exist as {0} when toUseCurrentClusterManifestForDeployment = true.", currentClusterManifestFile);
return(false);
}
}
else
{
clusterManifestFileToDeploy = this.GetBootstrapClusterManifestFile();
}
if (!this.FabricDeploy(deploymentType, clusterManifestFileToDeploy, this.DataRootDirectory, topologyToDeploy.ServiceFabricNodes, true))
{
TextLogger.LogError(
"Failed to invoke FabricDeployer with deployment type = {0}, cluster manifest file = {1}, data root directory = {2}.",
deploymentType,
clusterManifestFileToDeploy,
this.DataRootDirectory);
return(false);
}
return(true);
}
private void ConsoleCancelEventHanlder(object sender, ConsoleCancelEventArgs args)
{
TextLogger.LogInfo("Cancel event (Ctrl + C / Ctrl + Break) received. Stopping Service Fabric Autopilot Agent.");
if (this.agentStatus.GetCurrentState() < ServiceFabricAutopilotAgentState.Started)
{
return;
}
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.Stopping, false);
this.backgroundStateMachineCancellationTokenSource.Cancel();
this.localNodeManager.StopLocalNode();
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.Stopped, false);
TextLogger.LogInfo("Service Fabric Autopilot Agent has stopped. Exiting the process.");
this.Exit(true);
}
private string GetBootstrapClusterManifestFile()
{
string bootstrapClusterManifestFile;
if (ConfigurationManager.BootstrapClusterManifestLocation == BootstrapClusterManifestLocationType.Default || this.useTestMode)
{
bootstrapClusterManifestFile = Path.Combine(this.agentApplicationDirectory, StringConstants.DefaultBootstrapClusterManifestFileName);
}
else if (ConfigurationManager.BootstrapClusterManifestLocation == BootstrapClusterManifestLocationType.EnvironmentDefault)
{
bootstrapClusterManifestFile = Path.Combine(this.agentApplicationDirectory, StringConstants.EnvironmentDefaultConfigDirectoryName, APRuntime.ClusterName, APRuntime.EnvironmentName, StringConstants.DefaultBootstrapClusterManifestFileName);
}
else
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Bootstrap cluster manifest file is not available based on current BootstrapClusterManifestLocation {0}.", ConfigurationManager.BootstrapClusterManifestLocation));
}
TextLogger.LogInfo("Based on bootstrapClusterManifestLocation {0} and useTestMode {1}, bootstrapClusterManifestFile = {2}", ConfigurationManager.BootstrapClusterManifestLocation, this.useTestMode, bootstrapClusterManifestFile);
return(bootstrapClusterManifestFile);
}
internal void LogTopology()
{
if (this.ServiceFabricNodes.Count == 0)
{
TextLogger.LogInfo("Static topology defined in cluster manifest + dynamic topology update disabled.");
}
else
{
TextLogger.LogInfo("Static topology defined in cluster manifest + dynamic topology update with {0} nodes", this.ServiceFabricNodes.Count);
TextLogger.LogInfo("NodeName, NodeType, UpgradeDomain, FaultDomain, IPAddressOrFQDN");
foreach (InfrastructureNodeType node in this.ServiceFabricNodes)
{
TextLogger.LogInfo(
"{0}, {1}, {2}, {3}, {4}",
node.NodeName,
node.NodeTypeRef,
node.UpgradeDomain,
node.FaultDomain,
node.IPAddressOrFQDN
);
}
}
}
public void Start()
{
string applicationDirectory = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
this.certificateManager = new EnvironmentCertificateManager(applicationDirectory);
APRuntime.Initialize(StringConstants.ApplicationConfigurationFileName);
TextLogger.InitializeLogging(StringConstants.CustomLogIdName);
TextLogger.LogInfo("Starting Service Fabric Environment Sync.");
if (!ConfigurationManager.GetCurrentConfigurations())
{
this.Exit(false);
return;
}
TextLogger.SetLogLevel(ConfigurationManager.LogLevel);
if (!this.certificateManager.GetEnvironmentCertificatesFromCurrentConfigurations())
{
this.Exit(false);
return;
}
this.RegisterConsoleCancelEventNotifications();
this.certificateManagerTask = Task.Factory.StartNew(
() => this.RunCertificateManager(this.certificateManagerCancellationTokenSource.Token),
this.certificateManagerCancellationTokenSource.Token,
TaskCreationOptions.LongRunning,
TaskScheduler.Default);
}
private void RunStateMachine()
{
bool toRunStateMachineCore = false;
lock (this.agentStatus.StatusLock)
{
if (this.agentStatus.State < ServiceFabricAutopilotAgentState.Started)
{
return;
}
TextLogger.LogVerbose("RunStateMachine : {0}.", this.agentStatus);
if (this.agentStatus.LastFailedDeploymentAction != DateTime.MinValue && (DateTime.UtcNow - this.agentStatus.LastFailedDeploymentAction).TotalSeconds < ConfigurationManager.DeploymentRetryIntervalSeconds)
{
return;
}
// Note that when a deployer operation (node configuration creation or update) fails, it would be retried based on the same cluster manifest (bootstrap or current).
// I.e a retry would happen even when a previous operation failed with the same cluster manifest.
if ((this.agentStatus.State > ServiceFabricAutopilotAgentState.Started && agentStatus.State < ServiceFabricAutopilotAgentState.DiscoveringTopology) ||
(this.agentStatus.State == ServiceFabricAutopilotAgentState.DiscoveringTopology && ConfigurationManager.BootstrapClusterManifestLocation != BootstrapClusterManifestLocationType.NotReady) ||
(this.agentStatus.State > ServiceFabricAutopilotAgentState.DiscoveringTopology && agentStatus.State < ServiceFabricAutopilotAgentState.NodeStarted) ||
(this.agentStatus.State == ServiceFabricAutopilotAgentState.NodeStarted && !this.agentStatus.ExpectedTopologyRunning))
{
TextLogger.LogInfo(
"RunStateMachine: current agent state necessitates RunStateMachineCore. Agent status = {0}, ExpectedTopologyRunning = {1}, LastFailedDeploymentAction = {2}, BootstrapClusterManifestLocation = {3}.",
this.agentStatus.State,
this.agentStatus.ExpectedTopologyRunning,
this.agentStatus.LastFailedDeploymentAction,
ConfigurationManager.BootstrapClusterManifestLocation);
toRunStateMachineCore = true;
}
}
if (toRunStateMachineCore)
{
bool coreStateMachineRunResult;
ServiceFabricAutopilotAgentState currentState;
do
{
currentState = this.agentStatus.GetCurrentState();
if (currentState < ServiceFabricAutopilotAgentState.Started)
{
return;
}
coreStateMachineRunResult = this.RunStateMachineCore(currentState);
TextLogger.LogInfo("RunStateMachine: RunStateMachineCore completed with result {0}. State prior to the run : {1}.", coreStateMachineRunResult, currentState);
if (!coreStateMachineRunResult)
{
lock (agentStatus.StatusLock)
{
this.agentStatus.LastFailedDeploymentAction = DateTime.UtcNow;
}
}
else
{
lock (agentStatus.StatusLock)
{
this.agentStatus.LastFailedDeploymentAction = DateTime.MinValue;
}
}
currentState = this.agentStatus.GetCurrentState();
}while (currentState > ServiceFabricAutopilotAgentState.Started && currentState < ServiceFabricAutopilotAgentState.NodeStarted && coreStateMachineRunResult);
}
}
/*
* TODO: currently AP safe config deployment is not supported to update configurations.
* The only way to update the configurations is via an application upgrade.
* This is OK since none of the configurations are critical.
* To update all things critical in Service Fabric Autopilot Agent, an AP application upgrade is needed.
*/
public static bool GetCurrentConfigurations(bool useTestMode = false)
{
if (useTestMode)
{
GetTestConfigurations();
}
else
{
IConfiguration configuration = APConfiguration.GetConfiguration();
BackgroundStateMachineRunIntervalSeconds = configuration.GetInt32Value(StringConstants.AgentConfigurationSectionName, "BackgroundStateMachineRunIntervalSeconds", 10);
DeploymentRetryIntervalSeconds = configuration.GetInt32Value(StringConstants.AgentConfigurationSectionName, "DeploymentRetryIntervalSeconds", 120);
string bootstrapClusterManifestLocationString = configuration.GetStringValue(StringConstants.AgentConfigurationSectionName, "BootstrapClusterManifestLocation", string.Empty);
BootstrapClusterManifestLocationType bootstrapClusterManifestLocation;
if (!Enum.TryParse <BootstrapClusterManifestLocationType>(bootstrapClusterManifestLocationString, out bootstrapClusterManifestLocation))
{
TextLogger.LogError("Bootstrap cluster manifest location is invalid or not specified : {0}.", bootstrapClusterManifestLocationString);
return(false);
}
BootstrapClusterManifestLocation = bootstrapClusterManifestLocation;
DynamicPortRangeStart = configuration.GetInt32Value(StringConstants.AgentConfigurationSectionName, "DynamicPortRangeStart", -1);
DynamicPortRangeEnd = configuration.GetInt32Value(StringConstants.AgentConfigurationSectionName, "DynamicPortRangeEnd", -1);
// please refer to http://support.microsoft.com/kb/929851 for valid dynamic port range in Windows.
if (DynamicPortRangeStart < 1025 || DynamicPortRangeEnd > 65535 || DynamicPortRangeEnd - DynamicPortRangeStart + 1 < 255)
{
TextLogger.LogError("Dynamic port range is invalid or not specified : [{0}, {1}].", DynamicPortRangeStart, DynamicPortRangeEnd);
return(false);
}
DynamicTopologyUpdateMode = (DynamicTopologyUpdateMode)configuration.GetInt32Value(StringConstants.AgentConfigurationSectionName, "DynamicTopologyUpdateMode", 0);
string logLevelString = configuration.GetStringValue(StringConstants.AgentConfigurationSectionName, "LogLevel", "Info");
LogLevel logLevel;
if (!Enum.TryParse <LogLevel>(logLevelString, out logLevel))
{
TextLogger.LogError("Log level is invalid : {0}.", logLevelString);
return(false);
}
LogLevel = logLevel;
PortFowardingRules = new List <PortFowardingRule>();
if (configuration.SectionExists(StringConstants.PortForwardingConfigurationSectionName))
{
string[] portFowardingRuleNames = configuration.GetSectionKeys(StringConstants.PortForwardingConfigurationSectionName);
if (portFowardingRuleNames != null)
{
foreach (string portFowardingRuleName in portFowardingRuleNames)
{
string[] portFowardingRuleComponents = configuration.GetStringValueAndSplit(StringConstants.PortForwardingConfigurationSectionName, portFowardingRuleName, ",");
if (portFowardingRuleComponents == null || portFowardingRuleComponents.Length != 3)
{
TextLogger.LogError("Port forwarding rule {0} is invalid. Valid rule format : RuleName=ListenPort,ConnectPort,MachineFunction", portFowardingRuleName);
return(false);
}
int listenPort;
if (!int.TryParse(portFowardingRuleComponents[0], out listenPort) || listenPort <= 0 || listenPort > 65535)
{
TextLogger.LogError("Port forwarding rule {0} is invalid with invalid listen port {1}. Valid rule format : RuleName=ListenPort,ConnectPort,MachineFunction", portFowardingRuleName, portFowardingRuleComponents[0]);
return(false);
}
int connectPort;
if (!int.TryParse(portFowardingRuleComponents[1], out connectPort) || connectPort <= 0 || connectPort > 65535)
{
TextLogger.LogError("Port forwarding rule {0} is invalid with invalid connect port {1}. Valid rule format : RuleName=ListenPort,ConnectPort,MachineFunction", portFowardingRuleName, portFowardingRuleComponents[1]);
return(false);
}
PortFowardingRules.Add(new PortFowardingRule {
RuleName = portFowardingRuleName, ListenPort = listenPort, ConnectPort = connectPort, MachineFunction = portFowardingRuleComponents[2]
});
}
}
}
}
return(true);
}
private void Exit(bool succeeded)
{
TextLogger.Flush();
this.exitCallback(succeeded);
}
public bool DiscoverTopology(out ClusterTopology expectedTopology)
{
expectedTopology = null;
try
{
TextLogger.LogInfo("Discovering cluster topology from bootstrap cluster manifest file.");
if (ConfigurationManager.BootstrapClusterManifestLocation == BootstrapClusterManifestLocationType.NotReady)
{
TextLogger.LogWarning("Bootstrap cluster manifest is not ready. Cluster topology would not be discovered until bootstrap cluster manifest is ready.");
return(false);
}
ClusterManifestType bootstrapClusterManifest;
if (!this.ReadBootstrapClusterManifestFile(out bootstrapClusterManifest))
{
return(false);
}
TopologyProviderType topologyProviderType = TopologyProvider.GetTopologyProviderTypeFromClusterManifest(bootstrapClusterManifest);
TextLogger.LogInfo("Topology provider type : {0}.", topologyProviderType);
if (topologyProviderType != TopologyProviderType.StaticTopologyProvider)
{
TextLogger.LogInfo("Topology provider type {0} is not supported to provide static topology.", topologyProviderType);
return(false);
}
else
{
/*
* Static topology provider defines the authoritative cluster topology in the cluster manifest.
* If the local machine (physical or virtual) is not part of the cluster topology specified in static topology provider section, a Service Fabric node will not be started locally.
* During scale-up, bootstrap cluster manifest has to be upgraded to allow new Service Fabric nodes to be started on the new machines and join the ring.
* A scale-up is normally a two-phase process with allocation of new machines (and their IPs) and generation of a new bootstrap cluster manifest with updated topology.
*/
ClusterManifestTypeInfrastructureWindowsAzureStaticTopology staticTopologyProviderElement = TopologyProvider.GetStaticTopologyProviderElementFromClusterManifest(bootstrapClusterManifest);
if (staticTopologyProviderElement.NodeList == null || staticTopologyProviderElement.NodeList.Length == 0)
{
TextLogger.LogError("Static topology provider section of bootstrap cluster manifest does not specify topology of the Service Fabric cluster.");
return(false);
}
LogClusterTopology(staticTopologyProviderElement);
ClusterTopology staticTopologyProviderClusterTopology = ClusterTopology.GetClusterTopology();
staticTopologyProviderClusterTopology.LogTopology();
expectedTopology = staticTopologyProviderClusterTopology;
TextLogger.LogInfo("Successfully discovered cluster topology.");
return(true);
}
}
catch (Exception e)
{
TextLogger.LogError("Failed to discover cluster topology : {0}", e);
return(false);
}
}
public bool StartLocalNode()
{
try
{
TextLogger.LogInfo("Starting local Service Fabric node.");
// TODO: After current xcopyable issue is fixed, update the pattern here.
// TODO: [General] disposal of ServiceController
ServiceController fabricHostService = this.GetInstalledService(Constants.FabricHostServiceName);
if (fabricHostService != null)
{
if (fabricHostService.Status == ServiceControllerStatus.StartPending)
{
TextLogger.LogWarning("Fabric host service is starting.");
return(false);
}
else if (fabricHostService.Status == ServiceControllerStatus.Running)
{
TextLogger.LogInfo("Successfully started local Service Fabric node.");
return(true);
}
else if (!this.StartFabricHostService())
{
TextLogger.LogError("Failed to start local Service Fabric node ");
return(false);
}
}
else
{
ServiceController fabricInstallerService = this.GetInstalledService(Constants.FabricInstallerServiceName);
// Install and configure Fabric installer service if not installed.
if (fabricInstallerService == null)
{
TextLogger.LogInfo("Installing and configuring Fabric installer service.");
if (!this.InstallAndConfigureFabricInstallerService(this.agentApplicationDirectory))
{
TextLogger.LogError("Failed to install and configure Fabric installer service");
return(false);
}
fabricInstallerService = this.GetInstalledService(Constants.FabricInstallerServiceName);
}
// If Fabric installer service is not in stopped state, stop it.
if (fabricInstallerService.Status != ServiceControllerStatus.Stopped)
{
TextLogger.LogInfo("Fabric installer service is in {0} state. Stopping it.", fabricInstallerService.Status);
if (!this.StopFabricInstallerService())
{
TextLogger.LogError("Failed to stop Fabric installer service.");
return(false);
}
}
// Start Fabric installer service to start Fabric host service.
TextLogger.LogInfo("Starting Fabric installer service to start Fabric host service.");
if (!this.StartFabricInstallerService())
{
TextLogger.LogError("Failed to start Fabric installer service.");
return(false);
}
}
TextLogger.LogInfo("Successfully started local Service Fabric node.");
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to start local Service Fabric node : {0}", e);
return(false);
}
}
private bool RunStateMachineCore(ServiceFabricAutopilotAgentState currentState)
{
TextLogger.LogInfo("Starting RunStateMachineCore. Current state : {0}.", currentState);
switch (currentState)
{
case ServiceFabricAutopilotAgentState.ManagingNetworkConfiguration:
{
if (!this.localNodeManager.ManageNetworkConfiguration())
{
return(false);
}
ServiceFabricAutopilotAgentState nextState;
if (!this.ComputeNextStateFromManagingNetworkConfigurationState(out nextState))
{
return(false);
}
this.agentStatus.SetNextState(nextState);
break;
}
case ServiceFabricAutopilotAgentState.InstallingProduct:
{
if (!this.localNodeManager.InstallProduct())
{
return(false);
}
// For now, we assume the topology is static.
// TODO: when adding dynamic topology support, change state machine transition here.
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.DiscoveringTopology);
break;
}
case ServiceFabricAutopilotAgentState.DiscoveringTopology:
{
ClusterTopology expectedTopology;
if (!this.localNodeManager.DiscoverTopology(out expectedTopology))
{
return(false);
}
lock (this.agentStatus.StatusLock)
{
this.agentStatus.ExpectedTopology = expectedTopology;
this.agentStatus.ExpectedTopologyRunning = false;
}
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.ConfiguringNode);
break;
}
case ServiceFabricAutopilotAgentState.ConfiguringNode:
{
ClusterTopology topologyToDeploy = this.agentStatus.GetTopologyToDeploy();
if (!this.localNodeManager.ConfigureLocalNode(topologyToDeploy))
{
return(false);
}
lock (this.agentStatus.StatusLock)
{
this.agentStatus.DeployedTopology = topologyToDeploy;
this.agentStatus.ExpectedTopologyRunning = !this.agentStatus.DeployedTopology.IsClusterTopologyChanged(this.agentStatus.ExpectedTopology);
}
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.StartingNode);
break;
}
case ServiceFabricAutopilotAgentState.StartingNode:
{
if (!this.localNodeManager.StartLocalNode())
{
return(false);
}
this.agentStatus.SetNextState(ServiceFabricAutopilotAgentState.NodeStarted);
break;
}
case ServiceFabricAutopilotAgentState.NodeStarted:
{
// TODO: this only applies when DynamicTopologyUpdateMode == None or OnNodeConfigurationOnly.
// When we want to update topology whenever topology changes, this needs to change.
lock (this.agentStatus.StatusLock)
{
this.agentStatus.ExpectedTopologyRunning = true;
this.agentStatus.ExpectedTopology = ClusterTopology.GetClusterTopology();
this.agentStatus.DeployedTopology = this.agentStatus.ExpectedTopology;
}
break;
}
}
return(true);
}
private bool GetAndCreateDataRootDirectory(out string dataRootDirectory)
{
dataRootDirectory = null;
try
{
// On AP machines, data root would depend on machine SKU (specifically the SKU's disk layout). For test mode testing, use C:\SFRoot.
if (this.useTestMode)
{
dataRootDirectory = Path.Combine(Path.GetPathRoot("C:\\"), StringConstants.ServiceFabricRootDirectoryName);
TextLogger.LogInfo("Using {0} as data root directory in test mode.", dataRootDirectory);
}
else
{
string diskLayoutConfigurationFile = Path.Combine(this.agentApplicationDirectory, StringConstants.DiskLayoutConfigurationFileName);
TextLogger.LogInfo("Data root directory would be determined based on local machine SKU {0} and disk layout configuration file {1}.", this.LocalMachineSku, diskLayoutConfigurationFile);
string dataRootHint = null;
using (CsvReader reader = new CsvReader(diskLayoutConfigurationFile))
{
while (reader.Read())
{
if (Regex.IsMatch(this.LocalMachineSku, reader["SKU"], RegexOptions.IgnoreCase | RegexOptions.CultureInvariant))
{
dataRootHint = Path.GetFullPath(Environment.ExpandEnvironmentVariables(reader["DataRootHint"]));
TextLogger.LogInfo(
"Local machine SKU {0} matched disk layout SKU rule '{1}' -> '{2}' which expands to data root '{3}'",
this.LocalMachineSku,
reader["SKU"],
reader["DataRootHint"],
dataRootHint);
break;
}
}
}
if (string.IsNullOrEmpty(dataRootHint))
{
TextLogger.LogError("Disk layout configuration file {0} does not contain a valid entry for local machine SKU {1}.", diskLayoutConfigurationFile, this.LocalMachineSku);
return(false);
}
dataRootDirectory = Path.Combine(Path.GetPathRoot(dataRootHint), StringConstants.ServiceFabricRootDirectoryName);
TextLogger.LogInfo("Using {0} as data root directory based on local machine SKU {1} and disk layout configuration file {2}.", dataRootDirectory, this.LocalMachineSku, diskLayoutConfigurationFile);
}
TextLogger.LogInfo("Creating data root directory {0}.", dataRootDirectory);
Directory.CreateDirectory(dataRootDirectory);
return(true);
}
catch (Exception e)
{
TextLogger.LogError("Failed to get and create data root directory : {0}", e);
return(false);
}
}
