public async Task Run(ProcessMetadata metadata)
{
var result = new ProcessResult();
result.Metadata = metadata;
Stopwatch stw = new Stopwatch();
stw.Start();
// Simulate some long running process.
await Task.Delay((metadata.ProcessDuration * 1000));
stw.Stop();
result.ExecutionTime = stw.Elapsed.TotalMilliseconds;
result.IsSuccessful = true;
result.Message = "The process ran successfully!";
// Send result to SignlalR to pass on to the client.
using (var hubConnection = new HubConnection(ConfigurationManager.AppSettings["SignalRURL"]))
{
IHubProxy testHubProxy = hubConnection.CreateHubProxy(ConfigurationManager.AppSettings["SignalRHubName"]);
await hubConnection.Start();
await testHubProxy.Invoke("ProcessComplete", result);
}
}
public IPotentiallySigned <ServiceMetadata> ParseServiceMetadata(FetcherResponse fetcherResponse)
{
try
{
XmlDocument doc = new XmlDocument();
doc.Load(fetcherResponse.InputStream);
var o = ClassFactory.FromXmlStream(fetcherResponse.InputStream);
X509Certificate signer = null;
if (o is SignedServiceMetadata)
{
signer = XmldsigVerifier.Verify(doc);
}
ServiceInformationType serviceInformation = ((tns.ServiceMetadata)o).ServiceInformation;
List <ProcessMetadata <Endpoint> > processMetadatas = new List <ProcessMetadata <Endpoint> >();
foreach (ProcessType processType in serviceInformation.ProcessList.Process)
{
List <Endpoint> endpoints = new List <Endpoint>();
foreach (EndpointType endpointType in processType.ServiceEndpointList.Endpoint)
{
endpoints.Add(
Endpoint.Of(
TransportProfile.Of(endpointType.TransportProfile),
new Uri(endpointType.EndpointURI),
this.CertificateInstance(endpointType.Certificate.Data)));
}
processMetadatas.Add(
ProcessMetadata <Endpoint> .Of(
ProcessIdentifier.Of(
processType.ProcessIdentifier.PrimitiveValue,
Scheme.Of(processType.ProcessIdentifier.Scheme)),
endpoints));
}
return(Signed <ServiceMetadata> .Of(
ServiceMetadata.Of(
ParticipantIdentifier.Of(
serviceInformation.ParticipantIdentifier.PrimitiveValue,
Scheme.Of(serviceInformation.ParticipantIdentifier.Scheme)),
DocumentTypeIdentifier.Of(
serviceInformation.DocumentIdentifier.PrimitiveValue,
Scheme.Of(serviceInformation.DocumentIdentifier.Scheme)),
processMetadatas),
signer));
}
catch (Exception e) when(e is CertificateException | e is IOException)
{
throw new Exception(e.Message, e);
}
}
private static void SafelyKillProcess(ProcessMetadata metadata)
{
try
{
if (_isProcessManager)
{
metadata.SafelyForceKill();
}
}
catch (Exception ex)
{
Log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name);
}
}
private static void SafelyKillProcess(ProcessMetadata metadata)
{
try
{
if (metadata.Process != null && !metadata.Process.HasExited)
{
metadata.Process.Kill();
}
}
catch (Exception ex)
{
Log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name);
}
}
private static void SafelyKillProcess(ProcessMetadata metadata)
{
try
{
if (metadata.Process != null && !metadata.Process.HasExited)
{
metadata.Process.Kill();
}
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name));
}
}
private static void GrabFreePortForInstance(ProcessMetadata instance)
{
instance.Port = GetFreeTcpPort();
if (instance.Port == null)
{
throw new Exception($"Unable to secure a port for {instance.Name}");
}
instance.RefreshPortVars();
Log.Debug("Attempting to use port {0} for the {1}.", instance.Port, instance.Name);
if (instance.PortFilePath != null)
{
File.WriteAllText(instance.PortFilePath, instance.Port.Value.ToString(CultureInfo.InvariantCulture));
}
}
/// <summary>
/// Parses a process and builds an AST component for it
/// </summary>
/// <param name="network">The top-level network.</param>
/// <param name="process">The process to build the AST for.</param>
protected virtual Process Parse(NetworkState network, ProcessMetadata process, Simulation simulation)
{
var st = process.Instance.GetType();
var inputbusses = process.Instance.InputBusses.Union(process.Instance.ClockedInputBusses).ToArray();
var outputbusses = process.Instance.OutputBusses.Distinct().ToArray();
var res = new ProcessState()
{
Name = NameWithoutPrefix(network, st.FullName, st),
SourceType = st,
CecilType = LoadType(st),
SourceInstance = process,
InstanceName = process.InstanceName,
Parent = network,
IsSimulation = process.Instance is SimulationProcess,
IsClocked = st.GetCustomAttribute <ClockedProcessAttribute>() != null,
Decompile =
(
typeof(SimpleProcess).IsAssignableFrom(st)
||
typeof(StateProcess).IsAssignableFrom(st)
)
&&
!st.HasAttribute <SuppressOutputAttribute>()
&&
!st.HasAttribute <SuppressBodyAttribute>()
};
var proctype = res.CecilType.Resolve();
if (res.CecilType is Mono.Cecil.GenericInstanceType)
{
var gp = res.CecilType as Mono.Cecil.GenericInstanceType;
var names = gp.GenericArguments.ToArray();
var ga = proctype.GenericParameters.ToArray();
foreach (var g in ga)
{
res.GenericMap[g.Name] = g;
res.GenericTypes[g.Name] = names[g.Position];
}
}
res.InputBusses = inputbusses.Select(x => Parse(network, res, x, simulation)).ToArray();
res.OutputBusses = outputbusses.Select(x => Parse(network, res, x, simulation)).ToArray();
res.InternalBusses = process.Instance.InternalBusses.Select(x => Parse(network, res, x, simulation)).ToArray();
foreach (var ib in res.InternalBusses)
{
ib.IsInternal = true;
}
// Set up the local names by finding the field that holds the instance reference
foreach (var b in res.InputBusses.Union(res.OutputBusses).Union(res.InternalBusses))
{
var f = st
.GetFields(BindingFlags.Instance | BindingFlags.FlattenHierarchy | BindingFlags.Public | BindingFlags.NonPublic)
.FirstOrDefault(x => x.GetValue(process.Instance) == b.SourceInstance);
if (f != null)
{
res.LocalBusNames[b] = f.Name;
}
}
if (res.Decompile)
{
// Register all variables
foreach (var f in proctype.Fields)
{
var ft = res.ResolveGenericType(f.FieldType);
if (ft.IsBusType())
{
RegisterBusReference(network, res, f);
}
else
{
RegisterVariable(network, res, f);
}
}
}
foreach (var f in process.Initialization)
{
if (f.Value == null)
{
continue;
}
Variable v;
if (res.Variables.TryGetValue(f.Key, out v))
{
SetDataElementDefaultValue(network, res, v, f.Value, false);
}
}
res.SharedSignals = res.Signals.Values.ToArray();
res.SharedVariables = res.Variables.Values.ToArray();
res.InternalDataElements = new DataElement[0];
return(res);
}
public IPotentiallySigned <ServiceMetadata> ParseServiceMetadata(FetcherResponse fetcherResponse)
{
try
{
XmlDocument doc = new XmlDocument();
doc.PreserveWhitespace = true;
doc.Load(fetcherResponse.InputStream);
var o = ClassFactory.FromXmlElement(doc.DocumentElement);
Xml.Busdox.tns.ServiceMetadata serviceMetadata = o as Xml.Busdox.tns.ServiceMetadata;
X509Certificate signer = null;
if (o is SignedServiceMetadata)
{
signer = XmldsigVerifier.Verify(doc);
serviceMetadata = ((SignedServiceMetadata)o).ServiceMetadata;
}
if (serviceMetadata == null)
{
throw new LookupException("ServiceMetadata element not found");
}
ServiceInformationType serviceInformation = serviceMetadata.ServiceInformation;
List <ProcessMetadata <Endpoint> > processMetadatas = new List <ProcessMetadata <Endpoint> >();
foreach (ProcessType processType in serviceInformation.ProcessList.Process)
{
List <Endpoint> endpoints = new List <Endpoint>();
foreach (EndpointType endpointType in processType.ServiceEndpointList.Endpoint)
{
var certificate = this.CertificateInstance(Convert.FromBase64String(endpointType.Certificate));
var endpointUri = new Uri(endpointType.EndpointReference.Address.PrimitiveValue);
var profile = TransportProfile.Of(endpointType.TransportProfile);
endpoints.Add(Endpoint.Of(profile, endpointUri, certificate));
}
processMetadatas.Add(
ProcessMetadata <Endpoint> .Of(
ProcessIdentifier.Of(
processType.ProcessIdentifier.PrimitiveValue,
Scheme.Of(processType.ProcessIdentifier.Scheme)),
endpoints));
}
return(Signed <ServiceMetadata> .Of(
ServiceMetadata.Of(
ParticipantIdentifier.Of(
serviceInformation.ParticipantIdentifier.PrimitiveValue,
Scheme.Of(serviceInformation.ParticipantIdentifier.Scheme)),
DocumentTypeIdentifier.Of(
serviceInformation.DocumentIdentifier.PrimitiveValue,
Scheme.Of(serviceInformation.DocumentIdentifier.Scheme)),
processMetadatas),
signer));
}
catch (Exception e) when(e is CertificateException | e is IOException)
{
throw new Exception(e.Message, e);
}
}
private static Task StartProcessWithKeepAlive(ProcessMetadata metadata)
{
const int circuitBreakerMax = 3;
var path = metadata.ProcessPath;
Log.Debug("Starting keep alive for {0}.", path);
return(Task.Run(
async() =>
{
try
{
metadata.IsBeingManaged = true;
var sequentialFailures = 0;
while (true)
{
if (_cancellationTokenSource.Token.IsCancellationRequested)
{
Log.Debug("Shutdown triggered for keep alive {0}.", path);
return;
}
try
{
metadata.IsFaulted = false;
if (metadata.ProgramIsRunning())
{
Log.Debug("{0} is already running.", path);
continue;
}
var startInfo = new ProcessStartInfo {
FileName = path
};
if (!string.IsNullOrWhiteSpace(metadata.ProcessArguments))
{
startInfo.Arguments = metadata.ProcessArguments;
}
Log.Debug("Starting {0}.", path);
GrabFreePortForInstance(metadata);
metadata.Process = Process.Start(startInfo);
await Task.Delay(150, _cancellationTokenSource.Token);
if (metadata.Process == null || metadata.Process.HasExited)
{
Log.Error("{0} has failed to start.", path);
sequentialFailures++;
}
else
{
Log.Debug("Successfully started {0}.", path);
sequentialFailures = 0;
metadata.AlertSubscribers();
Log.Debug("Finished calling port subscribers for {0}.", metadata.Name);
}
}
catch (Exception ex)
{
metadata.IsFaulted = true;
Log.Error(ex, "Exception when trying to start an instance of {0}.", path);
sequentialFailures++;
}
finally
{
metadata.HasAttemptedStartup = true;
// Delay for a reasonable amount of time before we check to see if the process is alive again.
if (metadata.IsFaulted)
{
// Quicker retry in these cases
await Task.Delay(ExceptionRetryInterval, _cancellationTokenSource.Token);
}
else
{
await Task.Delay(KeepAliveInterval, _cancellationTokenSource.Token);
}
}
if (sequentialFailures >= circuitBreakerMax)
{
Log.Error("Circuit breaker triggered for {0}. Max failed retries reached ({1}).", path, sequentialFailures);
break;
}
}
}
finally
{
Log.Debug("Keep alive is dropping for {0}.", path);
metadata.IsBeingManaged = false;
}
},
_cancellationTokenSource.Token));
}
private static Task StartProcessWithKeepAlive(ProcessMetadata metadata)
{
var path = metadata.ProcessPath;
Log.Debug("Starting keep alive for {Process}.", path);
return(Task.Run(
async() =>
{
try
{
// Indicate that there is a worker responsible for doing this
// This acts as a lock
metadata.IsBeingManaged = true;
while (true)
{
if (metadata.SequentialFailures >= MaxFailures)
{
Log.Error <string, int>("Maximum retries ({ErrorCount}) reached starting {Process}.", path, MaxFailures);
metadata.ProcessState = ProcessState.Faulted;
return;
}
try
{
if (metadata.ProcessState == ProcessState.NeverChecked)
{
// This means we have never tried from this domain
if (metadata.ProcessIsHealthy())
{
// This means one of two things:
// - Another domain has already started this process
// - The previous instance is in the process of shutting down
// It is possible that if a pipe is bound it may yet be cleaned up from a previous shutdown
// Assume healthy to start, but look for problems
metadata.ProcessState = ProcessState.Healthy;
// Check on a delay to be sure we keep the process available after a shutdown
var attempts = 7;
var delay = 50d;
while (--attempts > 0)
{
await Task.Delay((int)delay).ConfigureAwait(false);
if (metadata.ProcessIsHealthy())
{
// Should result in a max delay of ~3.28 seconds before giving up
delay = delay * 1.75d;
continue;
}
// The previous instance is gone, time to start the process
Log.Information("Recovering from previous {Process} shutdown. Ready for start.", path);
metadata.ProcessState = ProcessState.ReadyToStart;
break;
}
}
else
{
// If no instance exists, just get ready to start
metadata.ProcessState = ProcessState.ReadyToStart;
}
}
else if (metadata.ProcessState == ProcessState.Healthy || metadata.ProcessState == ProcessState.Faulted)
{
// This means we have tried to start from this domain before
metadata.ProcessState = metadata.ProcessIsHealthy() ? ProcessState.Healthy : ProcessState.ReadyToStart;
}
if (metadata.ProcessState == ProcessState.ReadyToStart)
{
Log.Information("Attempting to start {Process}.", path);
var startInfo = new ProcessStartInfo
{
FileName = path,
UseShellExecute = false // Force consistency in behavior between Framework and Core
};
if (!string.IsNullOrWhiteSpace(metadata.ProcessArguments))
{
startInfo.Arguments = metadata.ProcessArguments;
}
metadata.Process = Process.Start(startInfo);
var timeout = 2000;
while (timeout > 0)
{
// Loop and watch for evidence that the process is up and running
if (metadata.ProcessIsHealthy())
{
metadata.SequentialFailures = 0;
metadata.ProcessState = ProcessState.Healthy;
Log.Information("Successfully started {Process}.", path);
break;
}
if (metadata.Process == null || metadata.Process.HasExited)
{
Log.Error("Failed to start {Process}.", path);
metadata.ProcessState = ProcessState.Faulted;
break;
}
await Task.Delay(100).ConfigureAwait(false);
timeout -= 100;
}
}
}
catch (Exception ex)
{
metadata.ProcessState = ProcessState.Faulted;
Log.Error(ex, "Failed to start {Process}.", path);
}
finally
{
if (metadata.ProcessState == ProcessState.Faulted)
{
metadata.SequentialFailures++;
// Quicker retry in these cases
await Task.Delay(ExceptionRetryInterval).ConfigureAwait(false);
}
else
{
// Delay for a reasonable amount of time before we check to see if the process is alive again.
await Task.Delay(KeepAliveInterval).ConfigureAwait(false);
}
}
}
}
finally
{
Log.Warning("Keep alive is dropping for {Process}.", path);
metadata.IsBeingManaged = false;
}
}));
}
private static Task StartProcessWithKeepAlive(string path, string args, ProcessMetadata metadata)
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Starting keep alive for {0}.", path));
return(Task.Run(
() =>
{
try
{
var circuitBreakerMax = 3;
var sequentialFailures = 0;
while (true)
{
if (_cancellationTokenSource.IsCancellationRequested)
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Shutdown triggered for keep alive {0}.", path));
return;
}
try
{
if (metadata.Process != null && metadata.Process.HasExited == false)
{
DatadogLogging.RegisterStartupLog(log => log.Debug("We already have an active reference to {0}.", path));
continue;
}
if (ProgramIsRunning(path))
{
DatadogLogging.RegisterStartupLog(log => log.Debug("{0} is already running.", path));
continue;
}
var startInfo = new ProcessStartInfo {
FileName = path
};
if (!string.IsNullOrWhiteSpace(args))
{
startInfo.Arguments = args;
}
DatadogLogging.RegisterStartupLog(log => log.Debug("Starting {0}.", path));
metadata.PreStartAction?.Invoke();
metadata.Process = Process.Start(startInfo);
Thread.Sleep(200);
if (metadata.Process == null || metadata.Process.HasExited)
{
DatadogLogging.RegisterStartupLog(log => log.Error("{0} has failed to start.", path));
sequentialFailures++;
}
else
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Successfully started {0}.", path));
sequentialFailures = 0;
foreach (var portSubscriber in metadata.PortSubscribers)
{
portSubscriber(metadata.Port.Value);
}
DatadogLogging.RegisterStartupLog(log => log.Debug("Finished calling port subscribers for {0}.", metadata.Name));
}
}
catch (Exception ex)
{
DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Exception when trying to start an instance of {0}.", path));
sequentialFailures++;
}
finally
{
// Delay for a reasonable amount of time before we check to see if the process is alive again.
Thread.Sleep(20_000);
}
if (sequentialFailures >= circuitBreakerMax)
{
DatadogLogging.RegisterStartupLog(log => log.Error("Circuit breaker triggered for {0}. Max failed retries reached ({1}).", path, sequentialFailures));
break;
}
}
}
finally
{
DatadogLogging.RegisterStartupLog(log => log.Debug("Keep alive is dropping for {0}.", path));
}
}));
}
public IHttpActionResult Run(ProcessMetadata metadata)
{
// Queue the long running action in the background.
HostingEnvironment.QueueBackgroundWorkItem(x => _service.Run(metadata));
return(Ok());
}
