public async Task Run(ProcessMetadata metadata) { var result = new ProcessResult(); result.Metadata = metadata; Stopwatch stw = new Stopwatch(); stw.Start(); // Simulate some long running process. await Task.Delay((metadata.ProcessDuration * 1000)); stw.Stop(); result.ExecutionTime = stw.Elapsed.TotalMilliseconds; result.IsSuccessful = true; result.Message = "The process ran successfully!"; // Send result to SignlalR to pass on to the client. using (var hubConnection = new HubConnection(ConfigurationManager.AppSettings["SignalRURL"])) { IHubProxy testHubProxy = hubConnection.CreateHubProxy(ConfigurationManager.AppSettings["SignalRHubName"]); await hubConnection.Start(); await testHubProxy.Invoke("ProcessComplete", result); } }
public IPotentiallySigned <ServiceMetadata> ParseServiceMetadata(FetcherResponse fetcherResponse) { try { XmlDocument doc = new XmlDocument(); doc.Load(fetcherResponse.InputStream); var o = ClassFactory.FromXmlStream(fetcherResponse.InputStream); X509Certificate signer = null; if (o is SignedServiceMetadata) { signer = XmldsigVerifier.Verify(doc); } ServiceInformationType serviceInformation = ((tns.ServiceMetadata)o).ServiceInformation; List <ProcessMetadata <Endpoint> > processMetadatas = new List <ProcessMetadata <Endpoint> >(); foreach (ProcessType processType in serviceInformation.ProcessList.Process) { List <Endpoint> endpoints = new List <Endpoint>(); foreach (EndpointType endpointType in processType.ServiceEndpointList.Endpoint) { endpoints.Add( Endpoint.Of( TransportProfile.Of(endpointType.TransportProfile), new Uri(endpointType.EndpointURI), this.CertificateInstance(endpointType.Certificate.Data))); } processMetadatas.Add( ProcessMetadata <Endpoint> .Of( ProcessIdentifier.Of( processType.ProcessIdentifier.PrimitiveValue, Scheme.Of(processType.ProcessIdentifier.Scheme)), endpoints)); } return(Signed <ServiceMetadata> .Of( ServiceMetadata.Of( ParticipantIdentifier.Of( serviceInformation.ParticipantIdentifier.PrimitiveValue, Scheme.Of(serviceInformation.ParticipantIdentifier.Scheme)), DocumentTypeIdentifier.Of( serviceInformation.DocumentIdentifier.PrimitiveValue, Scheme.Of(serviceInformation.DocumentIdentifier.Scheme)), processMetadatas), signer)); } catch (Exception e) when(e is CertificateException | e is IOException) { throw new Exception(e.Message, e); } }
private static void SafelyKillProcess(ProcessMetadata metadata) { try { if (_isProcessManager) { metadata.SafelyForceKill(); } } catch (Exception ex) { Log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name); } }
private static void SafelyKillProcess(ProcessMetadata metadata) { try { if (metadata.Process != null && !metadata.Process.HasExited) { metadata.Process.Kill(); } } catch (Exception ex) { Log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name); } }
private static void SafelyKillProcess(ProcessMetadata metadata) { try { if (metadata.Process != null && !metadata.Process.HasExited) { metadata.Process.Kill(); } } catch (Exception ex) { DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Failed to verify halt of the {0} process.", metadata.Name)); } }
private static void GrabFreePortForInstance(ProcessMetadata instance) { instance.Port = GetFreeTcpPort(); if (instance.Port == null) { throw new Exception($"Unable to secure a port for {instance.Name}"); } instance.RefreshPortVars(); Log.Debug("Attempting to use port {0} for the {1}.", instance.Port, instance.Name); if (instance.PortFilePath != null) { File.WriteAllText(instance.PortFilePath, instance.Port.Value.ToString(CultureInfo.InvariantCulture)); } }
/// <summary> /// Parses a process and builds an AST component for it /// </summary> /// <param name="network">The top-level network.</param> /// <param name="process">The process to build the AST for.</param> protected virtual Process Parse(NetworkState network, ProcessMetadata process, Simulation simulation) { var st = process.Instance.GetType(); var inputbusses = process.Instance.InputBusses.Union(process.Instance.ClockedInputBusses).ToArray(); var outputbusses = process.Instance.OutputBusses.Distinct().ToArray(); var res = new ProcessState() { Name = NameWithoutPrefix(network, st.FullName, st), SourceType = st, CecilType = LoadType(st), SourceInstance = process, InstanceName = process.InstanceName, Parent = network, IsSimulation = process.Instance is SimulationProcess, IsClocked = st.GetCustomAttribute <ClockedProcessAttribute>() != null, Decompile = ( typeof(SimpleProcess).IsAssignableFrom(st) || typeof(StateProcess).IsAssignableFrom(st) ) && !st.HasAttribute <SuppressOutputAttribute>() && !st.HasAttribute <SuppressBodyAttribute>() }; var proctype = res.CecilType.Resolve(); if (res.CecilType is Mono.Cecil.GenericInstanceType) { var gp = res.CecilType as Mono.Cecil.GenericInstanceType; var names = gp.GenericArguments.ToArray(); var ga = proctype.GenericParameters.ToArray(); foreach (var g in ga) { res.GenericMap[g.Name] = g; res.GenericTypes[g.Name] = names[g.Position]; } } res.InputBusses = inputbusses.Select(x => Parse(network, res, x, simulation)).ToArray(); res.OutputBusses = outputbusses.Select(x => Parse(network, res, x, simulation)).ToArray(); res.InternalBusses = process.Instance.InternalBusses.Select(x => Parse(network, res, x, simulation)).ToArray(); foreach (var ib in res.InternalBusses) { ib.IsInternal = true; } // Set up the local names by finding the field that holds the instance reference foreach (var b in res.InputBusses.Union(res.OutputBusses).Union(res.InternalBusses)) { var f = st .GetFields(BindingFlags.Instance | BindingFlags.FlattenHierarchy | BindingFlags.Public | BindingFlags.NonPublic) .FirstOrDefault(x => x.GetValue(process.Instance) == b.SourceInstance); if (f != null) { res.LocalBusNames[b] = f.Name; } } if (res.Decompile) { // Register all variables foreach (var f in proctype.Fields) { var ft = res.ResolveGenericType(f.FieldType); if (ft.IsBusType()) { RegisterBusReference(network, res, f); } else { RegisterVariable(network, res, f); } } } foreach (var f in process.Initialization) { if (f.Value == null) { continue; } Variable v; if (res.Variables.TryGetValue(f.Key, out v)) { SetDataElementDefaultValue(network, res, v, f.Value, false); } } res.SharedSignals = res.Signals.Values.ToArray(); res.SharedVariables = res.Variables.Values.ToArray(); res.InternalDataElements = new DataElement[0]; return(res); }
public IPotentiallySigned <ServiceMetadata> ParseServiceMetadata(FetcherResponse fetcherResponse) { try { XmlDocument doc = new XmlDocument(); doc.PreserveWhitespace = true; doc.Load(fetcherResponse.InputStream); var o = ClassFactory.FromXmlElement(doc.DocumentElement); Xml.Busdox.tns.ServiceMetadata serviceMetadata = o as Xml.Busdox.tns.ServiceMetadata; X509Certificate signer = null; if (o is SignedServiceMetadata) { signer = XmldsigVerifier.Verify(doc); serviceMetadata = ((SignedServiceMetadata)o).ServiceMetadata; } if (serviceMetadata == null) { throw new LookupException("ServiceMetadata element not found"); } ServiceInformationType serviceInformation = serviceMetadata.ServiceInformation; List <ProcessMetadata <Endpoint> > processMetadatas = new List <ProcessMetadata <Endpoint> >(); foreach (ProcessType processType in serviceInformation.ProcessList.Process) { List <Endpoint> endpoints = new List <Endpoint>(); foreach (EndpointType endpointType in processType.ServiceEndpointList.Endpoint) { var certificate = this.CertificateInstance(Convert.FromBase64String(endpointType.Certificate)); var endpointUri = new Uri(endpointType.EndpointReference.Address.PrimitiveValue); var profile = TransportProfile.Of(endpointType.TransportProfile); endpoints.Add(Endpoint.Of(profile, endpointUri, certificate)); } processMetadatas.Add( ProcessMetadata <Endpoint> .Of( ProcessIdentifier.Of( processType.ProcessIdentifier.PrimitiveValue, Scheme.Of(processType.ProcessIdentifier.Scheme)), endpoints)); } return(Signed <ServiceMetadata> .Of( ServiceMetadata.Of( ParticipantIdentifier.Of( serviceInformation.ParticipantIdentifier.PrimitiveValue, Scheme.Of(serviceInformation.ParticipantIdentifier.Scheme)), DocumentTypeIdentifier.Of( serviceInformation.DocumentIdentifier.PrimitiveValue, Scheme.Of(serviceInformation.DocumentIdentifier.Scheme)), processMetadatas), signer)); } catch (Exception e) when(e is CertificateException | e is IOException) { throw new Exception(e.Message, e); } }
private static Task StartProcessWithKeepAlive(ProcessMetadata metadata) { const int circuitBreakerMax = 3; var path = metadata.ProcessPath; Log.Debug("Starting keep alive for {0}.", path); return(Task.Run( async() => { try { metadata.IsBeingManaged = true; var sequentialFailures = 0; while (true) { if (_cancellationTokenSource.Token.IsCancellationRequested) { Log.Debug("Shutdown triggered for keep alive {0}.", path); return; } try { metadata.IsFaulted = false; if (metadata.ProgramIsRunning()) { Log.Debug("{0} is already running.", path); continue; } var startInfo = new ProcessStartInfo { FileName = path }; if (!string.IsNullOrWhiteSpace(metadata.ProcessArguments)) { startInfo.Arguments = metadata.ProcessArguments; } Log.Debug("Starting {0}.", path); GrabFreePortForInstance(metadata); metadata.Process = Process.Start(startInfo); await Task.Delay(150, _cancellationTokenSource.Token); if (metadata.Process == null || metadata.Process.HasExited) { Log.Error("{0} has failed to start.", path); sequentialFailures++; } else { Log.Debug("Successfully started {0}.", path); sequentialFailures = 0; metadata.AlertSubscribers(); Log.Debug("Finished calling port subscribers for {0}.", metadata.Name); } } catch (Exception ex) { metadata.IsFaulted = true; Log.Error(ex, "Exception when trying to start an instance of {0}.", path); sequentialFailures++; } finally { metadata.HasAttemptedStartup = true; // Delay for a reasonable amount of time before we check to see if the process is alive again. if (metadata.IsFaulted) { // Quicker retry in these cases await Task.Delay(ExceptionRetryInterval, _cancellationTokenSource.Token); } else { await Task.Delay(KeepAliveInterval, _cancellationTokenSource.Token); } } if (sequentialFailures >= circuitBreakerMax) { Log.Error("Circuit breaker triggered for {0}. Max failed retries reached ({1}).", path, sequentialFailures); break; } } } finally { Log.Debug("Keep alive is dropping for {0}.", path); metadata.IsBeingManaged = false; } }, _cancellationTokenSource.Token)); }
private static Task StartProcessWithKeepAlive(ProcessMetadata metadata) { var path = metadata.ProcessPath; Log.Debug("Starting keep alive for {Process}.", path); return(Task.Run( async() => { try { // Indicate that there is a worker responsible for doing this // This acts as a lock metadata.IsBeingManaged = true; while (true) { if (metadata.SequentialFailures >= MaxFailures) { Log.Error <string, int>("Maximum retries ({ErrorCount}) reached starting {Process}.", path, MaxFailures); metadata.ProcessState = ProcessState.Faulted; return; } try { if (metadata.ProcessState == ProcessState.NeverChecked) { // This means we have never tried from this domain if (metadata.ProcessIsHealthy()) { // This means one of two things: // - Another domain has already started this process // - The previous instance is in the process of shutting down // It is possible that if a pipe is bound it may yet be cleaned up from a previous shutdown // Assume healthy to start, but look for problems metadata.ProcessState = ProcessState.Healthy; // Check on a delay to be sure we keep the process available after a shutdown var attempts = 7; var delay = 50d; while (--attempts > 0) { await Task.Delay((int)delay).ConfigureAwait(false); if (metadata.ProcessIsHealthy()) { // Should result in a max delay of ~3.28 seconds before giving up delay = delay * 1.75d; continue; } // The previous instance is gone, time to start the process Log.Information("Recovering from previous {Process} shutdown. Ready for start.", path); metadata.ProcessState = ProcessState.ReadyToStart; break; } } else { // If no instance exists, just get ready to start metadata.ProcessState = ProcessState.ReadyToStart; } } else if (metadata.ProcessState == ProcessState.Healthy || metadata.ProcessState == ProcessState.Faulted) { // This means we have tried to start from this domain before metadata.ProcessState = metadata.ProcessIsHealthy() ? ProcessState.Healthy : ProcessState.ReadyToStart; } if (metadata.ProcessState == ProcessState.ReadyToStart) { Log.Information("Attempting to start {Process}.", path); var startInfo = new ProcessStartInfo { FileName = path, UseShellExecute = false // Force consistency in behavior between Framework and Core }; if (!string.IsNullOrWhiteSpace(metadata.ProcessArguments)) { startInfo.Arguments = metadata.ProcessArguments; } metadata.Process = Process.Start(startInfo); var timeout = 2000; while (timeout > 0) { // Loop and watch for evidence that the process is up and running if (metadata.ProcessIsHealthy()) { metadata.SequentialFailures = 0; metadata.ProcessState = ProcessState.Healthy; Log.Information("Successfully started {Process}.", path); break; } if (metadata.Process == null || metadata.Process.HasExited) { Log.Error("Failed to start {Process}.", path); metadata.ProcessState = ProcessState.Faulted; break; } await Task.Delay(100).ConfigureAwait(false); timeout -= 100; } } } catch (Exception ex) { metadata.ProcessState = ProcessState.Faulted; Log.Error(ex, "Failed to start {Process}.", path); } finally { if (metadata.ProcessState == ProcessState.Faulted) { metadata.SequentialFailures++; // Quicker retry in these cases await Task.Delay(ExceptionRetryInterval).ConfigureAwait(false); } else { // Delay for a reasonable amount of time before we check to see if the process is alive again. await Task.Delay(KeepAliveInterval).ConfigureAwait(false); } } } } finally { Log.Warning("Keep alive is dropping for {Process}.", path); metadata.IsBeingManaged = false; } })); }
private static Task StartProcessWithKeepAlive(string path, string args, ProcessMetadata metadata) { DatadogLogging.RegisterStartupLog(log => log.Debug("Starting keep alive for {0}.", path)); return(Task.Run( () => { try { var circuitBreakerMax = 3; var sequentialFailures = 0; while (true) { if (_cancellationTokenSource.IsCancellationRequested) { DatadogLogging.RegisterStartupLog(log => log.Debug("Shutdown triggered for keep alive {0}.", path)); return; } try { if (metadata.Process != null && metadata.Process.HasExited == false) { DatadogLogging.RegisterStartupLog(log => log.Debug("We already have an active reference to {0}.", path)); continue; } if (ProgramIsRunning(path)) { DatadogLogging.RegisterStartupLog(log => log.Debug("{0} is already running.", path)); continue; } var startInfo = new ProcessStartInfo { FileName = path }; if (!string.IsNullOrWhiteSpace(args)) { startInfo.Arguments = args; } DatadogLogging.RegisterStartupLog(log => log.Debug("Starting {0}.", path)); metadata.PreStartAction?.Invoke(); metadata.Process = Process.Start(startInfo); Thread.Sleep(200); if (metadata.Process == null || metadata.Process.HasExited) { DatadogLogging.RegisterStartupLog(log => log.Error("{0} has failed to start.", path)); sequentialFailures++; } else { DatadogLogging.RegisterStartupLog(log => log.Debug("Successfully started {0}.", path)); sequentialFailures = 0; foreach (var portSubscriber in metadata.PortSubscribers) { portSubscriber(metadata.Port.Value); } DatadogLogging.RegisterStartupLog(log => log.Debug("Finished calling port subscribers for {0}.", metadata.Name)); } } catch (Exception ex) { DatadogLogging.RegisterStartupLog(log => log.Error(ex, "Exception when trying to start an instance of {0}.", path)); sequentialFailures++; } finally { // Delay for a reasonable amount of time before we check to see if the process is alive again. Thread.Sleep(20_000); } if (sequentialFailures >= circuitBreakerMax) { DatadogLogging.RegisterStartupLog(log => log.Error("Circuit breaker triggered for {0}. Max failed retries reached ({1}).", path, sequentialFailures)); break; } } } finally { DatadogLogging.RegisterStartupLog(log => log.Debug("Keep alive is dropping for {0}.", path)); } })); }
public IHttpActionResult Run(ProcessMetadata metadata) { // Queue the long running action in the background. HostingEnvironment.QueueBackgroundWorkItem(x => _service.Run(metadata)); return(Ok()); }