示例#1
0
 /// <inheritdoc />
 public async Task StopAsync(CancellationToken cancellationToken)
 {
     if (releaseServers && Running)
     {
         var reattachInformation = new WatchdogReattachInformation
         {
             AlphaIsActive = AlphaIsActive
         };
         reattachInformation.Alpha = alphaServer?.Release();
         reattachInformation.Bravo = bravoServer?.Release();
         await reattachInfoHandler.Save(reattachInformation, cancellationToken).ConfigureAwait(false);
     }
     await Terminate(false, cancellationToken).ConfigureAwait(false);
 }
示例#2
0
 /// <summary>
 /// Starts all <see cref="ISessionController"/>s.
 /// </summary>
 /// <param name="callBeforeRecurse">An <see cref="Action"/> that must be run before making a recursive call to <see cref="LaunchImplNoLock(bool, bool, WatchdogReattachInformation, CancellationToken)"/>.</param>
 /// <param name="chatTask">A, possibly active, <see cref="Task"/> for an outgoing chat message.</param>
 /// <param name="reattachInfo"><see cref="WatchdogReattachInformation"/> to use, if any</param>
 /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
 /// <returns>A <see cref="Task"/> representing the running operation</returns>
 protected abstract Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken);
        /// <inheritdoc />
        protected sealed override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
        {
            Debug.Assert(Server == null, "Entered LaunchNoLock with server not being null!");

            // don't need a new dmb if reattaching
            var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null;
            var dmbToUse         = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(1);

            var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo;
            var serverToKill     = reattachInfo?.Bravo ?? reattachInfo?.Alpha;

            // vice versa
            if (reattachInfo?.AlphaIsActive == false)
            {
                var temp = serverToReattach;
                serverToReattach = serverToKill;
                serverToKill     = temp;
            }

            // if this try catches something, both servers are killed
            bool inactiveServerWasKilled = false;

            try
            {
                // start the alpha server task, either by launch a new process or attaching to an existing one
                // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
                // The tasks pertaining to server startup times are in the ISessionControllers
                Task <ISessionController> serverLaunchTask, inactiveReattachTask;
                if (!doesntNeedNewDmb)
                {
                    dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken).ConfigureAwait(false);

                    serverLaunchTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken);
                }
                else
                {
                    serverLaunchTask = SessionControllerFactory.Reattach(serverToReattach, cancellationToken);
                }

                bool thereIsAnInactiveServerToKill = serverToKill != null;
                if (thereIsAnInactiveServerToKill)
                {
                    inactiveReattachTask = SessionControllerFactory.Reattach(serverToKill, cancellationToken);
                }
                else
                {
                    inactiveReattachTask = Task.FromResult <ISessionController>(null);
                }

                // retrieve the session controller
                Server = await serverLaunchTask.ConfigureAwait(false);

                // failed reattaches will return null
                Server?.SetHighPriority();

                var inactiveServerController = await inactiveReattachTask.ConfigureAwait(false);

                inactiveServerController?.Dispose();
                inactiveServerWasKilled = inactiveServerController != null;

                // possiblity of null servers due to failed reattaches
                if (Server == null)
                {
                    callBeforeRecurse();
                    await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false);

                    return;
                }

                await CheckLaunchResult(Server, "Server", cancellationToken).ConfigureAwait(false);

                Server.EnableCustomChatCommands();
            }
            catch
            {
                // kill the controllers
                bool serverWasActive = Server != null;
                DisposeAndNullControllers();

                // server didn't get control of this dmb
                if (dmbToUse != null && !serverWasActive)
                {
                    dmbToUse.Dispose();
                }

                if (serverToKill != null && !inactiveServerWasKilled)
                {
                    serverToKill.Dmb.Dispose();
                }
                throw;
            }
        }
示例#4
0
        /// <summary>
        /// Launches the watchdog.
        /// </summary>
        /// <param name="startMonitor">If <see cref="MonitorLifetimes(CancellationToken)"/> should be started by this function</param>
        /// <param name="announce">If the launch should be announced to chat by this function</param>
        /// <param name="reattachInfo"><see cref="WatchdogReattachInformation"/> to use, if any</param>
        /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
        /// <returns>A <see cref="Task"/> representing the running operation</returns>
        protected async Task LaunchImplNoLock(bool startMonitor, bool announce, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
        {
            Logger.LogTrace("Begin LaunchNoLock");

            if (Running)
            {
                throw new JobException("Watchdog already running!");
            }

            if (!DmbFactory.DmbAvailable)
            {
                throw new JobException("Corrupted compilation, please redeploy!");
            }

            // this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
            Task chatTask;

            if (startMonitor && await StopMonitor().ConfigureAwait(false))
            {
                chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", cancellationToken);
            }
            else if (announce)
            {
                chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Starting..." : "Reattaching...", cancellationToken);                 // simple announce
            }
            else
            {
                chatTask = Task.CompletedTask;                 // no announce
            }
            // since neither server is running, this is safe to do
            LastLaunchParameters = ActiveLaunchParameters;

            // for when we call ourself and want to not catch thrown exceptions
            var ignoreNestedException = false;

            try
            {
                await InitControllers(() => ignoreNestedException = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false);

                await chatTask.ConfigureAwait(false);

                Logger.LogInformation("Launched servers successfully");
                Running = true;

                if (startMonitor)
                {
                    StartMonitor();
                }
            }
            catch (Exception e)
            {
                // don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled
                if (!ignoreNestedException && !cancellationToken.IsCancellationRequested)
                {
                    var originalChatTask = chatTask;
                    async Task ChainChatTaskWithErrorMessage()
                    {
                        await originalChatTask.ConfigureAwait(false);

                        await Chat.SendWatchdogMessage("Startup failed!", cancellationToken).ConfigureAwait(false);
                    }

                    chatTask = ChainChatTaskWithErrorMessage();
                    Logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
                }

                throw;
            }
            finally
            {
                // finish the chat task that's in flight
                try
                {
                    await chatTask.ConfigureAwait(false);
                }
                catch (OperationCanceledException) { }
            }
        }
                #pragma warning restore CA1502

        /// <inheritdoc />
                #pragma warning disable CA1502 // TODO: Decomplexify
        protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
        {
            Debug.Assert(alphaServer == null && bravoServer == null, "Entered LaunchNoLock with one or more of the servers not being null!");

            // don't need a new dmb if reattaching
            var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null;
            var dmbToUse         = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(2);

            // if this try catches something, both servers are killed
            try
            {
                // start the alpha server task, either by launch a new process or attaching to an existing one
                // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
                // The tasks pertaining to server startup times are in the ISessionControllers
                Task <ISessionController> alphaServerTask;
                if (!doesntNeedNewDmb)
                {
                    alphaServerTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken);
                }
                else
                {
                    alphaServerTask = SessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken);
                }

                // retrieve the session controller
                var startTime = DateTimeOffset.Now;
                alphaServer = await alphaServerTask.ConfigureAwait(false);

                // failed reattaches will return null
                alphaServer?.SetHighPriority();

                // extra delay for total ordering
                var now   = DateTimeOffset.Now;
                var delay = now - startTime;

                // definitely not if reattaching though
                if (reattachInfo == null && delay.TotalSeconds < AlphaBravoStartupSeperationInterval)
                {
                    await AsyncDelayer.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false);
                }

                // now bring bravo up
                if (!doesntNeedNewDmb)
                {
                    bravoServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken).ConfigureAwait(false);
                }
                else
                {
                    bravoServer = await SessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken).ConfigureAwait(false);
                }

                // failed reattaches will return null
                bravoServer?.SetHighPriority();

                // possiblity of null servers due to failed reattaches
                if (alphaServer == null || bravoServer == null)
                {
                    await chatTask.ConfigureAwait(false);

                    var bothServersDead = alphaServer == null && bravoServer == null;
                    if (bothServersDead ||
                        (alphaServer == null && reattachInfo.AlphaIsActive) ||
                        (bravoServer == null && !reattachInfo.AlphaIsActive))
                    {
                        // we lost the active server, just restart entirely
                        DisposeAndNullControllers();
                        const string FailReattachMessage = "Unable to properly reattach to active server! Restarting...";
                        Logger.LogWarning(FailReattachMessage);
                        Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
                        chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken);
                        callBeforeRecurse();
                        await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);

                        await chatTask.ConfigureAwait(false);

                        return;
                    }

                    // we still have the active server but the other one is dead to us, hand it off to the monitor to restart
                    const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot...";
                    chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken);
                    Logger.LogWarning(InactiveReattachFailureMessage);

                    if (reattachInfo.AlphaIsActive)
                    {
                        bravoServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Bravo.Dmb);
                    }
                    else
                    {
                        alphaServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Alpha.Dmb);
                    }
                }

                var alphaLrt = CheckLaunchResult(alphaServer, "Alpha", cancellationToken);
                var bravoLrt = CheckLaunchResult(bravoServer, "Bravo", cancellationToken);

                // this task completes when both serers have finished booting
                var allTask = Task.WhenAll(alphaLrt, bravoLrt);

                await allTask.ConfigureAwait(false);

                // both servers are now running, alpha is the active server(unless reattach), huzzah
                alphaIsActive = reattachInfo?.AlphaIsActive ?? true;

                var activeServer = AlphaIsActive ? alphaServer : bravoServer;
                activeServer.EnableCustomChatCommands();
                activeServer.ClosePortOnReboot = true;
            }
            catch
            {
                if (dmbToUse != null)
                {
                    // we locked 2 dmbs
                    if (bravoServer == null)
                    {
                        // bravo didn't get control of his
                        dmbToUse.Dispose();
                        if (alphaServer == null)
                        {
                            dmbToUse.Dispose();                             // alpha didn't get control of his
                        }
                    }
                }
                else if (doesntNeedNewDmb)                 // we have reattachInfo
                {
                    if (bravoServer == null)
                    {
                        // bravo didn't get control of his
                        reattachInfo.Bravo?.Dmb.Dispose();
                        if (alphaServer == null)
                        {
                            reattachInfo.Alpha?.Dmb.Dispose();                             // alpha didn't get control of his
                        }
                    }
                }

                // kill the controllers
                DisposeAndNullControllers();
                throw;
            }
        }