public async Task <bool> SendDeployStatusUpdate(DeployStatusApiResult updateStatusObj) { ITracer tracer = _traceFactory.GetTracer(); int attemptCount = 0; try { await OperationManager.AttemptAsync(async() => { attemptCount++; tracer.Trace($" PostAsync - Trying to send {updateStatusObj.DeploymentStatus} deployment status to {Constants.UpdateDeployStatusPath}. Deployment Id is {updateStatusObj.DeploymentId}"); await PostDeploymentHelper.PostAsync(Constants.UpdateDeployStatusPath, _environment.RequestId, content: JsonConvert.SerializeObject(updateStatusObj)); }, 3, 5 * 1000); // If no exception is thrown, the operation was a success return(true); } catch (Exception ex) { tracer.TraceError($"Failed to request a post deployment status. Number of attempts: {attemptCount}. Exception: {ex}"); // Do not throw the exception // We fail silently so that we do not fail the build altogether if this call fails //throw; return(false); } }
public async Task RestartMainSiteIfNeeded(ITracer tracer, ILogger logger, DeploymentInfoBase deploymentInfo) { // If post-deployment restart is disabled, do nothing. if (!_settings.RestartAppOnGitDeploy()) { return; } // Proceed only if 'restart' is allowed for this deployment if (deploymentInfo != null && !deploymentInfo.RestartAllowed) { return; } if (deploymentInfo != null && !string.IsNullOrEmpty(deploymentInfo.DeploymentTrackingId)) { // Send deployment status update to FE // FE will modify the operations table with the PostBuildRestartRequired status DeployStatusApiResult updateStatusObj = new DeployStatusApiResult(Constants.PostBuildRestartRequired, deploymentInfo.DeploymentTrackingId); bool isSuccess = await SendDeployStatusUpdate(updateStatusObj); if (isSuccess) { // If operation is a success and PostBuildRestartRequired was posted successfully to the operations DB, then return // Else fallthrough to RestartMainSiteAsync return; } } if (deploymentInfo != null && deploymentInfo.Deployer == Constants.OneDeploy) { await PostDeploymentHelper.RestartMainSiteAsync(_environment.RequestId, new PostDeploymentTraceListener(tracer, logger)); return; } if (_settings.RecylePreviewEnabled()) { logger.Log("Triggering recycle (preview mode enabled)."); tracer.Trace("Triggering recycle (preview mode enabled)."); await PostDeploymentHelper.RestartMainSiteAsync(_environment.RequestId, new PostDeploymentTraceListener(tracer, logger)); } else { logger.Log("Triggering recycle (preview mode disabled)."); tracer.Trace("Triggering recycle (preview mode disabled)."); DockerContainerRestartTrigger.RequestContainerRestart(_environment, RestartTriggerReason, tracer); } }
/// <summary> /// This method tries to send the deployment status update through frontend to be saved to db /// Since frontend throttling is in place, we retry 3 times with 5 sec gaps in between /// </summary> /// <param name="updateStatusObj">Obj containing status to save to DB</param> /// <returns></returns> private async Task SendDeployStatusUpdate(DeployStatusApiResult updateStatusObj) { int attemptCount = 0; try { await OperationManager.AttemptAsync(async() => { attemptCount++; _tracer.Trace($" PostAsync - Trying to send {updateStatusObj.DeploymentStatus} deployment status to {Constants.UpdateDeployStatusPath}"); await PostDeploymentHelper.PostAsync(Constants.UpdateDeployStatusPath, _environment.RequestId, JsonConvert.SerializeObject(updateStatusObj)); }, 3, 5 *1000); } catch (Exception ex) { _tracer.TraceError($"Failed to request a post deployment status. Number of attempts: {attemptCount}. Exception: {ex}"); throw; } }
public async Task PerformDeployment(DeploymentInfoBase deploymentInfo, IDisposable tempDeployment = null, ChangeSet tempChangeSet = null) { DateTime currentMarkerFileUTC; DateTime nextMarkerFileUTC = FileSystemHelpers.GetLastWriteTimeUtc(_markerFilePath); ChangeSet lastChange = null; do { // save the current marker currentMarkerFileUTC = nextMarkerFileUTC; string targetBranch = _settings.GetBranch(); using (_tracer.Step("Performing fetch based deployment")) { // create temporary deployment before the actual deployment item started // this allows portal ui to readily display on-going deployment (not having to wait for fetch to complete). // in addition, it captures any failure that may occur before the actual deployment item started tempDeployment = tempDeployment ?? _deploymentManager.CreateTemporaryDeployment( Resources.ReceivingChanges, out tempChangeSet, deploymentInfo.TargetChangeset, deploymentInfo.Deployer); ILogger innerLogger = null; DeployStatusApiResult updateStatusObj = null; try { ILogger logger = _deploymentManager.GetLogger(tempChangeSet.Id); // Fetch changes from the repository innerLogger = logger.Log(Resources.FetchingChanges); IRepository repository = deploymentInfo.GetRepository(); try { await deploymentInfo.Fetch(repository, deploymentInfo, targetBranch, innerLogger, _tracer); } catch (BranchNotFoundException) { // mark no deployment is needed deploymentInfo.TargetChangeset = null; } // set to null as Deploy() below takes over logging innerLogger = null; // The branch or commit id to deploy string deployBranch = !String.IsNullOrEmpty(deploymentInfo.CommitId) ? deploymentInfo.CommitId : targetBranch; try { _tracer.Trace($"Before sending {Constants.BuildRequestReceived} status to /api/updatedeploystatus"); if (PostDeploymentHelper.IsAzureEnvironment()) { if (deploymentInfo != null && !string.IsNullOrEmpty(deploymentInfo.DeploymentTrackingId)) { // Only send an updatedeploystatus request if DeploymentTrackingId is non null // This signifies the client has opted in for these deployment updates for this deploy request updateStatusObj = new DeployStatusApiResult(Constants.BuildRequestReceived, deploymentInfo.DeploymentTrackingId); await _deploymentManager.SendDeployStatusUpdate(updateStatusObj); } } } catch (Exception e) { _tracer.TraceError($"Exception while sending {Constants.BuildRequestReceived} status to /api/updatedeploystatus. " + $"Entry in the operations table for the deployment status may not have been created. {e}"); } // In case the commit or perhaps fetch do no-op. if (deploymentInfo.TargetChangeset != null && ShouldDeploy(repository, deploymentInfo, deployBranch)) { // Perform the actual deployment var changeSet = repository.GetChangeSet(deployBranch); if (changeSet == null && !String.IsNullOrEmpty(deploymentInfo.CommitId)) { throw new InvalidOperationException(String.Format("Invalid revision '{0}'!", deploymentInfo.CommitId)); } lastChange = changeSet; // Here, we don't need to update the working files, since we know Fetch left them in the correct state // unless for GenericHandler where specific commitId is specified bool deploySpecificCommitId = !String.IsNullOrEmpty(deploymentInfo.CommitId); if (updateStatusObj != null) { updateStatusObj.DeploymentStatus = Constants.BuildInProgress; await _deploymentManager.SendDeployStatusUpdate(updateStatusObj); } await _deploymentManager.DeployAsync( repository, changeSet, deploymentInfo.Deployer, clean : false, deploymentInfo : deploymentInfo, needFileUpdate : deploySpecificCommitId, fullBuildByDefault : deploymentInfo.DoFullBuildByDefault); if (updateStatusObj != null && !deploymentInfo.RestartAllowed) { // If restart is disallowed, send BuildSuccessful here as PostBuildRestartRequired was not sent // during the DeployAsync flow. updateStatusObj.DeploymentStatus = Constants.BuildSuccessful; await _deploymentManager.SendDeployStatusUpdate(updateStatusObj); } } } catch (Exception ex) { if (innerLogger != null) { innerLogger.Log(ex); } // In case the commit or perhaps fetch do no-op. if (deploymentInfo.TargetChangeset != null) { IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id); if (statusFile != null) { statusFile.MarkFailed(); } } try { if (updateStatusObj != null) { // Set deployment status as failure if exception is thrown updateStatusObj.DeploymentStatus = Constants.BuildFailed; await _deploymentManager.SendDeployStatusUpdate(updateStatusObj); } } catch { // no-op } throw; } // only clean up temp deployment if successful tempDeployment.Dispose(); } // check marker file and, if changed (meaning new /deploy request), redeploy. nextMarkerFileUTC = FileSystemHelpers.GetLastWriteTimeUtc(_markerFilePath); } while (deploymentInfo.IsReusable && currentMarkerFileUTC != nextMarkerFileUTC); if (lastChange != null && PostDeploymentHelper.IsAutoSwapEnabled()) { IDeploymentStatusFile statusFile = _status.Open(lastChange.Id); if (statusFile.Status == DeployStatus.Success) { // if last change is not null and finish successfully, mean there was at least one deployment happened // since deployment is now done, trigger swap if enabled await PostDeploymentHelper.PerformAutoSwap(_environment.RequestId, new PostDeploymentTraceListener(_tracer, _deploymentManager.GetLogger(lastChange.Id))); } } }