public async Task Deploy(string id = null)
{
JObject result = GetJsonContent();
// Just block here to read the json payload from the body
using (_tracer.Step("DeploymentService.Deploy(id)"))
{
await _deploymentLock.LockHttpOperationAsync(async() =>
{
try
{
if (_autoSwapHandler.IsAutoSwapOngoing())
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.Conflict, Resources.Error_AutoSwapDeploymentOngoing));
}
bool clean = false;
bool needFileUpdate = true;
if (result != null)
{
clean = result.Value <bool>("clean");
JToken needFileUpdateToken;
if (result.TryGetValue("needFileUpdate", out needFileUpdateToken))
{
needFileUpdate = needFileUpdateToken.Value <bool>();
}
}
string username = null;
AuthUtility.TryExtractBasicAuthUser(Request, out username);
IRepository repository = _repositoryFactory.GetRepository();
if (repository == null)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, Resources.Error_RepositoryNotFound));
}
ChangeSet changeSet = null;
if (!String.IsNullOrEmpty(id))
{
changeSet = repository.GetChangeSet(id);
if (changeSet == null)
{
string message = String.Format(CultureInfo.CurrentCulture, Resources.Error_DeploymentNotFound, id);
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, message));
}
}
await _deploymentManager.DeployAsync(repository, changeSet, username, clean, needFileUpdate);
_autoSwapHandler.HandleAutoSwap(verifyActiveDeploymentIdChanged: false);
}
catch (FileNotFoundException ex)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, ex));
}
});
}
}
public static void Deploy(this IDeploymentManager deploymentManager, IRepository repository, string id, string deployer, bool clean)
{
ChangeSet changeSet = null;
if (id != null)
{
changeSet = repository.GetChangeSet(id);
}
deploymentManager.Deploy(repository, changeSet, deployer, clean);
}
// For continuous integration, we will only build/deploy if fetch new changes
// The immediate goal is to address duplicated /deploy requests from Bitbucket (retry if taken > 20s)
private bool ShouldDeploy(IRepository repository, DeploymentInfoBase deploymentInfo, string targetBranch)
{
if (deploymentInfo.IsContinuous)
{
ChangeSet changeSet = repository.GetChangeSet(targetBranch);
return(!String.Equals(_status.ActiveDeploymentId, changeSet.Id, StringComparison.OrdinalIgnoreCase));
}
return(true);
}
public static void Deploy(this IDeploymentManager deploymentManager, IRepository repository, string id, string deployer, bool clean)
{
ChangeSet changeSet = null;
if (id != null)
{
changeSet = repository.GetChangeSet(id);
}
deploymentManager.Deploy(repository, changeSet, deployer, clean);
}
public async Task Deploy(string id = null)
{
JObject result = GetJsonContent();
// Just block here to read the json payload from the body
using (_tracer.Step("DeploymentService.Deploy(id)"))
{
await _deploymentLock.LockHttpOperationAsync(async() =>
{
try
{
bool clean = false;
if (result != null)
{
clean = result.Value <bool>("clean");
}
string username = null;
AuthUtility.TryExtractBasicAuthUser(Request, out username);
IRepository repository = _repositoryFactory.GetRepository();
if (repository == null)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, Resources.Error_RepositoryNotFound));
}
ChangeSet changeSet = null;
if (!String.IsNullOrEmpty(id))
{
changeSet = repository.GetChangeSet(id);
if (changeSet == null)
{
string message = String.Format(CultureInfo.CurrentCulture, Resources.Error_DeploymentNotFound, id);
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, message));
}
}
await _deploymentManager.DeployAsync(repository, changeSet, username, clean);
}
catch (FileNotFoundException ex)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, ex));
}
});
}
}
public async Task DeployAsync(IRepository repository, ChangeSet changeSet, string deployer, bool clean, bool needFileUpdate = true)
{
using (var deploymentAnalytics = new DeploymentAnalytics(_analytics, _settings))
{
Exception exception = null;
ITracer tracer = _traceFactory.GetTracer();
IDisposable deployStep = null;
ILogger innerLogger = null;
string targetBranch = null;
// If we don't get a changeset, find out what branch we should be deploying and get the commit ID from it
if (changeSet == null)
{
targetBranch = _settings.GetBranch();
changeSet = repository.GetChangeSet(targetBranch);
if (changeSet == null)
{
throw new InvalidOperationException(String.Format("The current deployment branch is '{0}', but nothing has been pushed to it", targetBranch));
}
}
string id = changeSet.Id;
IDeploymentStatusFile statusFile = null;
try
{
deployStep = tracer.Step("DeploymentManager.Deploy(id)");
// Remove the old log file for this deployment id
string logPath = GetLogPath(id);
FileSystemHelpers.DeleteFileSafe(logPath);
statusFile = GetOrCreateStatusFile(changeSet, tracer, deployer);
statusFile.MarkPending();
ILogger logger = GetLogger(changeSet.Id);
if (needFileUpdate)
{
using (tracer.Step("Updating to specific changeset"))
{
innerLogger = logger.Log(Resources.Log_UpdatingBranch, targetBranch ?? id);
using (var writer = new ProgressWriter())
{
// Update to the specific changeset or branch
repository.Update(targetBranch ?? id);
}
}
}
if (_settings.ShouldUpdateSubmodules())
{
using (tracer.Step("Updating submodules"))
{
innerLogger = logger.Log(Resources.Log_UpdatingSubmodules);
repository.UpdateSubmodules();
}
}
if (clean)
{
tracer.Trace("Cleaning {0} repository", repository.RepositoryType);
innerLogger = logger.Log(Resources.Log_CleaningRepository, repository.RepositoryType);
repository.Clean();
}
// set to null as Build() below takes over logging
innerLogger = null;
// Perform the build deployment of this changeset
await Build(changeSet, tracer, deployStep, repository, deploymentAnalytics);
}
catch (Exception ex)
{
exception = ex;
if (innerLogger != null)
{
innerLogger.Log(ex);
}
if (statusFile != null)
{
MarkStatusComplete(statusFile, success: false);
}
tracer.TraceError(ex);
deploymentAnalytics.Error = ex.ToString();
if (deployStep != null)
{
deployStep.Dispose();
}
}
// Reload status file with latest updates
statusFile = _status.Open(id);
if (statusFile != null)
{
await _hooksManager.PublishEventAsync(HookEventTypes.PostDeployment, statusFile);
}
if (exception != null)
{
throw new DeploymentFailedException(exception);
}
}
}
internal async Task <ChangeSet> Sync(DropboxInfo dropboxInfo, string branch, IRepository repository, ITracer tracer)
{
DropboxDeployInfo deployInfo = dropboxInfo.DeployInfo;
// use incoming tracer since it is background work
_tracer = tracer;
ResetStats();
// for Dropbox OAuth V2, the delta is collected and applied by SCM
// simply set OldCursor as current.
if (dropboxInfo.OAuthVersion == 2)
{
deployInfo.OldCursor = _settings.GetValue(CursorKey);
}
else if (_settings.GetValue(CursorKey) != deployInfo.OldCursor)
{
throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor);
}
// initial sync, remove default content
// for simplicity, we do it blindly whether or not in-place
// given the end result is the same
if (String.IsNullOrEmpty(deployInfo.OldCursor) && DeploymentHelper.IsDefaultWebRootContent(_environment.WebRootPath))
{
string hoststarthtml = Path.Combine(_environment.WebRootPath, Constants.HostingStartHtml);
FileSystemHelpers.DeleteFileSafe(hoststarthtml);
}
if (!repository.IsEmpty())
{
// git checkout --force <branch>
repository.Update(branch);
}
ChangeSet changeSet = null;
string message = null;
try
{
using (_tracer.Step("Sync with Dropbox"))
{
if (dropboxInfo.OAuthVersion == 2)
{
// Fetch the deltas
await UpdateDropboxDeployInfo(deployInfo);
}
// Sync dropbox => repository directory
await ApplyChanges(dropboxInfo, useOAuth20 : dropboxInfo.OAuthVersion == 2);
}
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_Synchronized,
deployInfo.Deltas.Count);
}
catch (Exception)
{
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_SynchronizedWithFailure,
_successCount,
deployInfo.Deltas.Count,
_failedCount);
throw;
}
finally
{
Logger.Log(message);
Logger.Log(String.Format("{0} downloaded files, {1} successful retries.", _fileCount, _retriedCount));
IDeploymentStatusFile statusFile = _status.Open(dropboxInfo.TargetChangeset.Id);
statusFile.UpdateMessage(message);
statusFile.UpdateProgress(String.Format(CultureInfo.CurrentCulture, Resources.Dropbox_Committing, _successCount));
// Commit anyway even partial change
if (repository.Commit(message, deployInfo.UserName, deployInfo.Email ?? deployInfo.UserName))
{
changeSet = repository.GetChangeSet("HEAD");
}
}
// Save new dropbox cursor
LogInfo("Update dropbox cursor");
_settings.SetValue(CursorKey, deployInfo.NewCursor);
return(changeSet);
}
// For continuous integration, we will only build/deploy if fetch new changes
// The immediate goal is to address duplicated /deploy requests from Bitbucket (retry if taken > 20s)
private bool ShouldDeploy(IRepository repository, DeploymentInfo deploymentInfo, string targetBranch)
{
if (deploymentInfo.IsContinuous)
{
ChangeSet changeSet = repository.GetChangeSet(targetBranch);
return !String.Equals(_status.ActiveDeploymentId, changeSet.Id, StringComparison.OrdinalIgnoreCase);
}
return true;
}
public async Task PerformDeployment(DeploymentInfo deploymentInfo, IDisposable tempDeployment = null, ChangeSet tempChangeSet = null)
{
DateTime currentMarkerFileUTC;
DateTime nextMarkerFileUTC = FileSystemHelpers.GetLastWriteTimeUtc(_markerFilePath);
ChangeSet lastChange = null;
do
{
// save the current marker
currentMarkerFileUTC = nextMarkerFileUTC;
string targetBranch = _settings.GetBranch();
using (_tracer.Step("Performing fetch based deployment"))
{
// create temporary deployment before the actual deployment item started
// this allows portal ui to readily display on-going deployment (not having to wait for fetch to complete).
// in addition, it captures any failure that may occur before the actual deployment item started
tempDeployment = tempDeployment ?? _deploymentManager.CreateTemporaryDeployment(
Resources.ReceivingChanges,
out tempChangeSet,
deploymentInfo.TargetChangeset,
deploymentInfo.Deployer);
ILogger innerLogger = null;
try
{
ILogger logger = _deploymentManager.GetLogger(tempChangeSet.Id);
// Fetch changes from the repository
innerLogger = logger.Log(Resources.FetchingChanges);
IRepository repository = _repositoryFactory.EnsureRepository(deploymentInfo.RepositoryType);
try
{
await deploymentInfo.Handler.Fetch(repository, deploymentInfo, targetBranch, innerLogger, _tracer);
}
catch (BranchNotFoundException)
{
// mark no deployment is needed
deploymentInfo.TargetChangeset = null;
}
// set to null as Deploy() below takes over logging
innerLogger = null;
// The branch or commit id to deploy
string deployBranch = !String.IsNullOrEmpty(deploymentInfo.CommitId) ? deploymentInfo.CommitId : targetBranch;
// In case the commit or perhaps fetch do no-op.
if (deploymentInfo.TargetChangeset != null && ShouldDeploy(repository, deploymentInfo, deployBranch))
{
// Perform the actual deployment
var changeSet = repository.GetChangeSet(deployBranch);
if (changeSet == null && !String.IsNullOrEmpty(deploymentInfo.CommitId))
{
throw new InvalidOperationException(String.Format("Invalid revision '{0}'!", deploymentInfo.CommitId));
}
lastChange = changeSet;
// Here, we don't need to update the working files, since we know Fetch left them in the correct state
// unless for GenericHandler where specific commitId is specified
bool deploySpecificCommitId = !String.IsNullOrEmpty(deploymentInfo.CommitId);
await _deploymentManager.DeployAsync(repository, changeSet, deploymentInfo.Deployer, clean : false, needFileUpdate : deploySpecificCommitId);
}
}
catch (Exception ex)
{
if (innerLogger != null)
{
innerLogger.Log(ex);
}
// In case the commit or perhaps fetch do no-op.
if (deploymentInfo.TargetChangeset != null)
{
IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id);
if (statusFile != null)
{
statusFile.MarkFailed();
}
}
throw;
}
// only clean up temp deployment if successful
tempDeployment.Dispose();
}
// check marker file and, if changed (meaning new /deploy request), redeploy.
nextMarkerFileUTC = FileSystemHelpers.GetLastWriteTimeUtc(_markerFilePath);
} while (deploymentInfo.IsReusable && currentMarkerFileUTC != nextMarkerFileUTC);
if (lastChange != null && PostDeploymentHelper.IsAutoSwapEnabled())
{
IDeploymentStatusFile statusFile = _status.Open(lastChange.Id);
if (statusFile.Status == DeployStatus.Success)
{
// if last change is not null and finish successfully, mean there was at least one deployoment happened
// since deployment is now done, trigger swap if enabled
await PostDeploymentHelper.PerformAutoSwap(_environment.RequestId, _environment.SiteRestrictedJwt, new PostDeploymentTraceListener(_tracer, _deploymentManager.GetLogger(lastChange.Id)));
}
}
}
public async Task DeployAsync(
IRepository repository,
ChangeSet changeSet,
string deployer,
bool clean,
DeploymentInfoBase deploymentInfo = null,
bool needFileUpdate = true,
bool fullBuildByDefault = true)
{
using (var deploymentAnalytics = new DeploymentAnalytics(_analytics, _settings))
{
Exception exception = null;
ITracer tracer = _traceFactory.GetTracer();
IDisposable deployStep = null;
ILogger innerLogger = null;
string targetBranch = null;
// If we don't get a changeset, find out what branch we should be deploying and get the commit ID from it
if (changeSet == null)
{
targetBranch = _settings.GetBranch();
changeSet = repository.GetChangeSet(targetBranch);
if (changeSet == null)
{
throw new InvalidOperationException(String.Format("The current deployment branch is '{0}', but nothing has been pushed to it", targetBranch));
}
}
string id = changeSet.Id;
IDeploymentStatusFile statusFile = null;
try
{
deployStep = tracer.Step($"DeploymentManager.Deploy(id:{id})");
// Remove the old log file for this deployment id
string logPath = GetLogPath(id);
FileSystemHelpers.DeleteFileSafe(logPath);
statusFile = GetOrCreateStatusFile(changeSet, tracer, deployer);
statusFile.MarkPending();
ILogger logger = GetLogger(changeSet.Id);
if (needFileUpdate)
{
using (tracer.Step("Updating to specific changeset"))
{
innerLogger = logger.Log(Resources.Log_UpdatingBranch, targetBranch ?? id);
using (var writer = new ProgressWriter())
{
// Update to the specific changeset or branch
repository.Update(targetBranch ?? id);
}
}
}
if (_settings.ShouldUpdateSubmodules())
{
using (tracer.Step("Updating submodules"))
{
innerLogger = logger.Log(Resources.Log_UpdatingSubmodules);
repository.UpdateSubmodules();
}
}
if (clean)
{
tracer.Trace("Cleaning {0} repository", repository.RepositoryType);
innerLogger = logger.Log(Resources.Log_CleaningRepository, repository.RepositoryType);
repository.Clean();
}
// set to null as Build() below takes over logging
innerLogger = null;
// Perform the build deployment of this changeset
await Build(changeSet, tracer, deployStep, repository, deploymentInfo, deploymentAnalytics, fullBuildByDefault);
}
catch (Exception ex)
{
exception = ex;
if (innerLogger != null)
{
innerLogger.Log(ex);
}
if (statusFile != null)
{
MarkStatusComplete(statusFile, success: false);
}
tracer.TraceError(ex);
deploymentAnalytics.Error = ex.ToString();
if (deployStep != null)
{
deployStep.Dispose();
}
}
// Remove leftover AppOffline file
PostDeploymentHelper.RemoveAppOfflineIfLeft(_environment, null, tracer);
// Reload status file with latest updates
statusFile = _status.Open(id);
if (statusFile != null)
{
await _hooksManager.PublishEventAsync(HookEventTypes.PostDeployment, statusFile);
}
if (exception != null)
{
throw new DeploymentFailedException(exception);
}
if (statusFile != null && statusFile.Status == DeployStatus.Success && _settings.RunFromLocalZip())
{
var zipDeploymentInfo = deploymentInfo as ArtifactDeploymentInfo;
if (zipDeploymentInfo != null)
{
await PostDeploymentHelper.UpdateSiteVersion(zipDeploymentInfo, _environment, tracer);
}
}
}
}
public async Task <HttpResponseMessage> Deploy(string id = null)
{
JObject result = GetJsonContent();
// Just block here to read the json payload from the body
using (_tracer.Step("DeploymentService.Deploy(id)"))
{
HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.OK);
await _deploymentLock.LockHttpOperationAsync(async() =>
{
try
{
if (PostDeploymentHelper.IsAutoSwapOngoing())
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.Conflict, Resources.Error_AutoSwapDeploymentOngoing));
}
DeployResult deployResult;
if (TryParseDeployResult(id, result, out deployResult))
{
using (_tracer.Step("DeploymentService.Create(id)"))
{
CreateDeployment(deployResult, result.Value <string>("details"));
// e.g if final url is "https://kudutry.scm.azurewebsites.net/api/deployments/ef52ec67fc9574e726955a9cbaf7bcba791e4e95/log"
// deploymentUri should be "https://kudutry.scm.azurewebsites.net/api/deployments/ef52ec67fc9574e726955a9cbaf7bcba791e4e95"
Uri deploymentUri = UriHelper.MakeRelative(UriHelper.GetBaseUri(Request), Request.RequestUri.AbsolutePath);
deployResult.Url = deploymentUri;
deployResult.LogUrl = UriHelper.MakeRelative(deploymentUri, "log");
response = Request.CreateResponse(HttpStatusCode.OK, ArmUtils.AddEnvelopeOnArmRequest(deployResult, Request));
return;
}
}
bool clean = false;
bool needFileUpdate = true;
if (result != null)
{
clean = result.Value <bool>("clean");
JToken needFileUpdateToken;
if (result.TryGetValue("needFileUpdate", out needFileUpdateToken))
{
needFileUpdate = needFileUpdateToken.Value <bool>();
}
}
string username = null;
AuthUtility.TryExtractBasicAuthUser(Request, out username);
IRepository repository = _repositoryFactory.GetRepository();
if (repository == null)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, Resources.Error_RepositoryNotFound));
}
ChangeSet changeSet = null;
if (!String.IsNullOrEmpty(id))
{
changeSet = repository.GetChangeSet(id);
if (changeSet == null)
{
string message = String.Format(CultureInfo.CurrentCulture, Resources.Error_DeploymentNotFound, id);
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, message));
}
}
try
{
await _deploymentManager.DeployAsync(repository, changeSet, username, clean, needFileUpdate);
}
catch (DeploymentFailedException ex)
{
if (!ArmUtils.IsArmRequest(Request))
{
throw;
}
// if requests comes thru ARM, we adjust the error code from 500 -> 400
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.BadRequest, ex.ToString()));
}
// auto-swap
if (PostDeploymentHelper.IsAutoSwapEnabled())
{
if (changeSet == null)
{
var targetBranch = _settings.GetBranch();
changeSet = repository.GetChangeSet(targetBranch);
}
IDeploymentStatusFile statusFile = _status.Open(changeSet.Id);
if (statusFile != null && statusFile.Status == DeployStatus.Success)
{
await PostDeploymentHelper.PerformAutoSwap(_environment.RequestId, _environment.SiteRestrictedJwt, new PostDeploymentTraceListener(_tracer, _deploymentManager.GetLogger(changeSet.Id)));
}
}
}
catch (FileNotFoundException ex)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, ex));
}
}, "Performing deployment");
return(response);
}
}
public async Task <HttpResponseMessage> Deploy(string id = null)
{
JObject result = GetJsonContent();
// Just block here to read the json payload from the body
using (_tracer.Step("DeploymentService.Deploy(id)"))
{
HttpResponseMessage response = Request.CreateResponse(HttpStatusCode.OK);
await _deploymentLock.LockHttpOperationAsync(async() =>
{
try
{
if (_autoSwapHandler.IsAutoSwapOngoing())
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.Conflict, Resources.Error_AutoSwapDeploymentOngoing));
}
DeployResult deployResult;
if (TryParseDeployResult(id, result, out deployResult))
{
using (_tracer.Step("DeploymentService.Create(id)"))
{
CreateDeployment(deployResult, result.Value <string>("details"));
// e.g if final url is "https://kudutry.scm.azurewebsites.net/api/deployments/ef52ec67fc9574e726955a9cbaf7bcba791e4e95/log"
// deploymentUri should be "https://kudutry.scm.azurewebsites.net/api/deployments/ef52ec67fc9574e726955a9cbaf7bcba791e4e95"
Uri deploymentUri = UriHelper.MakeRelative(UriHelper.GetBaseUri(Request), Request.RequestUri.AbsolutePath);
deployResult.Url = deploymentUri;
deployResult.LogUrl = UriHelper.MakeRelative(deploymentUri, "log");
response = Request.CreateResponse(HttpStatusCode.OK, ArmUtils.AddEnvelopeOnArmRequest(deployResult, Request));
return;
}
}
bool clean = false;
bool needFileUpdate = true;
if (result != null)
{
clean = result.Value <bool>("clean");
JToken needFileUpdateToken;
if (result.TryGetValue("needFileUpdate", out needFileUpdateToken))
{
needFileUpdate = needFileUpdateToken.Value <bool>();
}
}
string username = null;
AuthUtility.TryExtractBasicAuthUser(Request, out username);
IRepository repository = _repositoryFactory.GetRepository();
if (repository == null)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, Resources.Error_RepositoryNotFound));
}
ChangeSet changeSet = null;
if (!String.IsNullOrEmpty(id))
{
changeSet = repository.GetChangeSet(id);
if (changeSet == null)
{
string message = String.Format(CultureInfo.CurrentCulture, Resources.Error_DeploymentNotFound, id);
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, message));
}
}
await _deploymentManager.DeployAsync(repository, changeSet, username, clean, needFileUpdate);
}
catch (FileNotFoundException ex)
{
throw new HttpResponseException(Request.CreateErrorResponse(HttpStatusCode.NotFound, ex));
}
}, "Performing deployment");
return(response);
}
}
internal async Task <ChangeSet> Sync(DropboxHandler.DropboxInfo deploymentInfo, string branch, IRepository repository)
{
DropboxDeployInfo info = deploymentInfo.DeployInfo;
_totals = 0;
_successCount = 0;
_fileCount = 0;
_failedCount = 0;
_retriedCount = 0;
if (_settings.GetValue(CursorKey) != info.OldCursor)
{
throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor);
}
if (!repository.IsEmpty())
{
// git checkout --force <branch>
repository.ClearLock();
repository.Update(branch);
}
ChangeSet changeSet = null;
string message = null;
try
{
using (_tracer.Step("Sync with Dropbox"))
{
// Sync dropbox => repository directory
await ApplyChanges(deploymentInfo);
}
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_Synchronized,
deploymentInfo.DeployInfo.Deltas.Count);
}
catch (Exception)
{
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_SynchronizedWithFailure,
_successCount,
deploymentInfo.DeployInfo.Deltas.Count,
_failedCount);
throw;
}
finally
{
Logger.Log(message);
Logger.Log(String.Format("{0} downloaded files, {1} successful retries.", _fileCount, _retriedCount));
IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id);
statusFile.UpdateMessage(message);
statusFile.UpdateProgress(String.Format(CultureInfo.CurrentCulture, Resources.Dropbox_Committing, _successCount));
// Commit anyway even partial change
if (repository.Commit(message, String.Format("{0} <{1}>", info.UserName, info.Email)))
{
changeSet = repository.GetChangeSet("HEAD");
}
}
// Save new dropboc cursor
LogInfo("Update dropbox cursor");
_settings.SetValue(CursorKey, info.NewCursor);
return(changeSet);
}
public async Task Deploy(IRepository repository, ChangeSet changeSet, string deployer, bool clean, bool needFileUpdate)
{
ITracer tracer = _traceFactory.GetTracer();
IDisposable deployStep = null;
ILogger innerLogger = null;
string targetBranch = null;
// If we don't get a changeset, find out what branch we should be deploying and get the commit ID from it
if (changeSet == null)
{
targetBranch = _settings.GetBranch();
changeSet = repository.GetChangeSet(targetBranch);
}
string id = changeSet.Id;
IDeploymentStatusFile statusFile = null;
try
{
deployStep = tracer.Step("DeploymentManager.Deploy(id)");
// Remove the old log file for this deployment id
string logPath = GetLogPath(id);
FileSystemHelpers.DeleteFileSafe(logPath);
statusFile = GetOrCreateStatusFile(changeSet, tracer, deployer);
statusFile.MarkPending();
ILogger logger = GetLogger(changeSet.Id);
repository.ClearLock();
if (needFileUpdate)
{
using (tracer.Step("Updating to specific changeset"))
{
innerLogger = logger.Log(Resources.Log_UpdatingBranch, targetBranch ?? id);
using (var writer = new ProgressWriter())
{
// Update to the the specific changeset
repository.Update(id);
}
}
}
using (tracer.Step("Updating submodules"))
{
innerLogger = logger.Log(Resources.Log_UpdatingSubmodules);
repository.UpdateSubmodules();
}
if (clean)
{
tracer.Trace("Cleaning {0} repository", repository.RepositoryType);
innerLogger = logger.Log(Resources.Log_CleaningRepository, repository.RepositoryType);
repository.Clean();
}
// set to null as Build() below takes over logging
innerLogger = null;
// Perform the build deployment of this changeset
await Build(id, tracer, deployStep, repository);
}
catch (Exception ex)
{
if (innerLogger != null)
{
innerLogger.Log(ex);
}
if (statusFile != null)
{
statusFile.MarkFailed();
}
tracer.TraceError(ex);
if (deployStep != null)
{
deployStep.Dispose();
}
throw;
}
}
public void Deploy(IRepository repository, ChangeSet changeSet, string deployer, bool clean)
{
ITracer tracer = _traceFactory.GetTracer();
IDisposable deployStep = null;
ILogger innerLogger = null;
string targetBranch = null;
// If we don't get a changeset, find out what branch we should be deploying and get the commit ID from it
if (changeSet == null)
{
targetBranch = _settings.GetBranch();
changeSet = repository.GetChangeSet(targetBranch);
}
string id = changeSet.Id;
IDeploymentStatusFile statusFile = null;
try
{
deployStep = tracer.Step("DeploymentManager.Deploy(id)");
// Remove the old log file for this deployment id
string logPath = GetLogPath(id);
FileSystemHelpers.DeleteFileSafe(logPath);
statusFile = GetOrCreateStatusFile(changeSet, tracer, deployer);
statusFile.MarkPending();
ILogger logger = GetLogger(changeSet.Id);
using (tracer.Step("Updating to specific changeset"))
{
innerLogger = logger.Log(Resources.Log_UpdatingBranch, targetBranch ?? id);
using (var writer = new ProgressWriter())
{
// Update to the the specific changeset
repository.ClearLock();
repository.Update(id);
}
}
using (tracer.Step("Updating submodules"))
{
innerLogger = logger.Log(Resources.Log_UpdatingSubmodules);
repository.UpdateSubmodules();
}
if (clean)
{
tracer.Trace("Cleaning {0} repository", repository.RepositoryType);
innerLogger = logger.Log(Resources.Log_CleaningRepository, repository.RepositoryType);
repository.Clean();
}
// set to null as Build() below takes over logging
innerLogger = null;
// Perform the build deployment of this changeset
Build(id, tracer, deployStep);
}
catch (Exception ex)
{
if (innerLogger != null)
{
innerLogger.Log(ex);
}
if (statusFile != null)
{
statusFile.MarkFailed();
}
tracer.TraceError(ex);
if (deployStep != null)
{
deployStep.Dispose();
}
throw;
}
}
internal async Task<ChangeSet> Sync(DropboxInfo dropboxInfo, string branch, IRepository repository)
{
DropboxDeployInfo deployInfo = dropboxInfo.DeployInfo;
ResetStats();
if (_settings.GetValue(CursorKey) != deployInfo.OldCursor)
{
throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor);
}
// initial sync, remove default content
// for simplicity, we do it blindly whether or not in-place
// given the end result is the same
if (String.IsNullOrEmpty(deployInfo.OldCursor) && DeploymentHelper.IsDefaultWebRootContent(_environment.WebRootPath))
{
string hoststarthtml = Path.Combine(_environment.WebRootPath, Constants.HostingStartHtml);
FileSystemHelpers.DeleteFileSafe(hoststarthtml);
}
if (!repository.IsEmpty())
{
// git checkout --force <branch>
repository.Update(branch);
}
ChangeSet changeSet = null;
string message = null;
try
{
using (_tracer.Step("Sync with Dropbox"))
{
if (dropboxInfo.OAuthVersion == 2)
{
// Fetch the deltas
await UpdateDropboxDeployInfo(deployInfo);
}
// Sync dropbox => repository directory
await ApplyChanges(dropboxInfo, useOAuth20: dropboxInfo.OAuthVersion == 2);
}
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_Synchronized,
deployInfo.Deltas.Count);
}
catch (Exception)
{
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_SynchronizedWithFailure,
_successCount,
deployInfo.Deltas.Count,
_failedCount);
throw;
}
finally
{
Logger.Log(message);
Logger.Log(String.Format("{0} downloaded files, {1} successful retries.", _fileCount, _retriedCount));
IDeploymentStatusFile statusFile = _status.Open(dropboxInfo.TargetChangeset.Id);
statusFile.UpdateMessage(message);
statusFile.UpdateProgress(String.Format(CultureInfo.CurrentCulture, Resources.Dropbox_Committing, _successCount));
// Commit anyway even partial change
if (repository.Commit(message, String.Format("{0} <{1}>", deployInfo.UserName, deployInfo.Email ?? deployInfo.UserName)))
{
changeSet = repository.GetChangeSet("HEAD");
}
}
// Save new dropboc cursor
LogInfo("Update dropbox cursor");
_settings.SetValue(CursorKey, deployInfo.NewCursor);
return changeSet;
}
internal async Task<ChangeSet> Sync(DropboxHandler.DropboxInfo deploymentInfo, string branch, IRepository repository)
{
DropboxDeployInfo info = deploymentInfo.DeployInfo;
_totals = 0;
_successCount = 0;
_fileCount = 0;
_failedCount = 0;
_retriedCount = 0;
if (_settings.GetValue(CursorKey) != info.OldCursor)
{
throw new InvalidOperationException(Resources.Error_MismatchDropboxCursor);
}
if (!repository.IsEmpty())
{
// git checkout --force <branch>
repository.ClearLock();
repository.Update(branch);
}
ChangeSet changeSet = null;
string message = null;
try
{
using (_tracer.Step("Sync with Dropbox"))
{
// Sync dropbox => repository directory
await ApplyChanges(deploymentInfo);
}
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_Synchronized,
deploymentInfo.DeployInfo.Deltas.Count);
}
catch (Exception)
{
message = String.Format(CultureInfo.CurrentCulture,
Resources.Dropbox_SynchronizedWithFailure,
_successCount,
deploymentInfo.DeployInfo.Deltas.Count,
_failedCount);
throw;
}
finally
{
Logger.Log(message);
Logger.Log(String.Format("{0} downloaded files, {1} successful retries.", _fileCount, _retriedCount));
IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id);
statusFile.UpdateMessage(message);
statusFile.UpdateProgress(String.Format(CultureInfo.CurrentCulture, Resources.Dropbox_Committing, _successCount));
// Commit anyway even partial change
if (repository.Commit(message, String.Format("{0} <{1}>", info.UserName, info.Email)))
{
changeSet = repository.GetChangeSet("HEAD");
}
}
// Save new dropboc cursor
LogInfo("Update dropbox cursor");
_settings.SetValue(CursorKey, info.NewCursor);
return changeSet;
}
public async Task PerformDeployment(DeploymentInfoBase deploymentInfo, IDisposable tempDeployment = null, ChangeSet tempChangeSet = null)
{
DateTime currentMarkerFileUTC;
DateTime nextMarkerFileUTC = FileSystemHelpers.GetLastWriteTimeUtc(_markerFilePath);
ChangeSet lastChange = null;
do
{
// save the current marker
currentMarkerFileUTC = nextMarkerFileUTC;
string targetBranch = _settings.GetBranch();
using (_tracer.Step("Performing fetch based deployment"))
{
// create temporary deployment before the actual deployment item started
// this allows portal ui to readily display on-going deployment (not having to wait for fetch to complete).
// in addition, it captures any failure that may occur before the actual deployment item started
tempDeployment = tempDeployment ?? _deploymentManager.CreateTemporaryDeployment(
Resources.ReceivingChanges,
out tempChangeSet,
deploymentInfo.TargetChangeset,
deploymentInfo.Deployer);
ILogger innerLogger = null;
DeployStatusApiResult updateStatusObj = null;
try
{
ILogger logger = _deploymentManager.GetLogger(tempChangeSet.Id);
// Fetch changes from the repository
innerLogger = logger.Log(Resources.FetchingChanges);
IRepository repository = deploymentInfo.GetRepository();
try
{
await deploymentInfo.Fetch(repository, deploymentInfo, targetBranch, innerLogger, _tracer);
}
catch (BranchNotFoundException)
{
// mark no deployment is needed
deploymentInfo.TargetChangeset = null;
}
// set to null as Deploy() below takes over logging
innerLogger = null;
// The branch or commit id to deploy
string deployBranch = !String.IsNullOrEmpty(deploymentInfo.CommitId) ? deploymentInfo.CommitId : targetBranch;
try
{
_tracer.Trace($"Before sending {Constants.BuildRequestReceived} status to /api/updatedeploystatus");
if (PostDeploymentHelper.IsAzureEnvironment())
{
if (deploymentInfo != null &&
!string.IsNullOrEmpty(deploymentInfo.DeploymentTrackingId))
{
// Only send an updatedeploystatus request if DeploymentTrackingId is non null
// This signifies the client has opted in for these deployment updates for this deploy request
updateStatusObj = new DeployStatusApiResult(Constants.BuildRequestReceived, deploymentInfo.DeploymentTrackingId);
await _deploymentManager.SendDeployStatusUpdate(updateStatusObj);
}
}
}
catch (Exception e)
{
_tracer.TraceError($"Exception while sending {Constants.BuildRequestReceived} status to /api/updatedeploystatus. " +
$"Entry in the operations table for the deployment status may not have been created. {e}");
}
// In case the commit or perhaps fetch do no-op.
if (deploymentInfo.TargetChangeset != null && ShouldDeploy(repository, deploymentInfo, deployBranch))
{
// Perform the actual deployment
var changeSet = repository.GetChangeSet(deployBranch);
if (changeSet == null && !String.IsNullOrEmpty(deploymentInfo.CommitId))
{
throw new InvalidOperationException(String.Format("Invalid revision '{0}'!", deploymentInfo.CommitId));
}
lastChange = changeSet;
// Here, we don't need to update the working files, since we know Fetch left them in the correct state
// unless for GenericHandler where specific commitId is specified
bool deploySpecificCommitId = !String.IsNullOrEmpty(deploymentInfo.CommitId);
if (updateStatusObj != null)
{
updateStatusObj.DeploymentStatus = Constants.BuildInProgress;
await _deploymentManager.SendDeployStatusUpdate(updateStatusObj);
}
await _deploymentManager.DeployAsync(
repository,
changeSet,
deploymentInfo.Deployer,
clean : false,
deploymentInfo : deploymentInfo,
needFileUpdate : deploySpecificCommitId,
fullBuildByDefault : deploymentInfo.DoFullBuildByDefault);
if (updateStatusObj != null && !deploymentInfo.RestartAllowed)
{
// If restart is disallowed, send BuildSuccessful here as PostBuildRestartRequired was not sent
// during the DeployAsync flow.
updateStatusObj.DeploymentStatus = Constants.BuildSuccessful;
await _deploymentManager.SendDeployStatusUpdate(updateStatusObj);
}
}
}
catch (Exception ex)
{
if (innerLogger != null)
{
innerLogger.Log(ex);
}
// In case the commit or perhaps fetch do no-op.
if (deploymentInfo.TargetChangeset != null)
{
IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id);
if (statusFile != null)
{
statusFile.MarkFailed();
}
}
try
{
if (updateStatusObj != null)
{
// Set deployment status as failure if exception is thrown
updateStatusObj.DeploymentStatus = Constants.BuildFailed;
await _deploymentManager.SendDeployStatusUpdate(updateStatusObj);
}
}
catch
{
// no-op
}
throw;
}
// only clean up temp deployment if successful
tempDeployment.Dispose();
}
// check marker file and, if changed (meaning new /deploy request), redeploy.
nextMarkerFileUTC = FileSystemHelpers.GetLastWriteTimeUtc(_markerFilePath);
} while (deploymentInfo.IsReusable && currentMarkerFileUTC != nextMarkerFileUTC);
if (lastChange != null && PostDeploymentHelper.IsAutoSwapEnabled())
{
IDeploymentStatusFile statusFile = _status.Open(lastChange.Id);
if (statusFile.Status == DeployStatus.Success)
{
// if last change is not null and finish successfully, mean there was at least one deployment happened
// since deployment is now done, trigger swap if enabled
await PostDeploymentHelper.PerformAutoSwap(_environment.RequestId, new PostDeploymentTraceListener(_tracer, _deploymentManager.GetLogger(lastChange.Id)));
}
}
}
public async Task PerformDeployment(DeploymentInfo deploymentInfo)
{
DateTime currentMarkerFileUTC;
DateTime nextMarkerFileUTC = FileSystemHelpers.GetLastWriteTimeUtc(_markerFilePath);
do
{
// save the current marker
currentMarkerFileUTC = nextMarkerFileUTC;
string targetBranch = _settings.GetBranch();
using (_tracer.Step("Performing fetch based deployment"))
{
// create temporary deployment before the actual deployment item started
// this allows portal ui to readily display on-going deployment (not having to wait for fetch to complete).
// in addition, it captures any failure that may occur before the actual deployment item started
ChangeSet tempChangeSet;
IDisposable tempDeployment = _deploymentManager.CreateTemporaryDeployment(
Resources.ReceivingChanges,
out tempChangeSet,
deploymentInfo.TargetChangeset,
deploymentInfo.Deployer);
ILogger innerLogger = null;
try
{
ILogger logger = _deploymentManager.GetLogger(tempChangeSet.Id);
// Fetch changes from the repository
innerLogger = logger.Log(Resources.FetchingChanges);
IRepository repository = _repositoryFactory.EnsureRepository(deploymentInfo.RepositoryType);
try
{
await deploymentInfo.Handler.Fetch(repository, deploymentInfo, targetBranch, innerLogger);
}
catch (BranchNotFoundException)
{
// mark no deployment is needed
deploymentInfo.TargetChangeset = null;
}
// set to null as Deploy() below takes over logging
innerLogger = null;
// In case the commit or perhaps fetch do no-op.
if (deploymentInfo.TargetChangeset != null && ShouldDeploy(repository, deploymentInfo, targetBranch))
{
// Perform the actual deployment
var changeSet = repository.GetChangeSet(targetBranch);
// Here, we don't need to update the working files, since we know Fetch left them in the correct state
await _deploymentManager.DeployAsync(repository, changeSet, deploymentInfo.Deployer, clean : false, needFileUpdate : false);
}
}
catch (Exception ex)
{
if (innerLogger != null)
{
innerLogger.Log(ex);
}
// In case the commit or perhaps fetch do no-op.
if (deploymentInfo.TargetChangeset != null)
{
IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id);
if (statusFile != null)
{
statusFile.MarkFailed();
}
}
throw;
}
// only clean up temp deployment if successful
tempDeployment.Dispose();
}
// check marker file and, if changed (meaning new /deploy request), redeploy.
nextMarkerFileUTC = FileSystemHelpers.GetLastWriteTimeUtc(_markerFilePath);
} while (deploymentInfo.IsReusable && currentMarkerFileUTC != nextMarkerFileUTC);
}
internal async Task <ChangeSet> Sync(OneDriveInfo info, IRepository repository, ITracer tracer)
{
ChangeSet changeSet = null;
string cursor = _settings.GetValue(CursorKey);
ChangesResult changes = null;
// use incoming tracer since it is background work
_tracer = tracer;
// We truncate cursor value in filename but keep it unharmed in the trace content
using (_tracer.Step("Getting delta changes with cursor: {0}...", cursor.Truncate(5)))
using (_tracer.Step("cursor: {0}", cursor))
{
changes = await GetChanges(info.TargetChangeset.Id, info.AccessToken, info.RepositoryUrl, cursor);
}
if (changes == null || changes.Count == 0)
{
_tracer.Trace("No changes need to be applied.");
LogMessage(Resources.OneDriveNoChangesFound);
return(changeSet);
}
// for simplicity, use file changes as effective total
_totals = changes.FileChanges.Count > 0 ? changes.FileChanges.Count : changes.Count;
string hoststarthtml = Path.Combine(_environment.WebRootPath, Constants.HostingStartHtml);
FileSystemHelpers.DeleteFileSafe(hoststarthtml);
using (new Timer(UpdateStatusFile, state: info.TargetChangeset.Id, dueTime: TimeSpan.FromSeconds(5), period: TimeSpan.FromSeconds(5)))
using (_tracer.Step("Applying {0} changes ...", _totals))
{
LogMessage(Resources.OneDriveApplyingChanges, _totals);
// perform action seperately, so that can ensure timestamp on directory
// e.g two changes:
// (new) file /a/b/c.txt
// (new) dir /a/b
// if created dir first then create file. file creation will trigger folder timestamp change.
// which will result in "/a/b" has timestamp from the monent of file creation instead of the timestamp value from server, where value supposed to be set by code specifically.
await ApplyChangesParallel(changes.DeletionChanges, info.AccessToken, maxParallelCount : 1, countSuccess : changes.FileChanges.Count == 0);
await ApplyChangesParallel(changes.FileChanges, info.AccessToken, maxParallelCount : MaxConcurrentRequests, countSuccess : true);
// apply folder changes at last to maintain same timestamp as in OneDrive
await ApplyChangesParallel(changes.DirectoryChanges, info.AccessToken, maxParallelCount : 1, countSuccess : changes.FileChanges.Count == 0);
_tracer.Trace("{0} succeeded, {1} failed", _successCount, _failedCount);
LogMessage(Resources.OneDriveApplyResult, _successCount, _failedCount);
string message = _failedCount > 0 ?
string.Format(CultureInfo.CurrentCulture, Resources.OneDrive_SynchronizedWithFailure, _successCount, _totals, _failedCount) :
string.Format(CultureInfo.CurrentCulture, Resources.OneDrive_Synchronized, _totals);
// Commit anyway even partial change
if (repository.Commit(message, info.AuthorName, info.AuthorEmail))
{
changeSet = repository.GetChangeSet("HEAD");
}
if (_failedCount > 0)
{
// signal deployment failied
throw new Exception(string.Format(CultureInfo.CurrentCulture, Resources.OneDriveApplyResult, _successCount, _failedCount));
}
}
// finally keep a copy of the new cursor
_settings.SetValue(CursorKey, changes.Cursor);
return(changeSet);
}
public async Task DeployAsync(IRepository repository, ChangeSet changeSet, string deployer, bool clean, bool needFileUpdate)
{
using (var deploymentAnalytics = new DeploymentAnalytics(_analytics, _settings))
{
Exception exception = null;
ITracer tracer = _traceFactory.GetTracer();
IDisposable deployStep = null;
ILogger innerLogger = null;
string targetBranch = null;
// If we don't get a changeset, find out what branch we should be deploying and get the commit ID from it
if (changeSet == null)
{
targetBranch = _settings.GetBranch();
changeSet = repository.GetChangeSet(targetBranch);
}
string id = changeSet.Id;
IDeploymentStatusFile statusFile = null;
try
{
deployStep = tracer.Step("DeploymentManager.Deploy(id)");
// Remove the old log file for this deployment id
string logPath = GetLogPath(id);
FileSystemHelpers.DeleteFileSafe(logPath);
statusFile = GetOrCreateStatusFile(changeSet, tracer, deployer);
statusFile.MarkPending();
ILogger logger = GetLogger(changeSet.Id);
if (needFileUpdate)
{
using (tracer.Step("Updating to specific changeset"))
{
innerLogger = logger.Log(Resources.Log_UpdatingBranch, targetBranch ?? id);
using (var writer = new ProgressWriter())
{
// Update to the specific changeset or branch
repository.Update(targetBranch ?? id);
}
}
}
if (_settings.ShouldUpdateSubmodules())
{
using (tracer.Step("Updating submodules"))
{
innerLogger = logger.Log(Resources.Log_UpdatingSubmodules);
repository.UpdateSubmodules();
}
}
if (clean)
{
tracer.Trace("Cleaning {0} repository", repository.RepositoryType);
innerLogger = logger.Log(Resources.Log_CleaningRepository, repository.RepositoryType);
repository.Clean();
}
// set to null as Build() below takes over logging
innerLogger = null;
// Perform the build deployment of this changeset
await Build(id, tracer, deployStep, repository, deploymentAnalytics);
}
catch (Exception ex)
{
exception = ex;
if (innerLogger != null)
{
innerLogger.Log(ex);
}
if (statusFile != null)
{
MarkStatusComplete(statusFile, success: false);
}
tracer.TraceError(ex);
deploymentAnalytics.Error = ex.ToString();
if (deployStep != null)
{
deployStep.Dispose();
}
}
// Reload status file with latest updates
statusFile = _status.Open(id);
if (statusFile != null)
{
await _hooksManager.PublishEventAsync(HookEventTypes.PostDeployment, statusFile);
}
if (exception != null)
{
throw new DeploymentFailedException(exception);
}
}
}