public static async Task <int> CopyToAsync(this HttpContent content, string filePath, ITracer tracer)
{
const int ReadBufferSize = 4096;
var nextTraceTime = DateTime.UtcNow.AddSeconds(30);
var total = 0;
var buffer = new byte[ReadBufferSize];
using (var src = await content.ReadAsStreamAsync())
using (var dst = FileSystemHelpers.CreateFile(filePath))
{
while (true)
{
int read = await src.ReadAsync(buffer, 0, buffer.Length);
if (read <= 0)
{
tracer.Trace("Total {0:n0} bytes written to {1}", total, filePath);
break;
}
else if (DateTime.UtcNow > nextTraceTime)
{
tracer.Trace("Writing {0:n0} bytes to {1}", total, filePath);
nextTraceTime = DateTime.UtcNow.AddSeconds(30);
}
await dst.WriteAsync(buffer, 0, read);
total += read;
}
}
return(total);
}
public void Save()
{
if (String.IsNullOrEmpty(Id))
{
throw new InvalidOperationException();
}
var document = new XDocument(new XElement("deployment",
new XElement("id", Id),
new XElement("author", XmlUtility.Sanitize(Author)),
new XElement("deployer", Deployer),
new XElement("authorEmail", AuthorEmail),
new XElement("message", XmlUtility.Sanitize(Message)),
new XElement("progress", Progress),
new XElement("status", Status),
new XElement("statusText", StatusText),
new XElement("lastSuccessEndTime", LastSuccessEndTime),
new XElement("receivedTime", ReceivedTime),
new XElement("startTime", StartTime),
new XElement("endTime", EndTime),
new XElement("complete", Complete.ToString()),
new XElement("is_temp", IsTemporary.ToString()),
new XElement("is_readonly", IsReadOnly.ToString())
));
_statusLock.LockOperation(() =>
{
using (Stream stream = FileSystemHelpers.CreateFile(_statusFile))
{
document.Save(stream);
}
OperationManager.Attempt(() =>
{
// Used for ETAG
if (FileSystemHelpers.FileExists(_activeFile))
{
FileSystemHelpers.SetLastWriteTimeUtc(_activeFile, DateTime.UtcNow);
}
else
{
FileSystemHelpers.WriteAllText(_activeFile, String.Empty);
}
});
OperationManager.Attempt(() =>
{
// enable the feature thru configuration
if (ScmHostingConfigurations.DeploymentStatusCompleteFileEnabled && Complete)
{
FileSystemHelpers.CopyFile(_statusFile, _statusCompleteFile);
}
else if (FileSystemHelpers.FileExists(_statusCompleteFile))
{
FileSystemHelpers.DeleteFile(_statusCompleteFile);
}
});
}, "Updating deployment status", DeploymentStatusManager.LockTimeout);
}
private static void CreateRequirements()
{
if (!FileSystemHelpers.FileExists(Constants.RequirementsTxt))
{
FileSystemHelpers.CreateFile(Constants.RequirementsTxt);
}
else
{
ColoredConsole.WriteLine($"{Constants.RequirementsTxt} already exists. Skipped!");
}
}
private async Task WriteSitePackageZip(ZipDeploymentInfo zipDeploymentInfo, ITracer tracer)
{
var filePath = Path.Combine(_environment.SitePackagesPath, zipDeploymentInfo.ZipName);
// Make sure D:\home\data\SitePackages exists
FileSystemHelpers.EnsureDirectory(_environment.SitePackagesPath);
using (tracer.Step("Writing content to {0}", filePath))
{
using (var file = FileSystemHelpers.CreateFile(filePath))
{
await Request.Content.CopyToAsync(file);
}
}
}
private async Task WriteSitePackageZip(ZipDeploymentInfo zipDeploymentInfo, ITracer tracer, HttpContent content)
{
var filePath = Path.Combine(_environment.SitePackagesPath, zipDeploymentInfo.ZipName);
// Make sure D:\home\data\SitePackages exists
FileSystemHelpers.EnsureDirectory(_environment.SitePackagesPath);
using (tracer.Step("Writing content to {0}", filePath))
{
using (var file = FileSystemHelpers.CreateFile(filePath))
{
await content.CopyToAsync(file);
}
}
DeploymentHelper.PurgeZipsIfNecessary(_environment.SitePackagesPath, tracer, _settings.GetMaxZipPackageCount());
}
private async Task <string> DeployZipLocally(ZipDeploymentInfo zipDeploymentInfo, ITracer tracer)
{
var content = await DeploymentHelper.GetZipContentFromURL(zipDeploymentInfo, tracer);
var zipFileName = Path.ChangeExtension(Path.GetRandomFileName(), "zip");
var zipFilePath = Path.Combine(_environment.ZipTempPath, zipFileName);
using (_tracer.Step("Writing content to {0}", zipFilePath))
{
using (var file = FileSystemHelpers.CreateFile(zipFilePath))
{
await content.CopyToAsync(file);
}
}
zipDeploymentInfo.RepositoryUrl = zipFilePath;
return(zipFilePath);
}
public void Save()
{
if (String.IsNullOrEmpty(Id))
{
throw new InvalidOperationException();
}
var document = new XDocument(new XElement("deployment",
new XElement("id", Id),
new XElement("author", Author),
new XElement("deployer", Deployer),
new XElement("authorEmail", AuthorEmail),
new XElement("message", Message),
new XElement("progress", Progress),
new XElement("status", Status),
new XElement("statusText", StatusText),
new XElement("lastSuccessEndTime", LastSuccessEndTime),
new XElement("receivedTime", ReceivedTime),
new XElement("startTime", StartTime),
new XElement("endTime", EndTime),
new XElement("complete", Complete.ToString()),
new XElement("is_temp", IsTemporary.ToString()),
new XElement("is_readonly", IsReadOnly.ToString())
));
_statusLock.LockOperation(() =>
{
using (Stream stream = FileSystemHelpers.CreateFile(_statusFile))
{
document.Save(stream);
}
// Used for ETAG
if (FileSystemHelpers.FileExists(_activeFile))
{
FileSystemHelpers.SetLastWriteTimeUtc(_activeFile, DateTime.UtcNow);
}
else
{
FileSystemHelpers.WriteAllText(_activeFile, String.Empty);
}
}, DeploymentStatusManager.LockTimeout);
}
private async Task <HttpResponseMessage> PushDeployAsync(
ZipDeploymentInfo deploymentInfo,
bool isAsync)
{
var zipFileName = Path.ChangeExtension(Path.GetRandomFileName(), "zip");
var zipFilePath = Path.Combine(_environment.ZipTempPath, zipFileName);
using (_tracer.Step("Writing content to {0}", zipFilePath))
{
using (var file = FileSystemHelpers.CreateFile(zipFilePath))
{
await Request.Content.CopyToAsync(file);
}
}
deploymentInfo.RepositoryUrl = zipFilePath;
var result = await _deploymentManager.FetchDeploy(deploymentInfo, isAsync, UriHelper.GetRequestUri(Request), "HEAD");
var response = Request.CreateResponse();
switch (result)
{
case FetchDeploymentRequestResult.RunningAynschronously:
if (isAsync)
{
// latest deployment keyword reserved to poll till deployment done
response.Headers.Location = new Uri(UriHelper.GetRequestUri(Request),
String.Format("/api/deployments/{0}?deployer={1}&time={2}", Constants.LatestDeployment, deploymentInfo.Deployer, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ")));
}
response.StatusCode = HttpStatusCode.Accepted;
break;
case FetchDeploymentRequestResult.ForbiddenScmDisabled:
// Should never hit this for zip push deploy
response.StatusCode = HttpStatusCode.Forbidden;
_tracer.Trace("Scm is not enabled, reject all requests.");
break;
case FetchDeploymentRequestResult.ConflictAutoSwapOngoing:
response.StatusCode = HttpStatusCode.Conflict;
response.Content = new StringContent(Resources.Error_AutoSwapDeploymentOngoing);
break;
case FetchDeploymentRequestResult.Pending:
// Shouldn't happen here, as we disallow deferral for this use case
response.StatusCode = HttpStatusCode.Accepted;
break;
case FetchDeploymentRequestResult.RanSynchronously:
response.StatusCode = HttpStatusCode.OK;
break;
case FetchDeploymentRequestResult.ConflictDeploymentInProgress:
response.StatusCode = HttpStatusCode.Conflict;
response.Content = new StringContent(Resources.Error_DeploymentInProgress);
break;
default:
response.StatusCode = HttpStatusCode.BadRequest;
break;
}
return(response);
}
public async Task <ScanRequestResult> StartScan(String timeout, String mainScanDirPath, String id, String host)
{
using (_tracer.Step("Start scan in the background"))
{
String folderPath = Path.Combine(mainScanDirPath, Constants.ScanFolderName + id);
String filePath = Path.Combine(folderPath, Constants.ScanStatusFile);
Boolean hasFileModifcations = true;
if (_deploymentLock.IsHeld)
{
return(ScanRequestResult.ScanAlreadyInProgress);
}
//Create unique scan folder and scan status file
_deploymentLock.LockOperation(() =>
{
//Check if files are modified
if (CheckModifications(mainScanDirPath))
{
//Create unique scan directory for current scan
FileSystemHelpers.CreateDirectory(folderPath);
_tracer.Trace("Unique scan directory created for scan {0}", id);
//Create scan status file inside folder
FileSystemHelpers.CreateFile(filePath).Close();
//Create temp file to check if scan is still running
string tempScanFilePath = GetTempScanFilePath(mainScanDirPath);
tempScanFilePath = Path.Combine(mainScanDirPath, Constants.TempScanFile);
FileSystemHelpers.CreateFile(tempScanFilePath).Close();
UpdateScanStatus(folderPath, ScanStatus.Starting);
}
else
{
hasFileModifcations = false;
}
}, "Creating unique scan folder", TimeSpan.Zero);
if (!hasFileModifcations)
{
return(ScanRequestResult.NoFileModifications);
}
//Start Backgorund Scan
using (var timeoutCancellationTokenSource = new CancellationTokenSource())
{
var successfullyScanned = PerformBackgroundScan(_tracer, folderPath, timeoutCancellationTokenSource.Token, id, mainScanDirPath);
//Wait till scan task completes or the timeout goes off
if (await Task.WhenAny(successfullyScanned, Task.Delay(Int32.Parse(timeout), timeoutCancellationTokenSource.Token)) == successfullyScanned)
{
//If scan task completes before timeout
//Delete excess scan folders, just keep the maximum number allowed
await DeletePastScans(mainScanDirPath, _tracer);
//Create new Manifest file containing the modified timestamps
String manifestPath = Path.Combine(mainScanDirPath, Constants.ScanManifest);
if (FileSystemHelpers.FileExists(manifestPath))
{
FileSystemHelpers.DeleteFileSafe(manifestPath);
}
JObject manifestObj = new JObject();
//Write to the manifest with new timestamps of the modified file
ModifyManifestFile(manifestObj, Constants.ScanDir);
File.WriteAllText(manifestPath, JsonConvert.SerializeObject(manifestObj));
//Path to common log file for azure monitor
String aggrLogPath = Path.Combine(mainScanDirPath, Constants.AggregrateScanResults);
//This checks if result scan log is formed
//If yes, it will append necessary logs to the aggregrate log file
//Current appended logs will be "Scanned files","Infected files", and details of infected files
String currLogPath = Path.Combine(folderPath, Constants.ScanLogFile);
if (FileSystemHelpers.FileExists(currLogPath))
{
StreamReader file = new StreamReader(currLogPath);
string line;
while ((line = file.ReadLine()) != null)
{
if (line.Contains("FOUND") || line.Contains("Infected files") || line.Contains("Scanned files"))
{
//logType "Infected" means this log line represents details of infected files
String logType = "Infected";
if (line.Contains("Infected files") || line.Contains("Scanned files"))
{
//logType "Info" means this log line represents total number of scanned or infected files
logType = "Info";
}
FileSystemHelpers.AppendAllTextToFile(aggrLogPath, DateTime.UtcNow.ToString(@"M/d/yyyy hh:mm:ss tt") + "," + id + "," + logType + "," + host + "," + line + '\n');
}
}
}
return(successfullyScanned.Result
? ScanRequestResult.RunningAynschronously
: ScanRequestResult.AsyncScanFailed);
}
else
{
//Timeout went off before scan task completion
//Cancel scan task
timeoutCancellationTokenSource.Cancel();
//Scan process will be cancelled
//wait till scan status file is appropriately updated
await successfullyScanned;
//Delete excess scan folders, just keep the maximum number allowed
await DeletePastScans(mainScanDirPath, _tracer);
return(ScanRequestResult.AsyncScanFailed);
}
}
}
}
private static void CreateRequirements(string path = null)
{
var reqFile = string.IsNullOrEmpty(path) ? Constants.RequirementsTxt : Path.Combine(path, Constants.RequirementsTxt);
FileSystemHelpers.CreateFile(reqFile);
}
private async Task <HttpResponseMessage> PushDeployAsync(ZipDeploymentInfo deploymentInfo, bool isAsync)
{
var content = Request.Content;
var isRequestJSON = content.Headers?.ContentType?.MediaType?.Equals("application/json", StringComparison.OrdinalIgnoreCase);
if (isRequestJSON == true && ArmUtils.IsArmRequest(Request))
{
try
{
deploymentInfo.ZipURL = await GetZipURLFromJSON();
}
catch (Exception ex)
{
return(ArmUtils.CreateErrorResponse(Request, HttpStatusCode.BadRequest, ex));
}
}
else
{
if (_settings.RunFromLocalZip())
{
await WriteSitePackageZip(deploymentInfo, _tracer, Request.Content);
}
else
{
var zipFileName = Path.ChangeExtension(Path.GetRandomFileName(), "zip");
var zipFilePath = Path.Combine(_environment.ZipTempPath, zipFileName);
using (_tracer.Step("Writing content to {0}", zipFilePath))
{
using (var file = FileSystemHelpers.CreateFile(zipFilePath))
{
await content.CopyToAsync(file);
}
}
deploymentInfo.RepositoryUrl = zipFilePath;
}
}
isAsync = ArmUtils.IsArmRequest(Request) ? true : isAsync;
var result = await _deploymentManager.FetchDeploy(deploymentInfo, isAsync, UriHelper.GetRequestUri(Request), "HEAD");
var response = Request.CreateResponse();
switch (result)
{
case FetchDeploymentRequestResult.RunningAynschronously:
if (ArmUtils.IsArmRequest(Request))
{
DeployResult deployResult = new DeployResult();
response = Request.CreateResponse(HttpStatusCode.Accepted, ArmUtils.AddEnvelopeOnArmRequest(deployResult, Request));
string statusURL = GetStatusUrl();
// Should not happen: If we couldn't make the URL, there must have been an error in the request
if (string.IsNullOrEmpty(statusURL))
{
var badResponse = Request.CreateResponse();
badResponse.StatusCode = HttpStatusCode.BadRequest;
return(badResponse);
}
// latest deployment keyword reserved to poll till deployment done
response.Headers.Location = new Uri(statusURL +
String.Format("/deployments/{0}?api-version=2018-02-01&deployer={1}&time={2}", Constants.LatestDeployment, deploymentInfo.Deployer, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ")));
}
else if (isAsync)
{
// latest deployment keyword reserved to poll till deployment done
response.Headers.Location = new Uri(UriHelper.GetRequestUri(Request),
String.Format("/api/deployments/{0}?deployer={1}&time={2}", Constants.LatestDeployment, deploymentInfo.Deployer, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ")));
}
response.StatusCode = HttpStatusCode.Accepted;
break;
case FetchDeploymentRequestResult.ForbiddenScmDisabled:
// Should never hit this for zip push deploy
response.StatusCode = HttpStatusCode.Forbidden;
_tracer.Trace("Scm is not enabled, reject all requests.");
break;
case FetchDeploymentRequestResult.ConflictAutoSwapOngoing:
response.StatusCode = HttpStatusCode.Conflict;
response.Content = new StringContent(Resources.Error_AutoSwapDeploymentOngoing);
break;
case FetchDeploymentRequestResult.Pending:
// Shouldn't happen here, as we disallow deferral for this use case
response.StatusCode = HttpStatusCode.Accepted;
break;
case FetchDeploymentRequestResult.RanSynchronously:
response.StatusCode = HttpStatusCode.OK;
break;
case FetchDeploymentRequestResult.ConflictDeploymentInProgress:
response.StatusCode = HttpStatusCode.Conflict;
response.Content = new StringContent(Resources.Error_DeploymentInProgress);
break;
case FetchDeploymentRequestResult.ConflictRunFromRemoteZipConfigured:
response.StatusCode = HttpStatusCode.Conflict;
response.Content = new StringContent(Resources.Error_RunFromRemoteZipConfigured);
break;
default:
response.StatusCode = HttpStatusCode.BadRequest;
break;
}
return(response);
}
public async Task <HttpResponseMessage> ZipPushDeploy(
HttpRequestMessage request,
[FromUri] bool isAsync = false,
[FromUri] string author = null,
[FromUri] string authorEmail = null,
[FromUri] string deployer = DefaultDeployer,
[FromUri] string message = DefaultMessage)
{
using (_tracer.Step("ZipPushDeploy"))
{
var zipFileName = Path.ChangeExtension(Path.GetRandomFileName(), "zip");
var zipFilePath = Path.Combine(_environment.ZipTempPath, zipFileName);
using (_tracer.Step("Writing zip file to {0}", zipFilePath))
{
using (var file = FileSystemHelpers.CreateFile(zipFilePath))
{
await request.Content.CopyToAsync(file);
}
}
var deploymentInfo = new ZipDeploymentInfo(_environment, _traceFactory)
{
AllowDeploymentWhileScmDisabled = true,
Deployer = deployer,
IsContinuous = false,
AllowDeferredDeployment = false,
IsReusable = false,
RepositoryUrl = zipFilePath,
TargetChangeset = DeploymentManager.CreateTemporaryChangeSet(message: "Deploying from pushed zip file"),
CommitId = null,
RepositoryType = RepositoryType.None,
Fetch = LocalZipFetch,
DoFullBuildByDefault = false,
Author = author,
AuthorEmail = authorEmail,
Message = message
};
var result = await _deploymentManager.FetchDeploy(deploymentInfo, isAsync, UriHelper.GetRequestUri(Request), "HEAD");
var response = request.CreateResponse();
switch (result)
{
case FetchDeploymentRequestResult.RunningAynschronously:
if (isAsync)
{
// latest deployment keyword reserved to poll till deployment done
response.Headers.Location = new Uri(UriHelper.GetRequestUri(Request),
String.Format("/api/deployments/{0}?deployer={1}&time={2}", Constants.LatestDeployment, deploymentInfo.Deployer, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ")));
}
response.StatusCode = HttpStatusCode.Accepted;
break;
case FetchDeploymentRequestResult.ForbiddenScmDisabled:
// Should never hit this for zip push deploy
response.StatusCode = HttpStatusCode.Forbidden;
_tracer.Trace("Scm is not enabled, reject all requests.");
break;
case FetchDeploymentRequestResult.ConflictAutoSwapOngoing:
response.StatusCode = HttpStatusCode.Conflict;
response.Content = new StringContent(Resources.Error_AutoSwapDeploymentOngoing);
break;
case FetchDeploymentRequestResult.Pending:
// Shouldn't happen here, as we disallow deferral for this use case
response.StatusCode = HttpStatusCode.Accepted;
break;
case FetchDeploymentRequestResult.RanSynchronously:
response.StatusCode = HttpStatusCode.OK;
break;
case FetchDeploymentRequestResult.ConflictDeploymentInProgress:
response.StatusCode = HttpStatusCode.Conflict;
response.Content = new StringContent(Resources.Error_DeploymentInProgress);
break;
default:
response.StatusCode = HttpStatusCode.BadRequest;
break;
}
return(response);
}
}
