public static async Task <int> CopyToAsync(this HttpContent content, string filePath, ITracer tracer) { const int ReadBufferSize = 4096; var nextTraceTime = DateTime.UtcNow.AddSeconds(30); var total = 0; var buffer = new byte[ReadBufferSize]; using (var src = await content.ReadAsStreamAsync()) using (var dst = FileSystemHelpers.CreateFile(filePath)) { while (true) { int read = await src.ReadAsync(buffer, 0, buffer.Length); if (read <= 0) { tracer.Trace("Total {0:n0} bytes written to {1}", total, filePath); break; } else if (DateTime.UtcNow > nextTraceTime) { tracer.Trace("Writing {0:n0} bytes to {1}", total, filePath); nextTraceTime = DateTime.UtcNow.AddSeconds(30); } await dst.WriteAsync(buffer, 0, read); total += read; } } return(total); }
public void Save() { if (String.IsNullOrEmpty(Id)) { throw new InvalidOperationException(); } var document = new XDocument(new XElement("deployment", new XElement("id", Id), new XElement("author", XmlUtility.Sanitize(Author)), new XElement("deployer", Deployer), new XElement("authorEmail", AuthorEmail), new XElement("message", XmlUtility.Sanitize(Message)), new XElement("progress", Progress), new XElement("status", Status), new XElement("statusText", StatusText), new XElement("lastSuccessEndTime", LastSuccessEndTime), new XElement("receivedTime", ReceivedTime), new XElement("startTime", StartTime), new XElement("endTime", EndTime), new XElement("complete", Complete.ToString()), new XElement("is_temp", IsTemporary.ToString()), new XElement("is_readonly", IsReadOnly.ToString()) )); _statusLock.LockOperation(() => { using (Stream stream = FileSystemHelpers.CreateFile(_statusFile)) { document.Save(stream); } OperationManager.Attempt(() => { // Used for ETAG if (FileSystemHelpers.FileExists(_activeFile)) { FileSystemHelpers.SetLastWriteTimeUtc(_activeFile, DateTime.UtcNow); } else { FileSystemHelpers.WriteAllText(_activeFile, String.Empty); } }); OperationManager.Attempt(() => { // enable the feature thru configuration if (ScmHostingConfigurations.DeploymentStatusCompleteFileEnabled && Complete) { FileSystemHelpers.CopyFile(_statusFile, _statusCompleteFile); } else if (FileSystemHelpers.FileExists(_statusCompleteFile)) { FileSystemHelpers.DeleteFile(_statusCompleteFile); } }); }, "Updating deployment status", DeploymentStatusManager.LockTimeout); }
private static void CreateRequirements() { if (!FileSystemHelpers.FileExists(Constants.RequirementsTxt)) { FileSystemHelpers.CreateFile(Constants.RequirementsTxt); } else { ColoredConsole.WriteLine($"{Constants.RequirementsTxt} already exists. Skipped!"); } }
private async Task WriteSitePackageZip(ZipDeploymentInfo zipDeploymentInfo, ITracer tracer) { var filePath = Path.Combine(_environment.SitePackagesPath, zipDeploymentInfo.ZipName); // Make sure D:\home\data\SitePackages exists FileSystemHelpers.EnsureDirectory(_environment.SitePackagesPath); using (tracer.Step("Writing content to {0}", filePath)) { using (var file = FileSystemHelpers.CreateFile(filePath)) { await Request.Content.CopyToAsync(file); } } }
private async Task WriteSitePackageZip(ZipDeploymentInfo zipDeploymentInfo, ITracer tracer, HttpContent content) { var filePath = Path.Combine(_environment.SitePackagesPath, zipDeploymentInfo.ZipName); // Make sure D:\home\data\SitePackages exists FileSystemHelpers.EnsureDirectory(_environment.SitePackagesPath); using (tracer.Step("Writing content to {0}", filePath)) { using (var file = FileSystemHelpers.CreateFile(filePath)) { await content.CopyToAsync(file); } } DeploymentHelper.PurgeZipsIfNecessary(_environment.SitePackagesPath, tracer, _settings.GetMaxZipPackageCount()); }
private async Task <string> DeployZipLocally(ZipDeploymentInfo zipDeploymentInfo, ITracer tracer) { var content = await DeploymentHelper.GetZipContentFromURL(zipDeploymentInfo, tracer); var zipFileName = Path.ChangeExtension(Path.GetRandomFileName(), "zip"); var zipFilePath = Path.Combine(_environment.ZipTempPath, zipFileName); using (_tracer.Step("Writing content to {0}", zipFilePath)) { using (var file = FileSystemHelpers.CreateFile(zipFilePath)) { await content.CopyToAsync(file); } } zipDeploymentInfo.RepositoryUrl = zipFilePath; return(zipFilePath); }
public void Save() { if (String.IsNullOrEmpty(Id)) { throw new InvalidOperationException(); } var document = new XDocument(new XElement("deployment", new XElement("id", Id), new XElement("author", Author), new XElement("deployer", Deployer), new XElement("authorEmail", AuthorEmail), new XElement("message", Message), new XElement("progress", Progress), new XElement("status", Status), new XElement("statusText", StatusText), new XElement("lastSuccessEndTime", LastSuccessEndTime), new XElement("receivedTime", ReceivedTime), new XElement("startTime", StartTime), new XElement("endTime", EndTime), new XElement("complete", Complete.ToString()), new XElement("is_temp", IsTemporary.ToString()), new XElement("is_readonly", IsReadOnly.ToString()) )); _statusLock.LockOperation(() => { using (Stream stream = FileSystemHelpers.CreateFile(_statusFile)) { document.Save(stream); } // Used for ETAG if (FileSystemHelpers.FileExists(_activeFile)) { FileSystemHelpers.SetLastWriteTimeUtc(_activeFile, DateTime.UtcNow); } else { FileSystemHelpers.WriteAllText(_activeFile, String.Empty); } }, DeploymentStatusManager.LockTimeout); }
private async Task <HttpResponseMessage> PushDeployAsync( ZipDeploymentInfo deploymentInfo, bool isAsync) { var zipFileName = Path.ChangeExtension(Path.GetRandomFileName(), "zip"); var zipFilePath = Path.Combine(_environment.ZipTempPath, zipFileName); using (_tracer.Step("Writing content to {0}", zipFilePath)) { using (var file = FileSystemHelpers.CreateFile(zipFilePath)) { await Request.Content.CopyToAsync(file); } } deploymentInfo.RepositoryUrl = zipFilePath; var result = await _deploymentManager.FetchDeploy(deploymentInfo, isAsync, UriHelper.GetRequestUri(Request), "HEAD"); var response = Request.CreateResponse(); switch (result) { case FetchDeploymentRequestResult.RunningAynschronously: if (isAsync) { // latest deployment keyword reserved to poll till deployment done response.Headers.Location = new Uri(UriHelper.GetRequestUri(Request), String.Format("/api/deployments/{0}?deployer={1}&time={2}", Constants.LatestDeployment, deploymentInfo.Deployer, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ"))); } response.StatusCode = HttpStatusCode.Accepted; break; case FetchDeploymentRequestResult.ForbiddenScmDisabled: // Should never hit this for zip push deploy response.StatusCode = HttpStatusCode.Forbidden; _tracer.Trace("Scm is not enabled, reject all requests."); break; case FetchDeploymentRequestResult.ConflictAutoSwapOngoing: response.StatusCode = HttpStatusCode.Conflict; response.Content = new StringContent(Resources.Error_AutoSwapDeploymentOngoing); break; case FetchDeploymentRequestResult.Pending: // Shouldn't happen here, as we disallow deferral for this use case response.StatusCode = HttpStatusCode.Accepted; break; case FetchDeploymentRequestResult.RanSynchronously: response.StatusCode = HttpStatusCode.OK; break; case FetchDeploymentRequestResult.ConflictDeploymentInProgress: response.StatusCode = HttpStatusCode.Conflict; response.Content = new StringContent(Resources.Error_DeploymentInProgress); break; default: response.StatusCode = HttpStatusCode.BadRequest; break; } return(response); }
public async Task <ScanRequestResult> StartScan(String timeout, String mainScanDirPath, String id, String host) { using (_tracer.Step("Start scan in the background")) { String folderPath = Path.Combine(mainScanDirPath, Constants.ScanFolderName + id); String filePath = Path.Combine(folderPath, Constants.ScanStatusFile); Boolean hasFileModifcations = true; if (_deploymentLock.IsHeld) { return(ScanRequestResult.ScanAlreadyInProgress); } //Create unique scan folder and scan status file _deploymentLock.LockOperation(() => { //Check if files are modified if (CheckModifications(mainScanDirPath)) { //Create unique scan directory for current scan FileSystemHelpers.CreateDirectory(folderPath); _tracer.Trace("Unique scan directory created for scan {0}", id); //Create scan status file inside folder FileSystemHelpers.CreateFile(filePath).Close(); //Create temp file to check if scan is still running string tempScanFilePath = GetTempScanFilePath(mainScanDirPath); tempScanFilePath = Path.Combine(mainScanDirPath, Constants.TempScanFile); FileSystemHelpers.CreateFile(tempScanFilePath).Close(); UpdateScanStatus(folderPath, ScanStatus.Starting); } else { hasFileModifcations = false; } }, "Creating unique scan folder", TimeSpan.Zero); if (!hasFileModifcations) { return(ScanRequestResult.NoFileModifications); } //Start Backgorund Scan using (var timeoutCancellationTokenSource = new CancellationTokenSource()) { var successfullyScanned = PerformBackgroundScan(_tracer, folderPath, timeoutCancellationTokenSource.Token, id, mainScanDirPath); //Wait till scan task completes or the timeout goes off if (await Task.WhenAny(successfullyScanned, Task.Delay(Int32.Parse(timeout), timeoutCancellationTokenSource.Token)) == successfullyScanned) { //If scan task completes before timeout //Delete excess scan folders, just keep the maximum number allowed await DeletePastScans(mainScanDirPath, _tracer); //Create new Manifest file containing the modified timestamps String manifestPath = Path.Combine(mainScanDirPath, Constants.ScanManifest); if (FileSystemHelpers.FileExists(manifestPath)) { FileSystemHelpers.DeleteFileSafe(manifestPath); } JObject manifestObj = new JObject(); //Write to the manifest with new timestamps of the modified file ModifyManifestFile(manifestObj, Constants.ScanDir); File.WriteAllText(manifestPath, JsonConvert.SerializeObject(manifestObj)); //Path to common log file for azure monitor String aggrLogPath = Path.Combine(mainScanDirPath, Constants.AggregrateScanResults); //This checks if result scan log is formed //If yes, it will append necessary logs to the aggregrate log file //Current appended logs will be "Scanned files","Infected files", and details of infected files String currLogPath = Path.Combine(folderPath, Constants.ScanLogFile); if (FileSystemHelpers.FileExists(currLogPath)) { StreamReader file = new StreamReader(currLogPath); string line; while ((line = file.ReadLine()) != null) { if (line.Contains("FOUND") || line.Contains("Infected files") || line.Contains("Scanned files")) { //logType "Infected" means this log line represents details of infected files String logType = "Infected"; if (line.Contains("Infected files") || line.Contains("Scanned files")) { //logType "Info" means this log line represents total number of scanned or infected files logType = "Info"; } FileSystemHelpers.AppendAllTextToFile(aggrLogPath, DateTime.UtcNow.ToString(@"M/d/yyyy hh:mm:ss tt") + "," + id + "," + logType + "," + host + "," + line + '\n'); } } } return(successfullyScanned.Result ? ScanRequestResult.RunningAynschronously : ScanRequestResult.AsyncScanFailed); } else { //Timeout went off before scan task completion //Cancel scan task timeoutCancellationTokenSource.Cancel(); //Scan process will be cancelled //wait till scan status file is appropriately updated await successfullyScanned; //Delete excess scan folders, just keep the maximum number allowed await DeletePastScans(mainScanDirPath, _tracer); return(ScanRequestResult.AsyncScanFailed); } } } }
private static void CreateRequirements(string path = null) { var reqFile = string.IsNullOrEmpty(path) ? Constants.RequirementsTxt : Path.Combine(path, Constants.RequirementsTxt); FileSystemHelpers.CreateFile(reqFile); }
private async Task <HttpResponseMessage> PushDeployAsync(ZipDeploymentInfo deploymentInfo, bool isAsync) { var content = Request.Content; var isRequestJSON = content.Headers?.ContentType?.MediaType?.Equals("application/json", StringComparison.OrdinalIgnoreCase); if (isRequestJSON == true && ArmUtils.IsArmRequest(Request)) { try { deploymentInfo.ZipURL = await GetZipURLFromJSON(); } catch (Exception ex) { return(ArmUtils.CreateErrorResponse(Request, HttpStatusCode.BadRequest, ex)); } } else { if (_settings.RunFromLocalZip()) { await WriteSitePackageZip(deploymentInfo, _tracer, Request.Content); } else { var zipFileName = Path.ChangeExtension(Path.GetRandomFileName(), "zip"); var zipFilePath = Path.Combine(_environment.ZipTempPath, zipFileName); using (_tracer.Step("Writing content to {0}", zipFilePath)) { using (var file = FileSystemHelpers.CreateFile(zipFilePath)) { await content.CopyToAsync(file); } } deploymentInfo.RepositoryUrl = zipFilePath; } } isAsync = ArmUtils.IsArmRequest(Request) ? true : isAsync; var result = await _deploymentManager.FetchDeploy(deploymentInfo, isAsync, UriHelper.GetRequestUri(Request), "HEAD"); var response = Request.CreateResponse(); switch (result) { case FetchDeploymentRequestResult.RunningAynschronously: if (ArmUtils.IsArmRequest(Request)) { DeployResult deployResult = new DeployResult(); response = Request.CreateResponse(HttpStatusCode.Accepted, ArmUtils.AddEnvelopeOnArmRequest(deployResult, Request)); string statusURL = GetStatusUrl(); // Should not happen: If we couldn't make the URL, there must have been an error in the request if (string.IsNullOrEmpty(statusURL)) { var badResponse = Request.CreateResponse(); badResponse.StatusCode = HttpStatusCode.BadRequest; return(badResponse); } // latest deployment keyword reserved to poll till deployment done response.Headers.Location = new Uri(statusURL + String.Format("/deployments/{0}?api-version=2018-02-01&deployer={1}&time={2}", Constants.LatestDeployment, deploymentInfo.Deployer, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ"))); } else if (isAsync) { // latest deployment keyword reserved to poll till deployment done response.Headers.Location = new Uri(UriHelper.GetRequestUri(Request), String.Format("/api/deployments/{0}?deployer={1}&time={2}", Constants.LatestDeployment, deploymentInfo.Deployer, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ"))); } response.StatusCode = HttpStatusCode.Accepted; break; case FetchDeploymentRequestResult.ForbiddenScmDisabled: // Should never hit this for zip push deploy response.StatusCode = HttpStatusCode.Forbidden; _tracer.Trace("Scm is not enabled, reject all requests."); break; case FetchDeploymentRequestResult.ConflictAutoSwapOngoing: response.StatusCode = HttpStatusCode.Conflict; response.Content = new StringContent(Resources.Error_AutoSwapDeploymentOngoing); break; case FetchDeploymentRequestResult.Pending: // Shouldn't happen here, as we disallow deferral for this use case response.StatusCode = HttpStatusCode.Accepted; break; case FetchDeploymentRequestResult.RanSynchronously: response.StatusCode = HttpStatusCode.OK; break; case FetchDeploymentRequestResult.ConflictDeploymentInProgress: response.StatusCode = HttpStatusCode.Conflict; response.Content = new StringContent(Resources.Error_DeploymentInProgress); break; case FetchDeploymentRequestResult.ConflictRunFromRemoteZipConfigured: response.StatusCode = HttpStatusCode.Conflict; response.Content = new StringContent(Resources.Error_RunFromRemoteZipConfigured); break; default: response.StatusCode = HttpStatusCode.BadRequest; break; } return(response); }
public async Task <HttpResponseMessage> ZipPushDeploy( HttpRequestMessage request, [FromUri] bool isAsync = false, [FromUri] string author = null, [FromUri] string authorEmail = null, [FromUri] string deployer = DefaultDeployer, [FromUri] string message = DefaultMessage) { using (_tracer.Step("ZipPushDeploy")) { var zipFileName = Path.ChangeExtension(Path.GetRandomFileName(), "zip"); var zipFilePath = Path.Combine(_environment.ZipTempPath, zipFileName); using (_tracer.Step("Writing zip file to {0}", zipFilePath)) { using (var file = FileSystemHelpers.CreateFile(zipFilePath)) { await request.Content.CopyToAsync(file); } } var deploymentInfo = new ZipDeploymentInfo(_environment, _traceFactory) { AllowDeploymentWhileScmDisabled = true, Deployer = deployer, IsContinuous = false, AllowDeferredDeployment = false, IsReusable = false, RepositoryUrl = zipFilePath, TargetChangeset = DeploymentManager.CreateTemporaryChangeSet(message: "Deploying from pushed zip file"), CommitId = null, RepositoryType = RepositoryType.None, Fetch = LocalZipFetch, DoFullBuildByDefault = false, Author = author, AuthorEmail = authorEmail, Message = message }; var result = await _deploymentManager.FetchDeploy(deploymentInfo, isAsync, UriHelper.GetRequestUri(Request), "HEAD"); var response = request.CreateResponse(); switch (result) { case FetchDeploymentRequestResult.RunningAynschronously: if (isAsync) { // latest deployment keyword reserved to poll till deployment done response.Headers.Location = new Uri(UriHelper.GetRequestUri(Request), String.Format("/api/deployments/{0}?deployer={1}&time={2}", Constants.LatestDeployment, deploymentInfo.Deployer, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ"))); } response.StatusCode = HttpStatusCode.Accepted; break; case FetchDeploymentRequestResult.ForbiddenScmDisabled: // Should never hit this for zip push deploy response.StatusCode = HttpStatusCode.Forbidden; _tracer.Trace("Scm is not enabled, reject all requests."); break; case FetchDeploymentRequestResult.ConflictAutoSwapOngoing: response.StatusCode = HttpStatusCode.Conflict; response.Content = new StringContent(Resources.Error_AutoSwapDeploymentOngoing); break; case FetchDeploymentRequestResult.Pending: // Shouldn't happen here, as we disallow deferral for this use case response.StatusCode = HttpStatusCode.Accepted; break; case FetchDeploymentRequestResult.RanSynchronously: response.StatusCode = HttpStatusCode.OK; break; case FetchDeploymentRequestResult.ConflictDeploymentInProgress: response.StatusCode = HttpStatusCode.Conflict; response.Content = new StringContent(Resources.Error_DeploymentInProgress); break; default: response.StatusCode = HttpStatusCode.BadRequest; break; } return(response); } }