/// <summary> /// Downloads (And extracts) single artifacts from jobs. /// </summary> /// <param name="applicationId"></param> /// <param name="build"></param> /// <param name="artifactRegex"></param> /// <param name="destinationPath"></param> /// <param name="logger"></param> public void DownloadSingleArtifactFromBuild( string applicationId, Build build, string artifactRegex, string destinationPath, ILoggerInterface logger) { UtilsSystem.EnsureDirectoryExists(destinationPath, true); // Use the first job in the build... var job = build.jobs.First(); var artifact = this.FindDefaultArtifactForBuild(job, build, artifactRegex); var filename = artifact.fileName; var extension = Path.GetExtension(filename); string downloadTemporaryDir = UtilsSystem.EnsureDirectoryExists(UtilsSystem.CombinePaths(this.TempDir, "_appveyor", "dld", applicationId), true); int artifactRetentionNum = 5; int artifactAgeHoursForStale = 24; // Do not touch the latest artifactRetentionNum artifacts or artifacts that are not older than artifactAgeHoursForStale hours var staleFiles = Directory.EnumerateFiles(downloadTemporaryDir) .Select((i) => new FileInfo(i)) .Where((i) => i.Extension.Equals(".zip", StringComparison.CurrentCultureIgnoreCase)) .OrderByDescending((i) => i.CreationTimeUtc) .Skip(artifactRetentionNum) .Where((i) => (DateTime.UtcNow - i.LastWriteTime).TotalHours > artifactAgeHoursForStale) .ToList(); foreach (var f in staleFiles) { // Make this fail proof, it's just a cleanup. try { this.Logger.LogInfo(true, "Removing stale artifact cache file {0}", f.FullName); f.Delete(); } catch { // ignored } } // Use a short hash as the temporary file name, because long paths can have issues... var tmpFile = UtilsSystem.CombinePaths(downloadTemporaryDir, UtilsEncryption.GetShortHash(JsonConvert.SerializeObject(build) + filename) + extension); if (Path.GetExtension(tmpFile)?.ToLower() != ".zip") { throw new NotImplementedException("AppVeyor artifacts should only be Zip Files."); } if (!File.Exists(tmpFile)) { // Use an intermediate .tmp file just in case the files does not finish to download, // if it exists, clear it. string tmpFileDownload = tmpFile + ".tmp"; if (File.Exists(tmpFileDownload)) { UtilsSystem.RetryWhile(() => File.Delete(tmpFileDownload), (e) => true, 4000, this.Logger); } var url = $"/api/buildjobs/{job.jobId}/artifacts/{filename}"; logger.LogInfo(true, "Downloading artifact from: '{0}' to '{1}'", url, tmpFileDownload); this.ExecuteApiCallToFile(url, tmpFileDownload); // Rename to the final cached artifact file logger.LogInfo(true, "Download succesful, moving to '{0}'", tmpFile); UtilsSystem.RetryWhile(() => File.Move(tmpFileDownload, tmpFile), (e) => true, 4000, this.Logger); } else { logger.LogInfo(true, "Skipping artifact download, already in local cache: {0}", tmpFile); } logger.LogInfo(true, "Unzipping {1} file to '{0}'...", destinationPath, UtilsSystem.BytesToString(new FileInfo(tmpFile).Length)); ZipFile.ExtractToDirectory(tmpFile, destinationPath); logger.LogInfo(true, "Unzipping finished."); }
/// <summary> /// Execute the publish/republish commands. /// </summary> /// <param name="command"></param> /// <param name="build"></param> protected void RunPublishCommand(Message command, utils.AppVeyor.Build build) { double lifetime = double.Parse(command.arguments[0]); // Limit the lifetime to 7 days if (lifetime > 168) { lifetime = 168; } // To prevent colision between projects, add a small project hash var projectHash = UtilsEncryption.GetShortHash(build.project.name, 2); // Use the branch name to generate a unique application name var appId = $"{Application.AutoDeployApplicationIdPrefix}{projectHash}_{build.branch}".ToLower(); // Use an in-code template to deploy string template = string.Format( @"id: '{0}' expires: {4} tags: '{6}' autodeploy: true downloader: type: 'appveyor' project: '{1}' username: '******' apitoken: '{5}' branch: '{3}' ", appId, this.Settings.project, this.Settings.username, build.branch, lifetime, this.Settings.apitoken, // Tag to indicate that this is an autodeployment. "autodeploy"); // Update the deployment template string templateFilePath = Path.Combine(this.app.GetGlobalSettings().applicationTemplateDir, appId + ".yml"); File.WriteAllText(templateFilePath, template); if ("republish".Equals(command.command, StringComparison.CurrentCultureIgnoreCase)) { bool doRemove = true; // To prevent infinite redeployment loops // we need to make sure that we do not remove // the application if currently installed version (if any) var application = this.app.GetInstalledApp(appId); var buildVersion = new Version("0.0.0"); var deployedVersion = new Version("0.0.0"); if (application != null) { var deployer = this.app.GetDeployer(application); Version.TryParse(build.version, out buildVersion); Version.TryParse(deployer.DeploymentActive?.artifact?.id, out deployedVersion); doRemove = buildVersion > deployedVersion; } if (doRemove) { this.Logger.LogInfo(false, "Application republish from version '{0}' to '{1}'", deployedVersion, buildVersion); this.app.RemoveAppById(appId, true); } } // Deploy from template... this.app.RedeployInstalledApplication(true, appId, false); // Get rid of the template, this is not useful anymore. File.Delete(templateFilePath); }
/// <summary> /// Deploys an installed app. /// </summary> /// <param name="app"></param> /// <param name="force"></param> /// <param name="buildId"></param> /// <param name="sync"></param> protected Deployment _DeployApp( Application app, bool force = false, string buildId = null, bool sync = false) { DateTime start = DateTime.Now; // The parent application to inherit from (if needed) InstalledApplication parentApp = null; // Lo primero es ver si hay algo nuevo... var downloader = this.installedAppSettings.GetDownloader(this.GlobalSettings, this.Logger); if (!string.IsNullOrWhiteSpace(buildId)) { this.Logger.LogInfo(true, "Deploying specific version build: '{0}'", buildId); } string nextArtifactId; // Next artifact id might be pulled from a remote location, and this prompt to random failures (network, etc.) // so wrap this in a try/catch try { nextArtifactId = downloader.GetNextId(buildId == "latest" ? null : buildId); } catch (Exception e) { this.Logger.LogException(new Exception("Failure while looking for next build ID", e), EventLogEntryType.Warning); return(this.DeploymentActive); } string currentArtifactId = this.DeploymentActive != null ? this.DeploymentActive.artifact.id : string.Empty; bool isNew = this.DeploymentActive == null || this.DeploymentActive.artifact.id != nextArtifactId; // Check that Inherit application exists if (!string.IsNullOrEmpty(this.installedAppSettings.GetInherit())) { parentApp = app.GetInstalledApp(this.installedAppSettings.GetInherit()); if (parentApp == null) { throw new Exception( $"Application from inheritation: {this.installedAppSettings.GetInherit()}, can not be found"); } this.Logger.LogInfo(true, "Application configured to inherit from parent application '{0}'. Sync:{1}", parentApp.GetId(), sync ? "Yes" : "No"); } // Si no es nuevo y no estamos forzando, no hacer deploy. if (!isNew && !force) { this.Logger.LogInfo(true, "No new version found for Application {0}", this.installedAppSettings.GetId()); return(this.DeploymentActive); } // There is an existing deployment that had a manually enforced BuildId if (!string.IsNullOrEmpty(this.DeploymentActive?.enforceBuildId)) { if (force) { if (!string.IsNullOrWhiteSpace(buildId)) { // If there is a force and a buildId has been specified, override the next artifactId // with the requested BuildId, or if latest was specified use that. if (buildId != "latest") { nextArtifactId = buildId; } } else { // If no specific build was requested, override the nextArtifactId with // the stored build nextArtifactId = this.DeploymentActive.enforceBuildId; this.Logger.LogWarning(true, "Deploying stored version {0}", nextArtifactId); } } else if (buildId != this.DeploymentActive.enforceBuildId && buildId != "latest") { this.Logger.LogWarning(true, $"Deployment was skipped because previous deployment was a version-specific deployment. Previous buildId='{this.DeploymentActive.enforceBuildId}'. Requested buildId='{buildId}'. Use buildId='latest' to force deploying the latest succesful build or -Force to deploy this version."); return(this.DeploymentActive); } } this.Logger.LogInfo(false, "@@ Starting deployment for app: '{0}'", this.installedAppSettings.GetId()); this.Logger.LogInfo(false, "Current artifact: '{0}' || Previous artifact: '{1}'", nextArtifactId, currentArtifactId); // Specify a local temporary artifact location, in case this is supported by the downloader... // final path should be retrieved from artifact.localPath string preferredLocalArtifactPath = UtilsSystem.EnsureDirectoryExists( UtilsSystem.CombinePaths( this.GlobalSettings.GetDefaultApplicationStorage().path, "_tmp", this.installedAppSettings.GetId(), UtilsEncryption.GetShortHash(nextArtifactId, 12)), true); // Get from the ID... Artifact artifact = downloader.PullFromId(nextArtifactId, preferredLocalArtifactPath); if (string.IsNullOrWhiteSpace(this.GlobalSettings.id)) { throw new Exception("Environment settings cannot have an empty ID."); } this.Logger.LogInfo(false, "Environment id: '{0}'", this.GlobalSettings.id); this.Logger.LogInfo(false, "Environment options/tags: '{0}'", string.Join(",", this.GlobalSettings.getOptions())); this.Logger.LogInfo(false, "Pull artifact lapsed: {0}s", (DateTime.Now - start).TotalSeconds); start = DateTime.Now; // Look for a configuration file that fits this environment. string chefsettingsdir = UtilsSystem.CombinePaths(artifact.localPath, "chef"); // The final chef configuration files is a combination of Chef files var appSettings = this.LoadApplicationSettings( chefsettingsdir, this.GlobalSettings.id, artifact.artifactSettings.branch, out var loadedConfigurationFiles); // Storage for current deployment. Includes all possible environment data // in order to provide traceability + rollback capabilities. Deployment deployment = new Deployment( appSettings, this.GlobalSettings, artifact, this.installedAppSettings, parentApp); deployment.SetPreviousDeployment(this.DeploymentActive); // Check the deployment windows! var deploymentSettings = deployment.appSettings.getDeploymentSettings(); if (deploymentSettings != null) { if (deploymentSettings.deployment_windows != null && deploymentSettings.deployment_windows.Any()) { bool canDeploy = false; foreach (var deploymentWindow in deploymentSettings.deployment_windows) { TimeZoneInfo info = TimeZoneInfo.FindSystemTimeZoneById(deploymentWindow.Value.timezone); TimeSpan dtStart = TimeSpan.Parse(deploymentWindow.Value.start); TimeSpan dtEnd = TimeSpan.Parse(deploymentWindow.Value.end); DateTimeOffset localServerTime = DateTimeOffset.Now; DateTimeOffset windowTimeZone = TimeZoneInfo.ConvertTime(localServerTime, info); TimeSpan dtNow = windowTimeZone.TimeOfDay; if (dtStart <= dtEnd) { // start and stop times are in the same day if (dtNow >= dtStart && dtNow <= dtEnd) { // current time is between start and stop canDeploy = true; break; } } else { // start and stop times are in different days if (dtNow >= dtStart || dtNow <= dtEnd) { // current time is between start and stop canDeploy = true; break; } } } // Even if we are not in a deployment windows, // if we are forcing the deployment continue. if (!canDeploy && !force) { this.Logger.LogInfo(false, "Application deployment skipped. Current time not within allowed publishing windows."); return(this.DeploymentActive); } } } // Inform about the confiugration files that where used for loading deployment.SetRuntimeSetting("deployment.loaded_configuration_files", string.Join(",", loadedConfigurationFiles)); deployment.enforceBuildId = buildId == "latest" ? null : buildId; var deployersActive = this.DeploymentActive != null?this.DeploymentActive.GrabDeployers(this.Logger) : new DeployerCollection(this.GlobalSettings, null, this.Logger, parentApp); var servicesActive = this.DeploymentActive != null?this.DeploymentActive.GrabServices(this.Logger) : new DeployerCollection(this.GlobalSettings, null, this.Logger, parentApp); var deployers = deployment.GrabDeployers(this.Logger); var services = deployment.GrabServices(this.Logger); this.Logger.LogInfo(false, "Deployers and services gathered. Starting installation..."); var settingsConverter = new JObjectToKeyValueConverter(); try { // Deploy the application base storage (logs, runtime, etc.) deployers.DeployAll(); services.DeployAll(); // Move the application settings to runtime settings var userApplicationSettings = appSettings.getApplicationSettings(); foreach (var k in settingsConverter.NestedToKeyValue(userApplicationSettings)) { deployment.SetRuntimeSetting("app_settings." + k.Key, k.Value); } // Sync if (sync) { deployers.SyncAll(); services.SyncAll(); } // Time to hot switch the sites... we need to waitPauseMs for all // current requests to finish... because that way we ensure // that underlying storage updates will not collide if updates // are being deployed. servicesActive.StopAll(); deployersActive.StopAll(); // Some stuff requires the old services to be stopped in order to be deployed, such as IIS bindings and certificates deployers.BeforeDoneAll(); services.BeforeDoneAll(); var settingsToDeploy = deployment.GetRuntimeSettingsToDeploy(); // Store Key-Value settings in a JSON object (with keys as var jsonSettings = JsonConvert.SerializeObject( settingsToDeploy, Formatting.Indented); var jsonSettingsNested = JsonConvert.SerializeObject( settingsConverter.keyValueToNested(settingsToDeploy), Formatting.Indented); // Make sure we persist the settings AFTER all deployers have finished thri job deployers.DeploySettingsAll(jsonSettings, jsonSettingsNested); services.DeploySettingsAll(jsonSettings, jsonSettingsNested); // Time to start! deployers.StartAll(); services.StartAll(); DateTime dtStart = DateTime.Now; // Replace active configuration settings deployment.StoreInPath(this.activeDeploymentPathStorage); // Quitar el deployment anterior y si hay error seguir, // ya que los datos del deployment actual YA están guardados! servicesActive.UndeployAll(true); deployersActive.UndeployAll(true); // The done "event" is called on deployers // once everything is completed correctly. deployers.DoneAll(true); services.DoneAll(true); // Make sure that at least 2 seconds pass after deployment before // doing an OK to let IIS reconfigure. while ((DateTime.Now - dtStart).TotalSeconds < 1) { System.Threading.Thread.Sleep(500); } } catch (Exception e) { // Just in case.... log this ASAP this.Logger.LogException( new Exception("Error deploying APP: " + deployment.installedApplicationSettings.GetId(), e)); deployers.StopAll(true); deployers.UndeployAll(true); // Aquí hacemos un continue on error porque... estamos repescando algo que ya funcionaba // a toda costa queremos levantarlo! deployersActive.StartAll(true); servicesActive.StopAll(true); // In unit test rethrow to preserve stack trace in GUI if (UnitTestDetector.IsRunningInTests) { System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e) .Throw(); } else { throw new AlreadyHandledException(e.Message, e); } } finally { // Run cleanup, dot not fail if cleanup fails, it's just an extra... deployers.CleanupAll(true); services.CleanupAll(true); } // Done! this.Logger.LogInfo(false, "Deployment lapsed: {0}s", (DateTime.Now - start).TotalSeconds); return(deployment); }