/// <summary> /// Build the work area /// </summary> /// <param name="connection"></param> /// <param name="dumpOnly">Dump commands only to standard logging interface rather than execute them</param> /// <param name="failNow">Return true to abort right away</param> /// <returns></returns> public static async Task <ISSHConnection> BuildWorkAreaAsync(this ISSHConnection connection, Func <bool> failNow = null, bool dumpOnly = false) { string findPkgError = null; var buildLines = new List <string>(); try { await connection.ExecuteLinuxCommandAsync("rc find_packages", l => findPkgError = l, failNow : failNow, dumpOnly : dumpOnly); return(await connection.ExecuteLinuxCommandAsync("rc compile", l => buildLines.Add(l), failNow : failNow, dumpOnly : dumpOnly)); } catch (LinuxCommandErrorException lerr) { if (buildLines.Count > 0) { var errors = buildLines.Where(ln => ln.Contains("error:")); var err = new StringBuilder(); err.AppendLine("Unable to compile package:"); foreach (var errline in errors) { err.AppendLine(" -> " + errline); } throw new LinuxCommandErrorException(err.ToString(), lerr); } else { throw new LinuxCommandErrorException(string.Format("Failed to run 'rc find_packages': {0}", findPkgError), lerr); } } }
/// <summary> /// Check out a Git package from the CERN git repro, of what is pecified in the package. /// </summary> /// <param name="connection">Connection on which to do the checkout</param> /// <param name="scPackagePath">Path to the package, perhaps a short-cut (see remarks)</param> /// <param name="scRevision">Revision for the package, or empty to grab the head</param> /// <param name="failNow">Fuction that returns true if we should bail right away</param> /// <param name="dumpOnly">Just dump the commands output - don't actually do anything.</param> /// <returns></returns> public static async Task <ISSHConnection> CheckoutPackageGitAsync(this ISSHConnection connection, string scPackagePath, string scRevision, Func <bool> failNow = null, bool dumpOnly = false) { // The revision must be specified. if (string.IsNullOrWhiteSpace(scRevision)) { throw new ArgumentException($"The commit hash for {scPackagePath} must be specified - it may well move with time"); } if (scRevision.Length < 15) { throw new ArgumentException($"The SHA {scRevision} for git package {scPackagePath} doesn't look like a full git SHA (e.g. 77658117c62ac99610068228668563d29baa3912)."); } // Determine if this was short-hand, and we need to put the gitlab specification in front. if (!scPackagePath.Contains("://")) { scPackagePath = $"https://:@gitlab.cern.ch:8443/{scPackagePath}"; } // Do the check out await connection.ExecuteLinuxCommandAsync($"git clone --recursive {scPackagePath}", refreshTimeout : true, secondsTimeout : 8 *60, failNow : failNow, dumpOnly : dumpOnly); // Now, we have to move to that revision. var pkgName = Path.GetFileNameWithoutExtension(scPackagePath.Split('/').Last()); string error = ""; await connection.ExecuteLinuxCommandAsync($"cd {pkgName}; git checkout {scRevision}; cd ..", processLine : l => error = l.Contains("error")?l : error); if (error.Length > 0) { throw new LinuxCommandErrorException($"Unable to check out package {scPackagePath} with SHA {scRevision} ({connection.MachineName}): {error}"); } return(connection); }
/// <summary> /// Returns the list of files associated with a dataset, as fetched from the grid. /// </summary> /// <param name="connection"></param> /// <param name="dataSetName"></param> /// <param name="dumpOnly">Dump only commands that are issues to standard logging interface.</param> /// <param name="failNow">Returns true to bail out as quickly as possible</param> /// <returns></returns> public static async Task <List <GRIDFileInfo> > FileInfoFromGRIDAsync(this ISSHConnection connection, string dataSetName, Func <bool> failNow = null, bool dumpOnly = false) { // If we have a cache hit, then avoid the really slow lookup. var c = _GRIDFileInfoCache.Value[dataSetName] as GRIDFileInfo[]; if (c != null) { return(c.ToList());; } // Run it in rucio and bring back our answers. var fileNameList = new List <GRIDFileInfo>(); var filenameMatch = new Regex(@"\| +(?<fname>\S*) +\| +[^\|]+\| +[^\|]+\| +(?<fsize>[0-9\.]*) *(?<fsizeunits>[kMGTB]*) *\| +(?<events>[0-9]*) +\|"); bool bad = false; try { await connection.ExecuteLinuxCommandAsync(string.Format("rucio list-files {0}", dataSetName), l => { if (l.Contains("Data identifier not found")) { bad = true; } if (!bad) { var m = filenameMatch.Match(l); if (m.Success) { var fname = m.Groups["fname"].Value; if (fname != "SCOPE:NAME") { // Parse it all out var gi = new GRIDFileInfo() { name = fname, eventCount = m.Groups["events"].Value == "" ? 0 : int.Parse(m.Groups["events"].Value), size = ConvertToMB(m.Groups["fsize"].Value, m.Groups["fsizeunits"].Value) }; fileNameList.Add(gi); } } } }, failNow : failNow, dumpOnly : dumpOnly ); } catch (LinuxCommandErrorException e) when(e.Message.Contains("status error") && bad) { // Swallow a command status error that we "sort-of" know about. } if (bad & !dumpOnly) { throw new DataSetDoesNotExistException($"Dataset '{dataSetName}' does not exist - can't get its list of files ({connection.MachineName})."); } _GRIDFileInfoCache.Value[dataSetName] = fileNameList.ToArray(); return(fileNameList); }
/// <summary> /// Get the PID for a process back. /// </summary> /// <param name="c"></param> /// <returns></returns> private async Task <string> GetPID(ISSHConnection c) { var pid = ""; await c.ExecuteLinuxCommandAsync("echo $$", s => pid = s); Assert.IsTrue(!string.IsNullOrWhiteSpace(pid)); return(pid); }
/// <summary> /// Setup a release. If it can't be found, error will be thrown /// </summary> /// <param name="connection"></param> /// <param name="releaseName">Full release name (e.g. 'Base,2.3.30')</param> /// <param name="linuxLocation">Directory where the linux location can be found</param> /// <param name="dumpOnly">Dump commands to standard logging, but do not execute anything</param> /// <returns></returns> public static async Task <ISSHConnection> SetupRcReleaseAsync(this ISSHConnection connection, string linuxLocation, string releaseName, bool dumpOnly = false) { // Check the arguments for something dumb if (string.IsNullOrWhiteSpace(releaseName)) { throw new ArgumentException("A release name must be provided"); } if (string.IsNullOrWhiteSpace(linuxLocation) || !linuxLocation.StartsWith("/")) { throw new ArgumentException("The release directory must be an absolute Linux path (start with a '/')"); } // First we have to create the directory. bool dirCreated = true; bool dirAlreadyExists = false; await connection.ExecuteLinuxCommandAsync(string.Format("mkdir {0}", linuxLocation), l => { dirCreated = !dirCreated ? false : string.IsNullOrWhiteSpace(l); dirAlreadyExists = dirAlreadyExists ? true : l.Contains("File exists"); }, dumpOnly : dumpOnly); if (dirAlreadyExists) { throw new LinuxMissingConfigurationException($"Release directory '{linuxLocation}' already exists - we need a fresh start ({connection.MachineName})"); } if (!dirCreated) { throw new LinuxMissingConfigurationException(string.Format($"Unable to create release directory '{linuxLocation}' ({connection.MachineName}).")); } // Next, put our selves there await connection.ExecuteLinuxCommandAsync(string.Format("cd {0}", linuxLocation), dumpOnly : dumpOnly); // And then do the setup bool found = false; await connection.ExecuteLinuxCommandAsync(string.Format("rcSetup {0}", releaseName), l => found = found?true : l.Contains("Found ASG release with"), dumpOnly : dumpOnly); if (!found && !dumpOnly) { throw new LinuxMissingConfigurationException($"Unable to find release '{releaseName}' ({connection.MachineName})"); } // Return the connection to make it a functional interface. return(connection); }
/// <summary> /// Execute a Linux command. Throw if the command does not return 0 to the shell. Provides for ways to capture (or ignore) /// the output of the command. /// </summary> /// <param name="connection">The connection onwhich to execute the command.</param> /// <param name="command">The command to execute</param> /// <param name="dumpOnly">If true, then only print out the commands</param> /// <param name="failNow">If true, attempt to bail out of the command early.</param> /// <param name="processLine">A function called for each line read back while the command is executing</param> /// <param name="seeAndRespond">If a string is seen in the output, then the given response is sent</param> /// <param name="refreshTimeout">If we see text, reset the timeout counter</param> /// <param name="secondsTimeout">How many seconds with no output or since beginning of command before we declare failure?</param> /// <returns>The connection we ran this on. Enables fluent progreamming</returns> /// <remarks> /// We check the status by echoing the shell variable $? - so this actually runs two commands. /// </remarks> public static ISSHConnection ExecuteLinuxCommand(this ISSHConnection connection, string command, Action <string> processLine = null, Func <bool> failNow = null, bool dumpOnly = false, int secondsTimeout = 60 * 60, bool refreshTimeout = false, Dictionary <string, string> seeAndRespond = null) { return(connection.ExecuteLinuxCommandAsync(command, processLine, failNow, dumpOnly, secondsTimeout, refreshTimeout, seeAndRespond) .WaitAndUnwrapException()); }
/// <summary> /// Send all files over to the remote client. Make sure the directory exists. /// </summary> /// <param name="connection"></param> /// <param name="dumpLine"></param> private async Task SendAllFiles(ISSHConnection connection, Action <string> dumpLine) { foreach (var f in _filesToCopyOver) { string linuxPath = $"{f.remoteLinuxDirectory}/{f.localFileName.Name}"; dumpLine?.Invoke($"Copying {f.localFileName.Name} -> {linuxPath}"); await connection.ExecuteLinuxCommandAsync($"mkdir -p {f.remoteLinuxDirectory}", dumpLine); await connection.CopyLocalFileRemotelyAsync(f.localFileName, linuxPath); } _filesToCopyOver.Clear(); }
/// <summary> /// Run the setup ATLAS command asyncronously. /// </summary> /// <param name="connection">The connection that will understand the setupATLAS command</param> /// <param name="dumpOnly">If true, then tex tis dumpped to standard logging</param> /// <returns>A reconfigured SSH shell connection (same as what went in)</returns> public static async Task <ISSHConnection> setupATLASAsync(this ISSHConnection connection, bool dumpOnly = false) { bool badCommand = false; var r = await connection .ExecuteLinuxCommandAsync("setupATLAS", dumpOnly : dumpOnly, processLine : l => badCommand = badCommand || l.Contains("command not found"), secondsTimeout : 120); if (badCommand) { throw new LinuxConfigException($"Unable to setupATLAS - command is not known! ({connection.MachineName})"); } return(r); }
/// <summary> /// Setup Rucio. ATLAS must have been previously configured, or this will "crash". Execute Asyncrohonously. /// </summary> /// <param name="connection">Connection on-which we will set everything up</param> /// <param name="rucioUserName">The user alias used on the grid</param> /// <param name="dumpOnly">Dump output to the logging interface rather than actually executing anything</param> /// <returns>A shell on-which rucio has been setup (the same connection that went in)</returns> public static async Task <ISSHConnection> setupRucioAsync(this ISSHConnection connection, string rucioUserName, bool dumpOnly = false) { int hashCount = 0; var r = await connection.ExecuteLinuxCommandAsync(string.Format("export RUCIO_ACCOUNT={0}", rucioUserName), dumpOnly : dumpOnly, secondsTimeout : 30); r = await r.ExecuteLinuxCommandAsync("lsetup rucio", dumpOnly : dumpOnly, secondsTimeout : 30); r = await r.ExecuteLinuxCommandAsync("hash rucio", l => hashCount += 1, dumpOnly : dumpOnly, secondsTimeout : 30); if (hashCount != 0 && !dumpOnly) { throw new LinuxConfigException("Unable to setup Rucio... did you forget to setup ATLAS first? ({connection.MachineName})"); } return(connection); }
/// <summary> /// Execute a kinit /// </summary> /// <param name="connection"></param> /// <param name="userName"></param> /// <param name="password"></param> /// <param name="dumpOnly">Dump commands to logging interface rather than execute them</param> /// <returns></returns> public static async Task <ISSHConnection> KinitAsync(this ISSHConnection connection, string userName, string password, bool dumpOnly = false) { var allStrings = new List <string>(); await connection.ExecuteLinuxCommandAsync(string.Format("echo {0} | kinit {1}", password, userName), l => allStrings.Add(l), dumpOnly : dumpOnly); var errorStrings = allStrings.Where(l => l.StartsWith("kinit:")).ToArray(); if (errorStrings.Length > 0) { throw new LinuxCommandErrorException($"Failed to execute kinit command: {errorStrings[0]} ({connection.MachineName})"); } if (!allStrings.Where(l => l.StartsWith("Password for")).Any() && !dumpOnly) { throw new LinuxCommandErrorException($"Failed to execute kinit command: {allStrings[0]} ({connection.MachineName})"); } return(connection); }
/// <summary> /// Initializes the VOMS proxy for use on the GRID. The connection must have already been configured so that /// the command voms-proxy-init works. /// </summary> /// <param name="connection">The configured SSH shell</param> /// <param name="GRIDUsername">The username to use to fetch the password for the voms proxy file</param> /// <param name="voms">The name of the voms to connect to</param> /// <param name="dumpOnly">Only print out proposed commands</param> /// <returns>Connection on which the grid is setup and ready to go</returns> public static async Task <ISSHConnection> VomsProxyInitAsync(this ISSHConnection connection, string voms, Func <bool> failNow = null, bool dumpOnly = false) { // Get the GRID VOMS password var sclist = new CredentialSet("GRID"); var passwordInfo = sclist.Load().FirstOrDefault(); if (passwordInfo == null) { throw new ArgumentException("There is no generic windows credential targeting the network address 'GRID' for username. This password should be your cert pass phrase and your on-the-grid username. Please create one on this machine."); } // Run the command bool goodProxy = false; var whatHappened = new List <string>(); var r = await connection .ExecuteLinuxCommandAsync(string.Format("echo {0} | voms-proxy-init -voms {1}", passwordInfo.Password, voms), l => { goodProxy = goodProxy || l.Contains("Your proxy is valid"); whatHappened.Add(l); }, secondsTimeout : 20, failNow : failNow, dumpOnly : dumpOnly ); // If we failed to get the proxy, then build an error message that can be understood. Since this // could be for a large range of reasons, we are going to pass back a lot of info to the user // so they can figure it out (not likely a program will be able to sort this out). if (goodProxy == false && !dumpOnly) { var error = new StringBuilder(); error.AppendLine($"Failed to get the proxy ({connection.MachineName}): "); foreach (var l in whatHappened) { error.AppendLine(string.Format(" -> {0}", l)); } throw new ArgumentException(error.ToString()); } return(connection); }
private static async Task DoRucioDownloadAsync(ISSHConnection connection, string localDirectory, Action <string> fileStatus, Func <bool> failNow, int timeout, string fileListName) { string filesThatFailedToDownload = ""; bool foundClockSkewMessage = false; var messageNameMatch = new Regex(@"Starting the download of ([\w\.:]+)"); await connection.ExecuteLinuxCommandAsync($"rucio -T {timeout} download --dir {localDirectory} `cat {fileListName}`", l => { // Look for something that indicates which file we are currently getting from the GRID. if (fileStatus != null) { var m = messageNameMatch.Match(l); if (m.Success) { fileStatus(m.Groups[1].Value); } } // Watch for the end to see the overall status if (l.Contains("Files that cannot be downloaded :")) { filesThatFailedToDownload = l.Split(' ').Where(i => !string.IsNullOrWhiteSpace(i)).Last(); } foundClockSkewMessage |= l.Contains("check clock skew between hosts."); }, refreshTimeout : true, failNow : failNow, secondsTimeout : timeout); // Check for errors that happened while running the command. if (filesThatFailedToDownload != "0") { // Special case - there was a clock skew error. if (foundClockSkewMessage) { throw new ClockSkewException($"Failed to download {filesThatFailedToDownload} files due to clock skew. Please double check ({connection.MachineName})!"); } // Something else - will likely require a human to get involved. throw new FileFailedToDownloadException($"Failed to download all the files from the GRID - {filesThatFailedToDownload} files failed to download ({connection.MachineName})!"); } }
/// <summary> /// Check out a SVN package /// </summary> /// <param name="connection">Connection on which to check this guy out</param> /// <param name="scPackagePath">Path to the package</param> /// <param name="scRevision">tag/svn revision to check out</param> /// <param name="failNow">Function retursn true when we should quit</param> /// <param name="dumpOnly">Are we just dumping commands, or doing work?</param> /// <returns></returns> public static async Task <ISSHConnection> CheckoutPackageSVNAsync(this ISSHConnection connection, string scPackagePath, string scRevision, Func <bool> failNow = null, bool dumpOnly = false) { // Has the user asked us to do something we won't do? if (scPackagePath.EndsWith("/tags")) { // This throws because it moves with time - so we can't fully specify what package we are checking out. throw new ArgumentException(string.Format("The package path ({0}) can't end with a tags directory to indicate the latest tag - the tag will change over time and can't be tracked.", scPackagePath)); } if (scPackagePath.EndsWith("/trunk")) { // This throws because it moves with time. throw new ArgumentException(string.Format("The package path ({0}) can't end with a trunk directory to indicate the HEAD version - the tag will change over time and can't be tracked.", scPackagePath)); } if (scPackagePath.EndsWith("/")) { throw new ArgumentException(string.Format("The package path {0} ends with a slash - this is not allowed", scPackagePath)); } if (!string.IsNullOrWhiteSpace(scRevision) && !scPackagePath.Contains("/")) { throw new ArgumentException(string.Format("If a revision is specified then the package path must be fully specified (was '{0}' - {1})", scPackagePath, scRevision)); } // How we run the command will depend on if this is a release version of the package // or this is a revision specified. var fullPackagePath = scPackagePath; if (!string.IsNullOrWhiteSpace(scRevision)) { if (fullPackagePath.Contains("/trunk/")) { fullPackagePath += "@" + scRevision; } else { fullPackagePath += "/trunk@" + scRevision; } } var sawRevisionMessage = false; await connection.ExecuteLinuxCommandAsync(string.Format("rc checkout_pkg {0}", fullPackagePath), l => sawRevisionMessage = sawRevisionMessage?true : l.Contains("Checked out revision"), secondsTimeout : 120, failNow : failNow, dumpOnly : dumpOnly); if (!sawRevisionMessage && !dumpOnly) { throw new LinuxCommandErrorException($"Unable to check out svn package {scPackagePath} ({connection.MachineName})."); } // If this was checked out to trunk, then we need to fix it up. if (!string.IsNullOrWhiteSpace(scRevision)) { var packageName = scPackagePath.Split('/').Last(); var checkoutName = fullPackagePath.Split('/').Last(); bool lineSeen = false; await connection.ExecuteLinuxCommandAsync(string.Format("mv {0} {1}", checkoutName, packageName), l => lineSeen = true, dumpOnly : dumpOnly); if (lineSeen && !dumpOnly) { throw new LinuxCommandErrorException($"Unable to rename the downloaded trunk directory for package '{scPackagePath}' ({connection.MachineName})."); } } return(connection); }
/// <summary> /// Fetch a dataset from the grid using Rucio to a local directory. /// </summary> /// <param name="connection">A previously configured connection with everything ready to go for GRID access.</param> /// <param name="dataSetName">The rucio dataset name</param> /// <param name="localDirectory">The local directory (on Linux) where the file should be downloaded</param> /// <param name="fileStatus">Gets updates as new files are downloaded. This will contain just the filename.</param> /// <param name="fileNameFilter">Filter function to alter the files that are to be downloaded</param> /// <param name="failNow">Checked periodically, if ever returns true, then bail out</param> /// <param name="timeout">How long before we should timeout in seconds</param> /// <returns>The connections used so you can chain</returns> public static async Task <ISSHConnection> DownloadFromGRIDAsync(this ISSHConnection connection, string dataSetName, string localDirectory, Action <string> fileStatus = null, Func <string[], string[]> fileNameFilter = null, Func <bool> failNow = null, int timeout = 3600) { // Does the dataset exist? if (fileStatus != null) { fileStatus("Checking the dataset exists"); } var response = new List <string>(); await connection.ExecuteLinuxCommandAsync(string.Format("rucio ls {0}", dataSetName), l => response.Add(l), secondsTimeout : 60, failNow : failNow); var dsnames = response .Where(l => l.Contains("DATASET") | l.Contains("CONTAINER")) .Select(l => l.Split(' ')) .Where(sl => sl.Length > 1) .Select(sl => sl[1]) .ToArray(); if (!dsnames.Where(n => n.SantizeDSName() == dataSetName.SantizeDSName()).Any()) { throw new ArgumentException(string.Format("Unable to find any datasets on the GRID (in rucuio) with the name '{0}'.", dataSetName)); } // Get the complete list of files in the dataset. if (fileStatus != null) { fileStatus("Getting the complete list of files from the dataset"); } var fileNameList = await connection.FilelistFromGRIDAsync(dataSetName, failNow : failNow); // Filter them if need be. var goodFiles = fileNameFilter != null ? fileNameFilter(fileNameList) : fileNameList; // Create a file that contains all the files we want to download up on the host. var fileListName = string.Format("/tmp/{0}.filelist", dataSetName.SantizeDSName()); await connection.ExecuteLinuxCommandAsync("rm -rf " + fileListName); await connection.ApplyAsync(goodFiles, async (c, fname) => await c.ExecuteLinuxCommandAsync(string.Format("echo {0} >> {1}", fname, fileListName))); // We good on creating the directory? await connection.ExecuteLinuxCommandAsync(string.Format("mkdir -p {0}", localDirectory), l => { throw new ArgumentException($"Error trying to create directory {0} for dataset on remote machine ({connection.MachineName}).", localDirectory); }, secondsTimeout : 20); // If we have no files to download, then we are totally done! // We do this after the directory is created, so if there are no files, a check still // works. if (goodFiles.Length == 0) { return(connection); } // Next, do the download response.Clear(); fileStatus?.Invoke($"Starting GRID download of {dataSetName}..."); await Policy .Handle <ClockSkewException>() .WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(10), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(1) }, (e, ts) => { fileStatus?.Invoke("Clock Skew error - wait and re-try"); }) .ExecuteAsync(() => DoRucioDownloadAsync(connection, localDirectory, fileStatus, failNow, timeout, fileListName)); return(connection); }