static public bool BatchUploadFolder(string MacName, string SourceFolder, string DestFolder, bool bDeleteTarget)
{
if (Config.bUseRPCUtil)
{
if (!ConditionalInitRemoting(MacName))
{
return(false);
}
if (bDeleteTarget)
{
// tell Mac to delete target if requested
RPCCommandHelper.RemoveDirectory(RPCSocket, DestFolder);
}
}
Program.Log(" ... '" + SourceFolder + "' -> '" + DestFolder + "'");
DirectoryInfo SourceFolderInfo = new DirectoryInfo(SourceFolder);
if (!SourceFolderInfo.Exists)
{
Program.Error("Source folder does not exist");
return(false);
}
// gather the files to batch upload
List <string> FilesToUpload = new List <string>();
RecursiveBatchUploadFolder(SourceFolderInfo, DestFolder, FilesToUpload);
if (Config.bUseRPCUtil)
{
// send them off!
RPCCommandHelper.RPCBatchUpload(RPCSocket, FilesToUpload.ToArray());
}
else
{
SSHCommandHelper.BatchUpload(MacName, FilesToUpload.ToArray());
}
return(true);
}
/** Cranks up the remoting utility if needed */
static private bool ConditionalInitRemoting(string MacName)
{
if (RPCCommandHelper.PingRemoteHost(MacName))
{
try
{
RPCSocket = RPCCommandHelper.ConnectToUnrealRemoteTool(MacName);
}
catch (Exception)
{
Console.WriteLine("Failed to connect to UnrealRemoteTool running on {0}", MacName);
return false;
}
}
else
{
return false;
}
return true;
}
public static Hashtable Command(string WorkingDirectory, string Command, string CommandArgs, string RemoteOutputPath)
{
if (RemoteToolChain.bUseRPCUtil)
{
int RetriesRemaining = 6;
do
{
// a $ on the commandline will actually be converted, so we need to quote it
CommandArgs = CommandArgs.Replace("$", "\\$");
try
{
Hashtable Results = RPCCommandHelper.RPCCommand(GetSocket(), WorkingDirectory, Command, CommandArgs, RemoteOutputPath);
return(Results);
}
catch (Exception Ex)
{
if (RetriesRemaining > 0)
{
Int32 RetryTimeoutMS = 1000;
Debug.WriteLine("Retrying command after sleeping for " + RetryTimeoutMS + " milliseconds. Command is:" + Command + " " + CommandArgs);
Thread.Sleep(RetryTimeoutMS);
}
else
{
Log.TraceInformation("Out of retries, too many exceptions:" + Ex.ToString());
// We've tried enough times, just throw the error
throw new Exception("Deep Exception, retries exhausted... ", Ex);
}
RetriesRemaining--;
}
}while (RetriesRemaining > 0);
return(null);
}
else
{
return(RemoteToolChain.SSHCommand(WorkingDirectory, Command + " " + CommandArgs, RemoteOutputPath));
}
}
private static Socket GetSocket()
{
Socket ThreadSocket = null;
lock (CommandThreadSockets)
{
ThreadSocket = CommandThreadSockets[Thread.CurrentThread] as Socket;
if (ThreadSocket == null)
{
try
{
ThreadSocket = RPCCommandHelper.ConnectToUnrealRemoteTool(MacName);
}
catch (Exception Ex)
{
Log.TraceInformation("Failed to connect to UnrealRemoteTool running on {0}.", MacName);
throw new BuildException(Ex, "Failed to connect to UnrealRemoteTool running on {0}.", MacName);
}
CommandThreadSockets[Thread.CurrentThread] = ThreadSocket;
}
}
return(ThreadSocket);
}
static Int32 Main(string[] args)
{
Int64 ExitCode = 0;
// No parameters means run as a server
if (args.Length == 0)
{
Console.WriteLine("[{0} {1}]", DateTime.Now.ToLongDateString(), DateTime.Now.ToLongTimeString());
Console.WriteLine("RPCUtility no longer runs as a server. Use UnrealRemoteTool instead.");
}
else
{
Debug.WriteLine("Client starting up");
try
{
// Extract the command line parameters
if (args.Length < 2 ||
(args[1] == "RPCBatchUpload" && args.Length != 3) ||
(args[1] == "RPCSoakTest" && args.Length != 2) ||
((args[1] == "RPCUpload" || args[1] == "RPCDownload") && args.Length != 4))
{
throw new Exception("\n\nWrong number of arguments!\n" +
"Usage: RPCUtility.exe MachineName RemoteWorkingDirectory CommandName [CommandArg...]\n" +
" RPCUtility.exe MachineName RPCBatchUpload CommandFile\n" +
" RPCUtility.exe MachineName RPCUpload LocalFilePath RemoteFilePath\n" +
" RPCUtility.exe MachineName RPCDownload RemoteFilePath LocalFilePath\n" +
" RPCUtility.exe MachineName RPCSoakTest\n");
}
string MachineName = args[0];
// optional strings
string WorkingDirectory = "";
string CommandName = "";
string CommandArgs = "";
string SourceFilename = "";
string DestFilename = "";
if (args[1] == "RPCBatchUpload" || args[1] == "RPCGetFileInfo")
{
CommandName = args[1];
SourceFilename = args[2];
}
else if (args[1] == "RPCUpload" || args[1] == "RPCDownload")
{
CommandName = args[1];
SourceFilename = args[2];
DestFilename = args[3];
}
else if (args[1] == "RPCSoakTest")
{
CommandName = args[1];
}
else
{
WorkingDirectory = args[1];
CommandName = args[2];
if (args.Length > 3)
{
CommandArgs = String.Join(" ", args, 3, args.Length - 3);
}
Debug.WriteLine(" Machine name : " + MachineName);
Debug.WriteLine(" Working directory : " + WorkingDirectory);
Debug.WriteLine(" Command name : " + CommandName);
Debug.WriteLine(" Command args : " + CommandArgs);
}
// Make sure we can ping the remote machine before we continue
if (RPCCommandHelper.PingRemoteHost(MachineName))
{
Socket ClientSocket = RPCCommandHelper.ConnectToUnrealRemoteTool(MachineName);
// Hashtable Command = new Hashtable();
// Command["WorkingDirectory"] = WorkingDirectory;
// Command["CommandName"] = CommandName;
// Command["CommandArgs"] = CommandArgs;
//
// Hashtable Reply = SendMessageWithReply(ClientSocket, Command);
Random Generator = new Random();
Int32 RetriesRemaining = 2;
while (RetriesRemaining >= 0)
{
try
{
Hashtable CommandResult = null;
// Handle special copy command on the local side
if (CommandName == "RPCBatchUpload")
{
// one line per local/remote file pair
string[] FilePairs = File.ReadAllLines(SourceFilename);
RPCCommandHelper.RPCBatchUpload(ClientSocket, FilePairs);
}
else if (CommandName == "RPCUpload")
{
CommandResult = RPCCommandHelper.RPCUpload(ClientSocket, SourceFilename, DestFilename);
}
else if (CommandName == "RPCDownload")
{
CommandResult = RPCCommandHelper.RPCDownload(ClientSocket, SourceFilename, DestFilename);
}
else if (CommandName == "RPCGetFileInfo")
{
DateTime A;
long B;
RPCCommandHelper.GetFileInfo(ClientSocket, SourceFilename, DateTime.UtcNow, out A, out B);
Console.WriteLine("File info for {0}: Size = {1}, ModTime = {2}", SourceFilename, B, A);
}
else if (CommandName == "RPCSoakTest")
{
RPCCommandHelper.RPCSoakTest(ClientSocket);
}
else
{
CommandResult = RPCCommandHelper.RPCCommand(ClientSocket, WorkingDirectory, CommandName, CommandArgs, null);
}
if (CommandResult != null)
{
if (CommandResult["ExitCode"] != null)
{
ExitCode = (Int64)CommandResult["ExitCode"];
}
Console.WriteLine(CommandResult["CommandOutput"] as string);
}
break;
}
catch (Exception Ex3)
{
if (RetriesRemaining > 0)
{
// Generate a random retry timeout of 3-5 seconds
Int32 RetryTimeoutMS = 3000 + (Int32)(2000 * Generator.NextDouble());
Console.WriteLine("Retrying command after sleeping for " + RetryTimeoutMS + " milliseconds. Command is:" + CommandName + " " + CommandArgs);
Thread.Sleep(RetryTimeoutMS);
}
else
{
// We've tried enough times, just throw the error
throw new Exception("Deep Exception, retries exhausted... ", Ex3);
}
RetriesRemaining--;
}
}
}
else
{
Console.WriteLine("Failed to ping remote host: " + MachineName);
ExitCode = -3;
}
}
catch (Exception Ex)
{
Console.WriteLine("Outer Exception: " + Ex.ToString());
ExitCode = -1;
}
Debug.WriteLine("Client shutting down");
}
return((Int32)ExitCode);
}
static public RemoteToolChain.RemoteToolChainErrorCode Initialize(string InMacName, bool bFlushBuildDir)
{
MacName = InMacName;
// when not using RPCUtil, we do NOT want to ping the host
if (!RemoteToolChain.bUseRPCUtil || RPCCommandHelper.PingRemoteHost(MacName))
{
try
{
if (!RemoteToolChain.bUseRPCUtil)
{
// make sure we have SSH setup
if (RemoteToolChain.ResolvedSSHAuthentication == null)
{
Log.TraceError("SSH authentication required a key, but one was not found. Use Editor to setup remote authentication!");
return(RemoteToolChain.RemoteToolChainErrorCode.MissingSSHKey);
}
// ask for current time, free memory and num CPUs
string[] Commands = new string[]
{
"+\"%s\"",
"sysctl hw.memsize | awk '{print $2}'",
"sysctl hw.logicalcpu | awk '{print $2}'",
};
Hashtable Results = Command("/", "date", string.Join(" && ", Commands), null);
if ((Int64)Results["ExitCode"] != 0)
{
Log.TraceError("Failed to run init commands on {0}. Output = {1}", MacName, Results["CommandOutput"]);
return(RemoteToolChain.RemoteToolChainErrorCode.SSHCommandFailed);
}
string[] Lines = ((string)Results["CommandOutput"]).Split("\r\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
// convert it to a time, with TimeDifferenceFromRemote as 0
DateTime RemoteTimebase = RemoteToLocalTime(Lines[0]);
if (RemoteTimebase == DateTime.MinValue)
{
throw new BuildException("Failed to parse remote time on " + MacName);
}
// calculate the difference
TimeDifferenceFromRemote = DateTime.UtcNow - RemoteTimebase;
// now figure out max number of commands to run at once
// int PageSize = int.Parse(Lines[1]);
Int64 AvailableMem = Int64.Parse(Lines[1].Replace(".", "")); // *PageSize;
int NumProcesses = (int)Math.Max(1, AvailableMem / (RemoteToolChain.MemoryPerCompileMB * 1024 * 1024));
// now, combine that with actual number of cores
int NumCores = int.Parse(Lines[2]);
MaxRemoteCommandsAllowed = Math.Min(NumProcesses, NumCores);
Console.WriteLine("Remote time is {0}, difference is {1}", RemoteTimebase.ToString(), TimeDifferenceFromRemote.ToString());
}
if (bFlushBuildDir)
{
Command("/", "rm", "-rf /UE4/Builds/" + Environment.MachineName, null);
}
}
catch (Exception Ex)
{
Log.TraceVerbose("SSH Initialize exception {0}", Ex.ToString());
Log.TraceError("Failed to run init commands on {0}", MacName);
return(RemoteToolChain.RemoteToolChainErrorCode.SSHCommandFailed);
}
}
else
{
Log.TraceError("Failed to ping Mac named {0}", MacName);
return(RemoteToolChain.RemoteToolChainErrorCode.ServerNotResponding);
}
return(RemoteToolChain.RemoteToolChainErrorCode.NoError);
}
public static void BatchFileInfo(FileItem[] Files)
{
if (RemoteToolChain.bUseRPCUtil)
{
// build a list of file paths to get info about
StringBuilder FileList = new StringBuilder();
foreach (FileItem File in Files)
{
FileList.AppendFormat("{0}\n", File.AbsolutePath);
}
DateTime Now = DateTime.Now;
// execute the command!
Int64[] FileSizeAndDates = RPCCommandHelper.RPCBatchFileInfo(GetSocket(), FileList.ToString());
Console.WriteLine("BatchFileInfo version 1 took {0}", (DateTime.Now - Now).ToString());
// now update the source times
for (int Index = 0; Index < Files.Length; Index++)
{
Files[Index].Length = FileSizeAndDates[Index * 2 + 0];
Files[Index].LastWriteTime = new DateTimeOffset(RPCCommandHelper.FromRemoteTime(FileSizeAndDates[Index * 2 + 1]));
Files[Index].bExists = FileSizeAndDates[Index * 2 + 0] >= 0;
}
}
else
{
// build a list of file paths to get info about
StringBuilder Commands = new StringBuilder();
Commands.Append("#!/bin/bash\n");
foreach (FileItem File in Files)
{
Commands.AppendFormat("if [ -e \"{0}\" ]; then eval $(stat -s \"{0}\") && echo $st_mtime && echo $st_size; else echo 0 && echo -1; fi\n", File.AbsolutePath);
}
// write out locally
string LocalCommandsFile = Path.GetTempFileName();
System.IO.File.WriteAllText(LocalCommandsFile, Commands.ToString());
string RemoteDir = "/var/tmp/" + Environment.MachineName;
string RemoteCommandsFile = Path.GetFileName(LocalCommandsFile) + ".sh";
DateTime Now = DateTime.Now;
RemoteToolChain.UploadFile(LocalCommandsFile, RemoteDir + "/" + RemoteCommandsFile);
// execute the file, not a commandline
Hashtable Results = Command(RemoteDir, "sh", RemoteCommandsFile + " && rm " + RemoteCommandsFile, null);
Console.WriteLine("BatchFileInfo took {0}", (DateTime.Now - Now).ToString());
string[] Lines = ((string)Results["CommandOutput"]).Split("\r\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
if (Lines.Length != Files.Length * 2)
{
throw new BuildException("Received the wrong number of results from BatchFileInfo");
}
for (int Index = 0; Index < Files.Length; Index++)
{
Files[Index].LastWriteTime = new DateTimeOffset(RemoteToLocalTime(Lines[Index * 2 + 0]));
Files[Index].Length = long.Parse(Lines[Index * 2 + 1]);
Files[Index].bExists = Files[Index].Length >= 0;
}
}
}
public static void BatchUpload(string[] Commands)
{
// batch upload
RPCCommandHelper.RPCBatchUpload(GetSocket(), Commands);
}
