private void RunResavePackagesCommandlet(ProjectParams Params)
{
Log("Running Step:- ResavePackages::RunResavePackagesCommandlet");
// Find the commandlet binary
string UE4EditorExe = HostPlatform.Current.GetUE4ExePath(Params.UE4Exe);
if (!FileExists(UE4EditorExe))
{
LogError("Missing " + UE4EditorExe + " executable. Needs to be built first.");
throw new AutomationException("Missing " + UE4EditorExe + " executable. Needs to be built first.");
}
// Now let's rebuild lightmaps for the project
try
{
var CommandletParams = IsBuildMachine ? "-unattended -buildmachine -fileopenlog" : "-fileopenlog";
CommandletParams += " -AutoCheckOutPackages";
if (P4Enabled)
{
CommandletParams += String.Format(" -SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Changelist={4} -P4Passwd={5}", "Perforce", P4Env.ServerAndPort, P4Env.User, P4Env.Client, WorkingCL.ToString(), P4.GetAuthenticationToken());
}
ResavePackagesCommandlet(Params.RawProjectPath, Params.UE4Exe, Params.MapsToRebuildLightMaps.ToArray(), CommandletParams);
}
catch (Exception Ex)
{
string FinalLogLines = "No log file found";
CommandletException AEx = Ex as CommandletException;
if (AEx != null)
{
string LogFile = AEx.LogFileName;
UnrealBuildTool.Log.TraceWarning("Attempting to load file {0}", LogFile);
if (LogFile != "")
{
UnrealBuildTool.Log.TraceWarning("Attempting to read file {0}", LogFile);
try
{
string[] AllLogFile = ReadAllLines(LogFile);
FinalLogLines = "Important log entries\n";
foreach (string LogLine in AllLogFile)
{
if (LogLine.Contains("[REPORT]"))
{
FinalLogLines += LogLine + "\n";
}
}
}
catch (Exception)
{
// we don't care about this because if this is hit then there is no log file the exception probably has more info
LogError("Could not find log file " + LogFile);
}
}
}
// Something went wrong with the commandlet. Abandon this run, don't check in any updated files, etc.
LogError("Resave Packages has failed. because " + Ex.ToString());
throw new AutomationException(ExitCode.Error_Unknown, Ex, "ResavePackages failed. {0}", FinalLogLines);
}
}
public override void ExecuteBuild()
{
if (ParseParam("BuildEditor"))
{
UE4Build.BuildAgenda Agenda = new UE4Build.BuildAgenda();
Agenda.AddTarget("UE4Editor", HostPlatform.Current.HostEditorPlatform, UnrealTargetConfiguration.Development);
Agenda.AddTarget("ShaderCompileWorker", HostPlatform.Current.HostEditorPlatform, UnrealTargetConfiguration.Development);
UE4Build Builder = new UE4Build(this);
Builder.Build(Agenda, InDeleteBuildProducts: true, InUpdateVersionFiles: true, InForceNoXGE: true);
}
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.Branch + "/Engine/Config/...");
P4.Sync(P4Env.Branch + "/Engine/Content/...");
P4.Sync(P4Env.Branch + "/Engine/Source/...");
P4.Sync(P4Env.Branch + "/Portal/Config/...");
P4.Sync(P4Env.Branch + "/Portal/Content/...");
P4.Sync(P4Env.Branch + "/Portal/Source/...");
}
OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find("OneSkyConfig_EpicGames");
var oneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret);
// Export Launcher text from OneSky
{
var launcherGroup = GetLauncherGroup(oneSkyService);
var appProject = GetAppProject(oneSkyService);
var appFile = appProject.UploadedFiles.FirstOrDefault(f => f.Filename == "App.po");
//Export
if (appFile != null)
{
ExportFileToDirectory(appFile, new DirectoryInfo(CmdEnv.LocalRoot + "/Portal/Content/Localization/App"), launcherGroup.EnabledCultures);
}
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken());
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
// Setup commandlet arguments for SCC.
string CommandletSCCArguments = String.Empty;
if (P4Enabled)
{
CommandletSCCArguments += "-EnableSCC";
}
if (!AllowSubmit)
{
CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit";
}
// Setup commandlet arguments with configurations.
var CommandletArgumentSets = new string[]
{
String.Format("-config={0}", @"../Portal/Config/Localization/App.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments)
};
// Execute commandlet for each set of arguments.
foreach (var CommandletArguments in CommandletArgumentSets)
{
Log("Localization for {0} {1}", EditorArguments, CommandletArguments);
Log("Running UE4Editor to generate Localization data");
string Arguments = String.Format("-run=GatherText {0} {1}", EditorArguments, CommandletArguments);
var RunResult = Run(EditorExe, Arguments);
if (RunResult.ExitCode != 0)
{
throw new AutomationException("Error while executing localization commandlet '{0}'", Arguments);
}
}
// Upload Launcher text to OneSky
UploadDirectoryToProject(GetAppProject(oneSkyService), new DirectoryInfo(CmdEnv.LocalRoot + "/Portal/Content/Localization/App"), "*.po");
}
public override void ExecuteBuild()
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Config/...");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Content/...");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Source/...");
Log("Localize from label {0}", P4Env.LabelToSync);
}
OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find("OneSkyConfig_EpicGames");
var oneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret);
var projectGroup = GetProjectGroup(oneSkyService, "Unreal Engine");
// Create changelist for backed up POs from OneSky.
if (P4Enabled)
{
OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup.");
}
// Export all text from OneSky
ExportProjectToDirectory(oneSkyService, projectGroup, "Engine");
ExportProjectToDirectory(oneSkyService, projectGroup, "Editor");
ExportProjectToDirectory(oneSkyService, projectGroup, "EditorTutorials");
ExportProjectToDirectory(oneSkyService, projectGroup, "PropertyNames");
ExportProjectToDirectory(oneSkyService, projectGroup, "ToolTips");
ExportProjectToDirectory(oneSkyService, projectGroup, "Category");
ExportProjectToDirectory(oneSkyService, projectGroup, "Keywords");
// Submit changelist for backed up POs from OneSky.
if (P4Enabled)
{
int SubmittedChangeList;
P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList);
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken());
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
// Setup commandlet arguments for SCC.
string CommandletSCCArguments = String.Empty;
if (P4Enabled)
{
CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC";
}
if (!AllowSubmit)
{
CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit";
}
// Setup commandlet arguments with configurations.
var CommandletArgumentSets = new string[]
{
String.Format("-config={0}", @"./Config/Localization/Engine.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/Editor.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/EditorTutorials.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/PropertyNames.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/ToolTips.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/Category.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Config/Localization/Keywords.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
};
// Execute commandlet for each set of arguments.
foreach (var CommandletArguments in CommandletArgumentSets)
{
Log("Localization for {0} {1}", EditorArguments, CommandletArguments);
Log("Running UE4Editor to generate localization data");
string Arguments = String.Format("-run=GatherText {0} {1}", EditorArguments, CommandletArguments);
var RunResult = Run(EditorExe, Arguments);
if (RunResult.ExitCode != 0)
{
throw new AutomationException("Error while executing localization commandlet '{0}'", Arguments);
}
}
// Upload all text to OneSky
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Engine"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Engine"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Editor"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Editor"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "EditorTutorials"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/EditorTutorials"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "PropertyNames"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/PropertyNames"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "ToolTips"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/ToolTips"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Category"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Category"), "*.po");
UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Keywords"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Keywords"), "*.po");
// Localisation statistics estimator.
if (P4Enabled)
{
// Available only for P4
var EstimatorExePath = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/TranslatedWordsCountEstimator.exe");
var StatisticsFilePath = @"\\epicgames.net\root\UE3\Localization\WordCounts\udn.csv";
var Arguments = string.Format(
"{0} {1} {2} {3} {4}",
StatisticsFilePath,
P4Env.P4Port,
P4Env.User,
P4Env.Client,
Environment.GetEnvironmentVariable("P4PASSWD"));
var RunResult = Run(EstimatorExePath, Arguments);
if (RunResult.ExitCode != 0)
{
throw new AutomationException("Error while executing TranslatedWordsCountEstimator with arguments '{0}'", Arguments);
}
}
}
private void ProcessLocalizationProjects(LocalizationBatch LocalizationBatch, int PendingChangeList, string UEProjectRoot, string UEProjectName, string LocalizationProviderName, List <string> LocalizationSteps, string AdditionalCommandletArguments)
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
var RootWorkingDirectory = CombinePaths(UEProjectRoot, LocalizationBatch.UEProjectDirectory);
var RootLocalizationTargetDirectory = CombinePaths(UEProjectRoot, LocalizationBatch.LocalizationTargetDirectory);
// Try and find our localization provider
LocalizationProvider LocProvider = null;
{
LocalizationProvider.LocalizationProviderArgs LocProviderArgs;
LocProviderArgs.RootWorkingDirectory = RootWorkingDirectory;
LocProviderArgs.RootLocalizationTargetDirectory = RootLocalizationTargetDirectory;
LocProviderArgs.RemoteFilenamePrefix = LocalizationBatch.RemoteFilenamePrefix;
LocProviderArgs.Command = this;
LocProviderArgs.PendingChangeList = PendingChangeList;
LocProvider = LocalizationProvider.GetLocalizationProvider(LocalizationProviderName, LocProviderArgs);
}
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.BuildRootP4 + "/" + LocalizationBatch.LocalizationTargetDirectory + "/Config/Localization/...");
P4.Sync(P4Env.BuildRootP4 + "/" + LocalizationBatch.LocalizationTargetDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
var ProjectInfos = new List <ProjectInfo>();
foreach (var ProjectName in LocalizationBatch.LocalizationProjectNames)
{
ProjectInfos.Add(GenerateProjectInfo(RootLocalizationTargetDirectory, ProjectName, LocalizationSteps));
}
if (LocalizationSteps.Contains("Download") && LocProvider != null)
{
// Export all text from our localization provider
foreach (var ProjectInfo in ProjectInfos)
{
LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo);
}
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4} -P4Changelist={5} -EnableSCC -DisableSCCSubmit", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList);
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
EditorArguments += " -Unattended";
// Execute commandlet for each config in each project.
bool bLocCommandletFailed = false;
foreach (var ProjectInfo in ProjectInfos)
{
foreach (var LocalizationStep in ProjectInfo.LocalizationSteps)
{
if (!LocalizationSteps.Contains(LocalizationStep.Name))
{
continue;
}
var CommandletArguments = String.Format("-config=\"{0}\"", LocalizationStep.LocalizationConfigFile);
if (!String.IsNullOrEmpty(AdditionalCommandletArguments))
{
CommandletArguments += " " + AdditionalCommandletArguments;
}
string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments);
Log("Running localization commandlet: {0}", Arguments);
var StartTime = DateTime.UtcNow;
var RunResult = Run(EditorExe, Arguments, null, ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand); // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves below)
var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
Log("Localization commandlet finished in {0}s", RunDuration / 1000);
if (RunResult.ExitCode != 0)
{
LogWarning("The localization commandlet exited with code {0} which likely indicates a crash. It ran with the following arguments: '{1}'", RunResult.ExitCode, Arguments);
bLocCommandletFailed = true;
break; // We failed a step, so don't process any other steps in this config chain
}
}
}
if (LocalizationSteps.Contains("Upload") && LocProvider != null)
{
if (bLocCommandletFailed)
{
LogWarning("Skipping upload to the localization provider due to an earlier commandlet failure.");
}
else
{
// Upload all text to our localization provider
foreach (var ProjectInfo in ProjectInfos)
{
LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo);
}
}
}
}
public override void ExecuteBuild()
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Parse out the required command line arguments
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var OneSkyConfigName = ParseParamValue("OneSkyConfigName");
if (OneSkyConfigName == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyConfigName'");
}
var OneSkyProjectGroupName = ParseParamValue("OneSkyProjectGroupName");
if (OneSkyProjectGroupName == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyProjectGroupName'");
}
var OneSkyProjectNames = new List <string>();
{
var OneSkyProjectNamesStr = ParseParamValue("OneSkyProjectNames");
if (OneSkyProjectNamesStr == null)
{
throw new AutomationException("Missing required command line argument: 'OneSkyProjectNames'");
}
foreach (var ProjectName in OneSkyProjectNamesStr.Split(','))
{
OneSkyProjectNames.Add(ProjectName.Trim());
}
}
var OneSkyBranchSuffix = ParseParamValue("OneSkyBranchSuffix");
var RootWorkingDirectory = CombinePaths(CmdEnv.LocalRoot, UEProjectDirectory);
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Config/Localization/...");
P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
var ProjectInfos = new List <ProjectInfo>();
foreach (var ProjectName in OneSkyProjectNames)
{
ProjectInfos.Add(GenerateProjectInfo(RootWorkingDirectory, ProjectName));
}
OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find(OneSkyConfigName);
var OneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret);
var OneSkyProjectGroup = GetOneSkyProjectGroup(OneSkyService, OneSkyProjectGroupName);
// Create changelist for backed up POs from OneSky.
if (P4Enabled)
{
OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup.");
}
// Export all text from OneSky
foreach (var ProjectInfo in ProjectInfos)
{
ExportOneSkyProjectToDirectory(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo);
}
// Submit changelist for backed up POs from OneSky.
if (P4Enabled)
{
int SubmittedChangeList;
P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList);
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken());
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
// Setup commandlet arguments for SCC.
string CommandletSCCArguments = String.Empty;
if (P4Enabled)
{
CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC";
}
if (!AllowSubmit)
{
CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit";
}
// Execute commandlet for each config in each project.
foreach (var ProjectInfo in ProjectInfos)
{
foreach (var LocalizationConfigFile in ProjectInfo.LocalizationConfigFiles)
{
var CommandletArguments = String.Format("-config={0}", LocalizationConfigFile) + (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments);
Log("Localization for {0} {1}", EditorArguments, CommandletArguments);
Log("Running UE4Editor to generate localization data");
string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments);
var RunResult = Run(EditorExe, Arguments);
if (RunResult.ExitCode != 0)
{
Console.WriteLine("[ERROR] Error while executing localization commandlet '{0}'", Arguments);
}
}
}
// Upload all text to OneSky
foreach (var ProjectInfo in ProjectInfos)
{
UploadProjectToOneSky(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo);
}
}
public override void ExecuteBuild()
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Config/...");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Content/...");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Source/...");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Programs/NoRedist/UnrealEngineLauncher/Config/...");
P4.Sync(P4Env.BuildRootP4 + "/Engine/Programs/NoRedist/UnrealEngineLauncher/Content/...");
//P4.Sync(P4Env.BuildRootP4 + "/Engine/Source/..."); <- takes care of syncing Launcher source already
Log("Localize from label {0}", P4Env.LabelToSync);
}
OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find("OneSkyConfig_EpicGames");
var oneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret);
// Export Launcher text from OneSky
{
var launcherGroup = GetLauncherGroup(oneSkyService);
var appProject = GetAppProject(oneSkyService);
var appFile = appProject.UploadedFiles.FirstOrDefault(f => f.Filename == "App.po");
//Export
if (appFile != null)
{
ExportFileToDirectory(appFile, new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Programs/NoRedist/UnrealEngineLauncher/Content/Localization/App"), launcherGroup.EnabledCultures);
}
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken());
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
// Setup commandlet arguments for SCC.
string CommandletSCCArguments = String.Empty;
if (P4Enabled)
{
CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC";
}
if (!AllowSubmit)
{
CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit";
}
// Setup commandlet arguments with configurations.
var CommandletArgumentSets = new string[]
{
String.Format("-config={0}", @"./Programs/NoRedist/UnrealEngineLauncher/Config/Localization/App.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments),
String.Format("-config={0}", @"./Programs/NoRedist/UnrealEngineLauncher/Config/Localization/WordCount.ini"),
};
// Execute commandlet for each set of arguments.
foreach (var CommandletArguments in CommandletArgumentSets)
{
Log("Localization for {0} {1}", EditorArguments, CommandletArguments);
Log("Running UE4Editor to generate Localization data");
string Arguments = String.Format("-run=GatherText {0} {1}", EditorArguments, CommandletArguments);
var RunResult = Run(EditorExe, Arguments);
if (RunResult.ExitCode != 0)
{
throw new AutomationException("Error while executing localization commandlet '{0}'", Arguments);
}
}
// Upload Launcher text to OneSky
UploadDirectoryToProject(GetAppProject(oneSkyService), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Programs/NoRedist/UnrealEngineLauncher/Content/Localization/App"), "*.po");
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "";
}
var LocalizationStepNames = new List <string>();
{
var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepNamesStr == null)
{
LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepNamesStr.Split(','))
{
LocalizationStepNames.Add(StepName.Trim());
}
}
LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var ShouldGatherPlatforms = ParseParam("IncludePlatforms");
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var EnableParallelGather = ParseParam("ParallelGather");
var StartTime = DateTime.UtcNow;
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for platforms
if (ShouldGatherPlatforms)
{
var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms"));
if (DirectoryReference.Exists(PlatformsRootDirectory))
{
foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory))
{
// Find the localization targets defined for this platform
var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization")));
if (PlatformTargetNames.Count > 0)
{
var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames));
}
}
}
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory));
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
var AvailablePluginNames = new HashSet <string>();
foreach (var PluginInfo in AllPlugins)
{
AvailablePluginNames.Add(PluginInfo.Name);
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
// If we had an explicit list of plugins to include, warn if any were missing
foreach (string PluginName in IncludePlugins)
{
if (!AvailablePluginNames.Contains(PluginName))
{
LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName);
}
}
}
// Create a single changelist to use for all changes
int PendingChangeList = 0;
if (P4Enabled)
{
var ChangeListCommitMessage = String.Format("Localization Automation using CL {0}", P4Env.Changelist);
if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Restricted/NotForLicensees/Build/EpicInternal.txt")))
{
ChangeListCommitMessage += "\n#okforgithub ignore";
}
PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage);
}
// Prepare to process each localization batch
var LocalizationTasks = new List <LocalizationTask>();
foreach (var LocalizationBatch in LocalizationBatches)
{
var LocalizationTask = new LocalizationTask(LocalizationBatch, UEProjectRoot, LocalizationProviderName, PendingChangeList, this);
LocalizationTasks.Add(LocalizationTask);
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
LogInformation("Sync necessary content to head revision");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Config/Localization/...");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
foreach (var ProjectName in LocalizationTask.Batch.LocalizationProjectNames)
{
LocalizationTask.ProjectInfos.Add(GenerateProjectInfo(LocalizationTask.RootLocalizationTargetDirectory, ProjectName, LocalizationStepNames));
}
}
// Hash the current PO files on disk so we can work out whether they actually change
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Download the latest translations from our localization provider
if (LocalizationStepNames.Contains("Download"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
LocalizationTask.LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo);
}
}
}
}
// Begin the gather command for each task
// These can run in parallel when ParallelGather is enabled
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Set the common basic editor arguments
var EditorArguments = P4Enabled
? String.Format("-SCCProvider=Perforce -P4Port={0} -P4User={1} -P4Client={2} -P4Passwd={3} -P4Changelist={4} -EnableSCC -DisableSCCSubmit", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList)
: "-SCCProvider=None";
if (IsBuildMachine)
{
EditorArguments += " -BuildMachine";
}
EditorArguments += " -Unattended -LogLocalizationConflicts";
if (EnableParallelGather)
{
EditorArguments += " -multiprocess";
}
if (!String.IsNullOrEmpty(AdditionalCommandletArguments))
{
EditorArguments += " " + AdditionalCommandletArguments;
}
// Set the common process run options
var CommandletRunOptions = ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand; // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves later)
if (EnableParallelGather)
{
CommandletRunOptions |= ERunOptions.NoWaitForExit;
}
foreach (var LocalizationTask in LocalizationTasks)
{
var ProjectArgument = String.IsNullOrEmpty(UEProjectName) ? "" : String.Format("\"{0}\"", Path.Combine(LocalizationTask.RootWorkingDirectory, String.Format("{0}.uproject", UEProjectName)));
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
var LocalizationConfigFiles = new List <string>();
foreach (var LocalizationStep in ProjectInfo.LocalizationSteps)
{
if (LocalizationStepNames.Contains(LocalizationStep.Name))
{
LocalizationConfigFiles.Add(LocalizationStep.LocalizationConfigFile);
}
}
if (LocalizationConfigFiles.Count > 0)
{
var Arguments = String.Format("{0} -run=GatherText -config=\"{1}\" {2}", ProjectArgument, String.Join(";", LocalizationConfigFiles), EditorArguments);
LogInformation("Running localization commandlet for '{0}': {1}", ProjectInfo.ProjectName, Arguments);
LocalizationTask.GatherProcessResults.Add(Run(EditorExe, Arguments, null, CommandletRunOptions));
}
else
{
LocalizationTask.GatherProcessResults.Add(null);
}
}
}
}
// Wait for each commandlet process to finish and report the result.
// This runs even for non-parallel execution to log the exit state of the process.
foreach (var LocalizationTask in LocalizationTasks)
{
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null)
{
RunResult.WaitForExit();
RunResult.OnProcessExited();
RunResult.DisposeProcess();
if (RunResult.ExitCode == 0)
{
LogInformation("The localization commandlet for '{0}' exited with code 0.", ProjectInfo.ProjectName);
}
else
{
LogWarning("The localization commandlet for '{0}' exited with code {1} which likely indicates a crash.", ProjectInfo.ProjectName, RunResult.ExitCode);
}
}
}
}
// Upload the latest sources to our localization provider
if (LocalizationStepNames.Contains("Upload"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
// Upload all text to our localization provider
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null && RunResult.ExitCode == 0)
{
// Recalculate the split platform paths before doing the upload, as the export may have changed them
ProjectInfo.ExportInfo.CalculateSplitPlatformNames(LocalizationTask.RootLocalizationTargetDirectory);
LocalizationTask.LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo);
}
else
{
LogWarning("Skipping upload to the localization provider for '{0}' due to an earlier commandlet failure.", ProjectInfo.ProjectName);
}
}
}
}
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
LogInformation("Localize command finished in {0} seconds", RunDuration / 1000);
}
public override void ExecuteBuild()
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Parse out the required command line arguments
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr == null)
{
throw new AutomationException("Missing required command line argument: 'LocalizationProjectNames'");
}
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "OneSky";
}
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var RootWorkingDirectory = CombinePaths(CmdEnv.LocalRoot, UEProjectDirectory);
// Try and find our localization provider
LocalizationProvider LocProvider = null;
{
LocalizationProvider.LocalizationProviderArgs LocProviderArgs;
LocProviderArgs.RootWorkingDirectory = RootWorkingDirectory;
LocProviderArgs.CommandUtils = this;
LocProvider = LocalizationProvider.GetLocalizationProvider(LocalizationProviderName, LocProviderArgs);
}
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
Log("Sync necessary content to head revision");
P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Config/Localization/...");
P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
var ProjectInfos = new List <ProjectInfo>();
foreach (var ProjectName in LocalizationProjectNames)
{
ProjectInfos.Add(GenerateProjectInfo(RootWorkingDirectory, ProjectName));
}
// Export all text from our localization provider
foreach (var ProjectInfo in ProjectInfos)
{
LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo);
}
// Setup editor arguments for SCC.
string EditorArguments = String.Empty;
if (P4Enabled)
{
EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken());
}
else
{
EditorArguments = String.Format("-SCCProvider={0}", "None");
}
// Setup commandlet arguments for SCC.
string CommandletSCCArguments = String.Empty;
if (P4Enabled)
{
CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC";
}
if (!AllowSubmit)
{
CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit";
}
// Execute commandlet for each config in each project.
bool bLocCommandletFailed = false;
foreach (var ProjectInfo in ProjectInfos)
{
foreach (var LocalizationConfigFile in ProjectInfo.LocalizationConfigFiles)
{
var CommandletArguments = String.Format("-config={0}", LocalizationConfigFile) + (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments);
if (!String.IsNullOrEmpty(AdditionalCommandletArguments))
{
CommandletArguments += " " + AdditionalCommandletArguments;
}
string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments);
Log("Running localization commandlet: {0}", Arguments);
var StartTime = DateTime.UtcNow;
var RunResult = Run(EditorExe, Arguments, null, ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand); // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves below)
var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
Log("Localization commandlet finished in {0}s", RunDuration / 1000);
if (RunResult.ExitCode != 0)
{
LogWarning("The localization commandlet exited with code {0} which likely indicates a crash. It ran with the following arguments: '{1}'", RunResult.ExitCode, Arguments);
bLocCommandletFailed = true;
break; // We failed a step, so don't process any other steps in this config chain
}
}
}
if (bLocCommandletFailed)
{
LogWarning("Skipping upload to the localization provider due to an earlier commandlet failure.");
}
else
{
// Upload all text to our localization provider
foreach (var ProjectInfo in ProjectInfos)
{
LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo);
}
}
}
