private void RunResavePackagesCommandlet(ProjectParams Params) { Log("Running Step:- ResavePackages::RunResavePackagesCommandlet"); // Find the commandlet binary string UE4EditorExe = HostPlatform.Current.GetUE4ExePath(Params.UE4Exe); if (!FileExists(UE4EditorExe)) { LogError("Missing " + UE4EditorExe + " executable. Needs to be built first."); throw new AutomationException("Missing " + UE4EditorExe + " executable. Needs to be built first."); } // Now let's rebuild lightmaps for the project try { var CommandletParams = IsBuildMachine ? "-unattended -buildmachine -fileopenlog" : "-fileopenlog"; CommandletParams += " -AutoCheckOutPackages"; if (P4Enabled) { CommandletParams += String.Format(" -SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Changelist={4} -P4Passwd={5}", "Perforce", P4Env.ServerAndPort, P4Env.User, P4Env.Client, WorkingCL.ToString(), P4.GetAuthenticationToken()); } ResavePackagesCommandlet(Params.RawProjectPath, Params.UE4Exe, Params.MapsToRebuildLightMaps.ToArray(), CommandletParams); } catch (Exception Ex) { string FinalLogLines = "No log file found"; CommandletException AEx = Ex as CommandletException; if (AEx != null) { string LogFile = AEx.LogFileName; UnrealBuildTool.Log.TraceWarning("Attempting to load file {0}", LogFile); if (LogFile != "") { UnrealBuildTool.Log.TraceWarning("Attempting to read file {0}", LogFile); try { string[] AllLogFile = ReadAllLines(LogFile); FinalLogLines = "Important log entries\n"; foreach (string LogLine in AllLogFile) { if (LogLine.Contains("[REPORT]")) { FinalLogLines += LogLine + "\n"; } } } catch (Exception) { // we don't care about this because if this is hit then there is no log file the exception probably has more info LogError("Could not find log file " + LogFile); } } } // Something went wrong with the commandlet. Abandon this run, don't check in any updated files, etc. LogError("Resave Packages has failed. because " + Ex.ToString()); throw new AutomationException(ExitCode.Error_Unknown, Ex, "ResavePackages failed. {0}", FinalLogLines); } }
public override void ExecuteBuild() { if (ParseParam("BuildEditor")) { UE4Build.BuildAgenda Agenda = new UE4Build.BuildAgenda(); Agenda.AddTarget("UE4Editor", HostPlatform.Current.HostEditorPlatform, UnrealTargetConfiguration.Development); Agenda.AddTarget("ShaderCompileWorker", HostPlatform.Current.HostEditorPlatform, UnrealTargetConfiguration.Development); UE4Build Builder = new UE4Build(this); Builder.Build(Agenda, InDeleteBuildProducts: true, InUpdateVersionFiles: true, InForceNoXGE: true); } var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); if (P4Enabled) { Log("Sync necessary content to head revision"); P4.Sync(P4Env.Branch + "/Engine/Config/..."); P4.Sync(P4Env.Branch + "/Engine/Content/..."); P4.Sync(P4Env.Branch + "/Engine/Source/..."); P4.Sync(P4Env.Branch + "/Portal/Config/..."); P4.Sync(P4Env.Branch + "/Portal/Content/..."); P4.Sync(P4Env.Branch + "/Portal/Source/..."); } OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find("OneSkyConfig_EpicGames"); var oneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret); // Export Launcher text from OneSky { var launcherGroup = GetLauncherGroup(oneSkyService); var appProject = GetAppProject(oneSkyService); var appFile = appProject.UploadedFiles.FirstOrDefault(f => f.Filename == "App.po"); //Export if (appFile != null) { ExportFileToDirectory(appFile, new DirectoryInfo(CmdEnv.LocalRoot + "/Portal/Content/Localization/App"), launcherGroup.EnabledCultures); } } // Setup editor arguments for SCC. string EditorArguments = String.Empty; if (P4Enabled) { EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken()); } else { EditorArguments = String.Format("-SCCProvider={0}", "None"); } // Setup commandlet arguments for SCC. string CommandletSCCArguments = String.Empty; if (P4Enabled) { CommandletSCCArguments += "-EnableSCC"; } if (!AllowSubmit) { CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit"; } // Setup commandlet arguments with configurations. var CommandletArgumentSets = new string[] { String.Format("-config={0}", @"../Portal/Config/Localization/App.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments) }; // Execute commandlet for each set of arguments. foreach (var CommandletArguments in CommandletArgumentSets) { Log("Localization for {0} {1}", EditorArguments, CommandletArguments); Log("Running UE4Editor to generate Localization data"); string Arguments = String.Format("-run=GatherText {0} {1}", EditorArguments, CommandletArguments); var RunResult = Run(EditorExe, Arguments); if (RunResult.ExitCode != 0) { throw new AutomationException("Error while executing localization commandlet '{0}'", Arguments); } } // Upload Launcher text to OneSky UploadDirectoryToProject(GetAppProject(oneSkyService), new DirectoryInfo(CmdEnv.LocalRoot + "/Portal/Content/Localization/App"), "*.po"); }
public override void ExecuteBuild() { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); if (P4Enabled) { Log("Sync necessary content to head revision"); P4.Sync(P4Env.BuildRootP4 + "/Engine/Config/..."); P4.Sync(P4Env.BuildRootP4 + "/Engine/Content/..."); P4.Sync(P4Env.BuildRootP4 + "/Engine/Source/..."); Log("Localize from label {0}", P4Env.LabelToSync); } OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find("OneSkyConfig_EpicGames"); var oneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret); var projectGroup = GetProjectGroup(oneSkyService, "Unreal Engine"); // Create changelist for backed up POs from OneSky. if (P4Enabled) { OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup."); } // Export all text from OneSky ExportProjectToDirectory(oneSkyService, projectGroup, "Engine"); ExportProjectToDirectory(oneSkyService, projectGroup, "Editor"); ExportProjectToDirectory(oneSkyService, projectGroup, "EditorTutorials"); ExportProjectToDirectory(oneSkyService, projectGroup, "PropertyNames"); ExportProjectToDirectory(oneSkyService, projectGroup, "ToolTips"); ExportProjectToDirectory(oneSkyService, projectGroup, "Category"); ExportProjectToDirectory(oneSkyService, projectGroup, "Keywords"); // Submit changelist for backed up POs from OneSky. if (P4Enabled) { int SubmittedChangeList; P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList); } // Setup editor arguments for SCC. string EditorArguments = String.Empty; if (P4Enabled) { EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken()); } else { EditorArguments = String.Format("-SCCProvider={0}", "None"); } // Setup commandlet arguments for SCC. string CommandletSCCArguments = String.Empty; if (P4Enabled) { CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC"; } if (!AllowSubmit) { CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit"; } // Setup commandlet arguments with configurations. var CommandletArgumentSets = new string[] { String.Format("-config={0}", @"./Config/Localization/Engine.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/Editor.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/EditorTutorials.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/PropertyNames.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/ToolTips.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/Category.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Config/Localization/Keywords.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), }; // Execute commandlet for each set of arguments. foreach (var CommandletArguments in CommandletArgumentSets) { Log("Localization for {0} {1}", EditorArguments, CommandletArguments); Log("Running UE4Editor to generate localization data"); string Arguments = String.Format("-run=GatherText {0} {1}", EditorArguments, CommandletArguments); var RunResult = Run(EditorExe, Arguments); if (RunResult.ExitCode != 0) { throw new AutomationException("Error while executing localization commandlet '{0}'", Arguments); } } // Upload all text to OneSky UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Engine"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Engine"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Editor"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Editor"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "EditorTutorials"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/EditorTutorials"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "PropertyNames"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/PropertyNames"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "ToolTips"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/ToolTips"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Category"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Category"), "*.po"); UploadDirectoryToProject(GetProject(oneSkyService, "Unreal Engine", "Keywords"), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Content/Localization/Keywords"), "*.po"); // Localisation statistics estimator. if (P4Enabled) { // Available only for P4 var EstimatorExePath = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/DotNET/TranslatedWordsCountEstimator.exe"); var StatisticsFilePath = @"\\epicgames.net\root\UE3\Localization\WordCounts\udn.csv"; var Arguments = string.Format( "{0} {1} {2} {3} {4}", StatisticsFilePath, P4Env.P4Port, P4Env.User, P4Env.Client, Environment.GetEnvironmentVariable("P4PASSWD")); var RunResult = Run(EstimatorExePath, Arguments); if (RunResult.ExitCode != 0) { throw new AutomationException("Error while executing TranslatedWordsCountEstimator with arguments '{0}'", Arguments); } } }
private void ProcessLocalizationProjects(LocalizationBatch LocalizationBatch, int PendingChangeList, string UEProjectRoot, string UEProjectName, string LocalizationProviderName, List <string> LocalizationSteps, string AdditionalCommandletArguments) { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); var RootWorkingDirectory = CombinePaths(UEProjectRoot, LocalizationBatch.UEProjectDirectory); var RootLocalizationTargetDirectory = CombinePaths(UEProjectRoot, LocalizationBatch.LocalizationTargetDirectory); // Try and find our localization provider LocalizationProvider LocProvider = null; { LocalizationProvider.LocalizationProviderArgs LocProviderArgs; LocProviderArgs.RootWorkingDirectory = RootWorkingDirectory; LocProviderArgs.RootLocalizationTargetDirectory = RootLocalizationTargetDirectory; LocProviderArgs.RemoteFilenamePrefix = LocalizationBatch.RemoteFilenamePrefix; LocProviderArgs.Command = this; LocProviderArgs.PendingChangeList = PendingChangeList; LocProvider = LocalizationProvider.GetLocalizationProvider(LocalizationProviderName, LocProviderArgs); } // Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on if (P4Enabled) { Log("Sync necessary content to head revision"); P4.Sync(P4Env.BuildRootP4 + "/" + LocalizationBatch.LocalizationTargetDirectory + "/Config/Localization/..."); P4.Sync(P4Env.BuildRootP4 + "/" + LocalizationBatch.LocalizationTargetDirectory + "/Content/Localization/..."); } // Generate the info we need to gather for each project var ProjectInfos = new List <ProjectInfo>(); foreach (var ProjectName in LocalizationBatch.LocalizationProjectNames) { ProjectInfos.Add(GenerateProjectInfo(RootLocalizationTargetDirectory, ProjectName, LocalizationSteps)); } if (LocalizationSteps.Contains("Download") && LocProvider != null) { // Export all text from our localization provider foreach (var ProjectInfo in ProjectInfos) { LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo); } } // Setup editor arguments for SCC. string EditorArguments = String.Empty; if (P4Enabled) { EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4} -P4Changelist={5} -EnableSCC -DisableSCCSubmit", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList); } else { EditorArguments = String.Format("-SCCProvider={0}", "None"); } EditorArguments += " -Unattended"; // Execute commandlet for each config in each project. bool bLocCommandletFailed = false; foreach (var ProjectInfo in ProjectInfos) { foreach (var LocalizationStep in ProjectInfo.LocalizationSteps) { if (!LocalizationSteps.Contains(LocalizationStep.Name)) { continue; } var CommandletArguments = String.Format("-config=\"{0}\"", LocalizationStep.LocalizationConfigFile); if (!String.IsNullOrEmpty(AdditionalCommandletArguments)) { CommandletArguments += " " + AdditionalCommandletArguments; } string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments); Log("Running localization commandlet: {0}", Arguments); var StartTime = DateTime.UtcNow; var RunResult = Run(EditorExe, Arguments, null, ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand); // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves below) var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds; Log("Localization commandlet finished in {0}s", RunDuration / 1000); if (RunResult.ExitCode != 0) { LogWarning("The localization commandlet exited with code {0} which likely indicates a crash. It ran with the following arguments: '{1}'", RunResult.ExitCode, Arguments); bLocCommandletFailed = true; break; // We failed a step, so don't process any other steps in this config chain } } } if (LocalizationSteps.Contains("Upload") && LocProvider != null) { if (bLocCommandletFailed) { LogWarning("Skipping upload to the localization provider due to an earlier commandlet failure."); } else { // Upload all text to our localization provider foreach (var ProjectInfo in ProjectInfos) { LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo); } } } }
public override void ExecuteBuild() { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); // Parse out the required command line arguments var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var OneSkyConfigName = ParseParamValue("OneSkyConfigName"); if (OneSkyConfigName == null) { throw new AutomationException("Missing required command line argument: 'OneSkyConfigName'"); } var OneSkyProjectGroupName = ParseParamValue("OneSkyProjectGroupName"); if (OneSkyProjectGroupName == null) { throw new AutomationException("Missing required command line argument: 'OneSkyProjectGroupName'"); } var OneSkyProjectNames = new List <string>(); { var OneSkyProjectNamesStr = ParseParamValue("OneSkyProjectNames"); if (OneSkyProjectNamesStr == null) { throw new AutomationException("Missing required command line argument: 'OneSkyProjectNames'"); } foreach (var ProjectName in OneSkyProjectNamesStr.Split(',')) { OneSkyProjectNames.Add(ProjectName.Trim()); } } var OneSkyBranchSuffix = ParseParamValue("OneSkyBranchSuffix"); var RootWorkingDirectory = CombinePaths(CmdEnv.LocalRoot, UEProjectDirectory); // Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on if (P4Enabled) { Log("Sync necessary content to head revision"); P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Config/Localization/..."); P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Content/Localization/..."); } // Generate the info we need to gather for each project var ProjectInfos = new List <ProjectInfo>(); foreach (var ProjectName in OneSkyProjectNames) { ProjectInfos.Add(GenerateProjectInfo(RootWorkingDirectory, ProjectName)); } OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find(OneSkyConfigName); var OneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret); var OneSkyProjectGroup = GetOneSkyProjectGroup(OneSkyService, OneSkyProjectGroupName); // Create changelist for backed up POs from OneSky. if (P4Enabled) { OneSkyDownloadedPOChangeList = P4.CreateChange(P4Env.Client, "OneSky downloaded PO backup."); } // Export all text from OneSky foreach (var ProjectInfo in ProjectInfos) { ExportOneSkyProjectToDirectory(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo); } // Submit changelist for backed up POs from OneSky. if (P4Enabled) { int SubmittedChangeList; P4.Submit(OneSkyDownloadedPOChangeList, out SubmittedChangeList); } // Setup editor arguments for SCC. string EditorArguments = String.Empty; if (P4Enabled) { EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken()); } else { EditorArguments = String.Format("-SCCProvider={0}", "None"); } // Setup commandlet arguments for SCC. string CommandletSCCArguments = String.Empty; if (P4Enabled) { CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC"; } if (!AllowSubmit) { CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit"; } // Execute commandlet for each config in each project. foreach (var ProjectInfo in ProjectInfos) { foreach (var LocalizationConfigFile in ProjectInfo.LocalizationConfigFiles) { var CommandletArguments = String.Format("-config={0}", LocalizationConfigFile) + (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments); Log("Localization for {0} {1}", EditorArguments, CommandletArguments); Log("Running UE4Editor to generate localization data"); string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments); var RunResult = Run(EditorExe, Arguments); if (RunResult.ExitCode != 0) { Console.WriteLine("[ERROR] Error while executing localization commandlet '{0}'", Arguments); } } } // Upload all text to OneSky foreach (var ProjectInfo in ProjectInfos) { UploadProjectToOneSky(RootWorkingDirectory, OneSkyService, OneSkyProjectGroup, OneSkyBranchSuffix, ProjectInfo); } }
public override void ExecuteBuild() { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); if (P4Enabled) { Log("Sync necessary content to head revision"); P4.Sync(P4Env.BuildRootP4 + "/Engine/Config/..."); P4.Sync(P4Env.BuildRootP4 + "/Engine/Content/..."); P4.Sync(P4Env.BuildRootP4 + "/Engine/Source/..."); P4.Sync(P4Env.BuildRootP4 + "/Engine/Programs/NoRedist/UnrealEngineLauncher/Config/..."); P4.Sync(P4Env.BuildRootP4 + "/Engine/Programs/NoRedist/UnrealEngineLauncher/Content/..."); //P4.Sync(P4Env.BuildRootP4 + "/Engine/Source/..."); <- takes care of syncing Launcher source already Log("Localize from label {0}", P4Env.LabelToSync); } OneSkyConfigData OneSkyConfig = OneSkyConfigHelper.Find("OneSkyConfig_EpicGames"); var oneSkyService = new OneSkyService(OneSkyConfig.ApiKey, OneSkyConfig.ApiSecret); // Export Launcher text from OneSky { var launcherGroup = GetLauncherGroup(oneSkyService); var appProject = GetAppProject(oneSkyService); var appFile = appProject.UploadedFiles.FirstOrDefault(f => f.Filename == "App.po"); //Export if (appFile != null) { ExportFileToDirectory(appFile, new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Programs/NoRedist/UnrealEngineLauncher/Content/Localization/App"), launcherGroup.EnabledCultures); } } // Setup editor arguments for SCC. string EditorArguments = String.Empty; if (P4Enabled) { EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken()); } else { EditorArguments = String.Format("-SCCProvider={0}", "None"); } // Setup commandlet arguments for SCC. string CommandletSCCArguments = String.Empty; if (P4Enabled) { CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC"; } if (!AllowSubmit) { CommandletSCCArguments += (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit"; } // Setup commandlet arguments with configurations. var CommandletArgumentSets = new string[] { String.Format("-config={0}", @"./Programs/NoRedist/UnrealEngineLauncher/Config/Localization/App.ini") + (string.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments), String.Format("-config={0}", @"./Programs/NoRedist/UnrealEngineLauncher/Config/Localization/WordCount.ini"), }; // Execute commandlet for each set of arguments. foreach (var CommandletArguments in CommandletArgumentSets) { Log("Localization for {0} {1}", EditorArguments, CommandletArguments); Log("Running UE4Editor to generate Localization data"); string Arguments = String.Format("-run=GatherText {0} {1}", EditorArguments, CommandletArguments); var RunResult = Run(EditorExe, Arguments); if (RunResult.ExitCode != 0) { throw new AutomationException("Error while executing localization commandlet '{0}'", Arguments); } } // Upload Launcher text to OneSky UploadDirectoryToProject(GetAppProject(oneSkyService), new DirectoryInfo(CmdEnv.LocalRoot + "/Engine/Programs/NoRedist/UnrealEngineLauncher/Content/Localization/App"), "*.po"); }
public override void ExecuteBuild() { var UEProjectRoot = ParseParamValue("UEProjectRoot"); if (UEProjectRoot == null) { UEProjectRoot = CmdEnv.LocalRoot; } var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var LocalizationProjectNames = new List <string>(); { var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames"); if (LocalizationProjectNamesStr != null) { foreach (var ProjectName in LocalizationProjectNamesStr.Split(',')) { LocalizationProjectNames.Add(ProjectName.Trim()); } } } var LocalizationProviderName = ParseParamValue("LocalizationProvider"); if (LocalizationProviderName == null) { LocalizationProviderName = ""; } var LocalizationStepNames = new List <string>(); { var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps"); if (LocalizationStepNamesStr == null) { LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" }); } else { foreach (var StepName in LocalizationStepNamesStr.Split(',')) { LocalizationStepNames.Add(StepName.Trim()); } } LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do } var ShouldGatherPlugins = ParseParam("IncludePlugins"); var IncludePlugins = new List <string>(); var ExcludePlugins = new List <string>(); if (ShouldGatherPlugins) { var IncludePluginsStr = ParseParamValue("IncludePlugins"); if (IncludePluginsStr != null) { foreach (var PluginName in IncludePluginsStr.Split(',')) { IncludePlugins.Add(PluginName.Trim()); } } var ExcludePluginsStr = ParseParamValue("ExcludePlugins"); if (ExcludePluginsStr != null) { foreach (var PluginName in ExcludePluginsStr.Split(',')) { ExcludePlugins.Add(PluginName.Trim()); } } } var ShouldGatherPlatforms = ParseParam("IncludePlatforms"); var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments"); if (AdditionalCommandletArguments == null) { AdditionalCommandletArguments = ""; } var EnableParallelGather = ParseParam("ParallelGather"); var StartTime = DateTime.UtcNow; var LocalizationBatches = new List <LocalizationBatch>(); // Add the static set of localization projects as a batch if (LocalizationProjectNames.Count > 0) { LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames)); } // Build up any additional batches needed for platforms if (ShouldGatherPlatforms) { var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms")); if (DirectoryReference.Exists(PlatformsRootDirectory)) { foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory)) { // Find the localization targets defined for this platform var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization"))); if (PlatformTargetNames.Count > 0) { var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames)); } } } } // Build up any additional batches needed for plugins if (ShouldGatherPlugins) { var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory)); IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project); // Add a batch for each plugin that meets our criteria var AvailablePluginNames = new HashSet <string>(); foreach (var PluginInfo in AllPlugins) { AvailablePluginNames.Add(PluginInfo.Name); bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name); if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0) { var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot)); RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4 var PluginTargetNames = new List <string>(); foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets) { PluginTargetNames.Add(LocalizationTarget.Name); } LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames)); } } // If we had an explicit list of plugins to include, warn if any were missing foreach (string PluginName in IncludePlugins) { if (!AvailablePluginNames.Contains(PluginName)) { LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName); } } } // Create a single changelist to use for all changes int PendingChangeList = 0; if (P4Enabled) { var ChangeListCommitMessage = String.Format("Localization Automation using CL {0}", P4Env.Changelist); if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Restricted/NotForLicensees/Build/EpicInternal.txt"))) { ChangeListCommitMessage += "\n#okforgithub ignore"; } PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage); } // Prepare to process each localization batch var LocalizationTasks = new List <LocalizationTask>(); foreach (var LocalizationBatch in LocalizationBatches) { var LocalizationTask = new LocalizationTask(LocalizationBatch, UEProjectRoot, LocalizationProviderName, PendingChangeList, this); LocalizationTasks.Add(LocalizationTask); // Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on if (P4Enabled) { LogInformation("Sync necessary content to head revision"); P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Config/Localization/..."); P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Content/Localization/..."); } // Generate the info we need to gather for each project foreach (var ProjectName in LocalizationTask.Batch.LocalizationProjectNames) { LocalizationTask.ProjectInfos.Add(GenerateProjectInfo(LocalizationTask.RootLocalizationTargetDirectory, ProjectName, LocalizationStepNames)); } } // Hash the current PO files on disk so we can work out whether they actually change Dictionary <string, byte[]> InitalPOFileHashes = null; if (P4Enabled) { InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); } // Download the latest translations from our localization provider if (LocalizationStepNames.Contains("Download")) { foreach (var LocalizationTask in LocalizationTasks) { if (LocalizationTask.LocProvider != null) { foreach (var ProjectInfo in LocalizationTask.ProjectInfos) { LocalizationTask.LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo); } } } } // Begin the gather command for each task // These can run in parallel when ParallelGather is enabled { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); // Set the common basic editor arguments var EditorArguments = P4Enabled ? String.Format("-SCCProvider=Perforce -P4Port={0} -P4User={1} -P4Client={2} -P4Passwd={3} -P4Changelist={4} -EnableSCC -DisableSCCSubmit", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList) : "-SCCProvider=None"; if (IsBuildMachine) { EditorArguments += " -BuildMachine"; } EditorArguments += " -Unattended -LogLocalizationConflicts"; if (EnableParallelGather) { EditorArguments += " -multiprocess"; } if (!String.IsNullOrEmpty(AdditionalCommandletArguments)) { EditorArguments += " " + AdditionalCommandletArguments; } // Set the common process run options var CommandletRunOptions = ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand; // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves later) if (EnableParallelGather) { CommandletRunOptions |= ERunOptions.NoWaitForExit; } foreach (var LocalizationTask in LocalizationTasks) { var ProjectArgument = String.IsNullOrEmpty(UEProjectName) ? "" : String.Format("\"{0}\"", Path.Combine(LocalizationTask.RootWorkingDirectory, String.Format("{0}.uproject", UEProjectName))); foreach (var ProjectInfo in LocalizationTask.ProjectInfos) { var LocalizationConfigFiles = new List <string>(); foreach (var LocalizationStep in ProjectInfo.LocalizationSteps) { if (LocalizationStepNames.Contains(LocalizationStep.Name)) { LocalizationConfigFiles.Add(LocalizationStep.LocalizationConfigFile); } } if (LocalizationConfigFiles.Count > 0) { var Arguments = String.Format("{0} -run=GatherText -config=\"{1}\" {2}", ProjectArgument, String.Join(";", LocalizationConfigFiles), EditorArguments); LogInformation("Running localization commandlet for '{0}': {1}", ProjectInfo.ProjectName, Arguments); LocalizationTask.GatherProcessResults.Add(Run(EditorExe, Arguments, null, CommandletRunOptions)); } else { LocalizationTask.GatherProcessResults.Add(null); } } } } // Wait for each commandlet process to finish and report the result. // This runs even for non-parallel execution to log the exit state of the process. foreach (var LocalizationTask in LocalizationTasks) { for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex) { var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex]; var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex]; if (RunResult != null) { RunResult.WaitForExit(); RunResult.OnProcessExited(); RunResult.DisposeProcess(); if (RunResult.ExitCode == 0) { LogInformation("The localization commandlet for '{0}' exited with code 0.", ProjectInfo.ProjectName); } else { LogWarning("The localization commandlet for '{0}' exited with code {1} which likely indicates a crash.", ProjectInfo.ProjectName, RunResult.ExitCode); } } } } // Upload the latest sources to our localization provider if (LocalizationStepNames.Contains("Upload")) { foreach (var LocalizationTask in LocalizationTasks) { if (LocalizationTask.LocProvider != null) { // Upload all text to our localization provider for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex) { var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex]; var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex]; if (RunResult != null && RunResult.ExitCode == 0) { // Recalculate the split platform paths before doing the upload, as the export may have changed them ProjectInfo.ExportInfo.CalculateSplitPlatformNames(LocalizationTask.RootLocalizationTargetDirectory); LocalizationTask.LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo); } else { LogWarning("Skipping upload to the localization provider for '{0}' due to an earlier commandlet failure.", ProjectInfo.ProjectName); } } } } } // Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to) if (P4Enabled) { // Revert any PO files that haven't changed aside from their header { var POFilesToRevert = new List <string>(); var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot); foreach (var CurrentPOFileHashPair in CurrentPOFileHashes) { byte[] InitialPOFileHash; if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value)) { POFilesToRevert.Add(CurrentPOFileHashPair.Key); } } if (POFilesToRevert.Count > 0) { var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString())); using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename)) { foreach (var POFileToRevert in POFilesToRevert) { P4RevertArgsWriter.WriteLine(POFileToRevert); } } P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename)); DeleteFile_NoExceptions(P4RevertArgsFilename); } } // Revert any other unchanged files P4.RevertUnchanged(PendingChangeList); // Submit that single changelist now if (AllowSubmit) { int SubmittedChangeList; P4.Submit(PendingChangeList, out SubmittedChangeList); } } var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds; LogInformation("Localize command finished in {0} seconds", RunDuration / 1000); }
public override void ExecuteBuild() { var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe"); // Parse out the required command line arguments var UEProjectDirectory = ParseParamValue("UEProjectDirectory"); if (UEProjectDirectory == null) { throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'"); } var UEProjectName = ParseParamValue("UEProjectName"); if (UEProjectName == null) { UEProjectName = ""; } var LocalizationProjectNames = new List <string>(); { var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames"); if (LocalizationProjectNamesStr == null) { throw new AutomationException("Missing required command line argument: 'LocalizationProjectNames'"); } foreach (var ProjectName in LocalizationProjectNamesStr.Split(',')) { LocalizationProjectNames.Add(ProjectName.Trim()); } } var LocalizationProviderName = ParseParamValue("LocalizationProvider"); if (LocalizationProviderName == null) { LocalizationProviderName = "OneSky"; } var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments"); if (AdditionalCommandletArguments == null) { AdditionalCommandletArguments = ""; } var RootWorkingDirectory = CombinePaths(CmdEnv.LocalRoot, UEProjectDirectory); // Try and find our localization provider LocalizationProvider LocProvider = null; { LocalizationProvider.LocalizationProviderArgs LocProviderArgs; LocProviderArgs.RootWorkingDirectory = RootWorkingDirectory; LocProviderArgs.CommandUtils = this; LocProvider = LocalizationProvider.GetLocalizationProvider(LocalizationProviderName, LocProviderArgs); } // Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on if (P4Enabled) { Log("Sync necessary content to head revision"); P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Config/Localization/..."); P4.Sync(P4Env.BuildRootP4 + "/" + UEProjectDirectory + "/Content/Localization/..."); } // Generate the info we need to gather for each project var ProjectInfos = new List <ProjectInfo>(); foreach (var ProjectName in LocalizationProjectNames) { ProjectInfos.Add(GenerateProjectInfo(RootWorkingDirectory, ProjectName)); } // Export all text from our localization provider foreach (var ProjectInfo in ProjectInfos) { LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo); } // Setup editor arguments for SCC. string EditorArguments = String.Empty; if (P4Enabled) { EditorArguments = String.Format("-SCCProvider={0} -P4Port={1} -P4User={2} -P4Client={3} -P4Passwd={4}", "Perforce", P4Env.P4Port, P4Env.User, P4Env.Client, P4.GetAuthenticationToken()); } else { EditorArguments = String.Format("-SCCProvider={0}", "None"); } // Setup commandlet arguments for SCC. string CommandletSCCArguments = String.Empty; if (P4Enabled) { CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-EnableSCC"; } if (!AllowSubmit) { CommandletSCCArguments += (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " ") + "-DisableSCCSubmit"; } // Execute commandlet for each config in each project. bool bLocCommandletFailed = false; foreach (var ProjectInfo in ProjectInfos) { foreach (var LocalizationConfigFile in ProjectInfo.LocalizationConfigFiles) { var CommandletArguments = String.Format("-config={0}", LocalizationConfigFile) + (String.IsNullOrEmpty(CommandletSCCArguments) ? "" : " " + CommandletSCCArguments); if (!String.IsNullOrEmpty(AdditionalCommandletArguments)) { CommandletArguments += " " + AdditionalCommandletArguments; } string Arguments = String.Format("{0} -run=GatherText {1} {2}", UEProjectName, EditorArguments, CommandletArguments); Log("Running localization commandlet: {0}", Arguments); var StartTime = DateTime.UtcNow; var RunResult = Run(EditorExe, Arguments, null, ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand); // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves below) var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds; Log("Localization commandlet finished in {0}s", RunDuration / 1000); if (RunResult.ExitCode != 0) { LogWarning("The localization commandlet exited with code {0} which likely indicates a crash. It ran with the following arguments: '{1}'", RunResult.ExitCode, Arguments); bLocCommandletFailed = true; break; // We failed a step, so don't process any other steps in this config chain } } } if (bLocCommandletFailed) { LogWarning("Skipping upload to the localization provider due to an earlier commandlet failure."); } else { // Upload all text to our localization provider foreach (var ProjectInfo in ProjectInfos) { LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo); } } }