public void StageAppLocalDependencies(ProjectParams Params, DeploymentContext SC, string PlatformDir)
{
Dictionary <string, string> PathVariables = new Dictionary <string, string>();
PathVariables["EngineDir"] = SC.EngineRoot.FullName;
PathVariables["ProjectDir"] = SC.ProjectRoot.FullName;
string ExpandedAppLocalDir = Utils.ExpandVariables(Params.AppLocalDirectory, PathVariables);
DirectoryReference BaseAppLocalDependenciesPath = Path.IsPathRooted(ExpandedAppLocalDir) ? new DirectoryReference(CombinePaths(ExpandedAppLocalDir, PlatformDir)) : DirectoryReference.Combine(SC.ProjectRoot, ExpandedAppLocalDir, PlatformDir);
if (DirectoryReference.Exists(BaseAppLocalDependenciesPath))
{
StagedDirectoryReference ProjectBinaryPath = new StagedDirectoryReference(SC.ProjectBinariesFolder.MakeRelativeTo(SC.ProjectRoot.ParentDirectory));
StagedDirectoryReference EngineBinaryPath = new StagedDirectoryReference(CombinePaths("Engine", "Binaries", PlatformDir));
Log("Copying AppLocal dependencies from {0} to {1} and {2}", BaseAppLocalDependenciesPath, ProjectBinaryPath, EngineBinaryPath);
foreach (DirectoryReference DependencyDirectory in DirectoryReference.EnumerateDirectories(BaseAppLocalDependenciesPath))
{
SC.StageFiles(StagedFileType.NonUFS, DependencyDirectory, "*", false, null, ProjectBinaryPath);
SC.StageFiles(StagedFileType.NonUFS, DependencyDirectory, "*", false, null, EngineBinaryPath);
}
}
else
{
throw new AutomationException("Unable to deploy AppLocalDirectory dependencies. No such path: {0}", BaseAppLocalDependenciesPath);
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the list of symbol file name patterns from the platform.
Platform TargetPlatform = Platform.GetPlatform(Parameters.Platform);
string[] DirectoryStructure = TargetPlatform.SymbolServerDirectoryStructure;
if (DirectoryStructure == null)
{
throw new AutomationException("Platform does not specify the symbol server structure. Cannot age the symbol server.");
}
string Filter = string.IsNullOrWhiteSpace(Parameters.Filter)
? string.Empty
: Parameters.Filter.Trim();
// Eumerate the root directory of builds for buildversions to check against
// Folder names in the root directory should match the name of the .version files
HashSet <string> ExistingBuilds = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
if (!string.IsNullOrWhiteSpace(Parameters.BuildDir))
{
DirectoryReference BuildDir = new DirectoryReference(Parameters.BuildDir);
if (DirectoryReference.Exists(BuildDir))
{
foreach (string BuildName in DirectoryReference.EnumerateDirectories(BuildDir).Select(Build => Build.GetDirectoryName()))
{
ExistingBuilds.Add(BuildName);
}
}
else
{
CommandUtils.LogWarning("BuildDir of {0} was provided but it doesn't exist! Will not check buildversions against it.", Parameters.BuildDir);
}
}
// Get the time at which to expire files
DateTime ExpireTimeUtc = DateTime.UtcNow - TimeSpan.FromDays(Parameters.Days);
CommandUtils.LogInformation("Expiring all files before {0}...", ExpireTimeUtc);
// Scan the store directory and delete old symbol files
DirectoryReference SymbolServerDirectory = ResolveDirectory(Parameters.StoreDir);
CommandUtils.OptionallyTakeLock(TargetPlatform.SymbolServerRequiresLock, SymbolServerDirectory, TimeSpan.FromMinutes(15), () =>
{
RecurseDirectory(ExpireTimeUtc, new DirectoryInfo(SymbolServerDirectory.FullName), DirectoryStructure, 0, Filter, ExistingBuilds, TargetPlatform.SymbolServerDeleteIndividualFiles);
});
}
public static void FindProjects(DirectoryReference BaseDir, List <FileReference> ProjectFiles)
{
if (BaseDir != CommandUtils.EngineDirectory)
{
int InitialProjectFileCount = ProjectFiles.Count;
ProjectFiles.AddRange(DirectoryReference.EnumerateFiles(BaseDir, "*.uproject"));
if (InitialProjectFileCount == ProjectFiles.Count)
{
foreach (DirectoryReference SubDir in DirectoryReference.EnumerateDirectories(BaseDir))
{
FindProjects(SubDir, ProjectFiles);
}
}
}
}
static void StageAppLocalDependenciesToDir(DeploymentContext SC, DirectoryReference BaseAppLocalDependenciesPath, StagedDirectoryReference StagedBinariesDir)
{
// Check if there are any executables being staged in this directory. Usually we only need to stage runtime dependencies next to the executable, but we may be staging
// other engine executables too (eg. CEF)
List <StagedFileReference> FilesInTargetDir = SC.FilesToStage.NonUFSFiles.Keys.Where(x => x.IsUnderDirectory(StagedBinariesDir) && (x.HasExtension(".exe") || x.HasExtension(".dll"))).ToList();
if (FilesInTargetDir.Count > 0)
{
LogInformation("Copying AppLocal dependencies from {0} to {1}", BaseAppLocalDependenciesPath, StagedBinariesDir);
// Stage files in subdirs
foreach (DirectoryReference DependencyDirectory in DirectoryReference.EnumerateDirectories(BaseAppLocalDependenciesPath))
{
SC.StageFiles(StagedFileType.NonUFS, DependencyDirectory, StageFilesSearch.AllDirectories, StagedBinariesDir);
}
}
}
/// <summary>
/// Finds files to stage under a given base directory.
/// </summary>
/// <param name="BaseDir">The directory to search under</param>
/// <param name="Pattern">Pattern for files to match</param>
/// <param name="ExcludePatterns">Patterns to exclude from staging</param>
/// <param name="Option">Options for the search</param>
/// <param name="Files">List to receive the enumerated files</param>
private void FindFilesToStageInternal(DirectoryReference BaseDir, string Pattern, StageFilesSearch Option, List <FileReference> Files)
{
// Enumerate all the files in this directory
Files.AddRange(DirectoryReference.EnumerateFiles(BaseDir, Pattern));
// Recurse through subdirectories if necessary
if (Option == StageFilesSearch.AllDirectories)
{
foreach (DirectoryReference SubDir in DirectoryReference.EnumerateDirectories(BaseDir))
{
FileSystemName Name = new FileSystemName(SubDir);
if (!RestrictedFolderNames.Contains(Name))
{
FindFilesToStageInternal(SubDir, Pattern, Option, Files);
}
}
}
}
public void CalculateSplitPlatformNames(string RootWorkingDirectory)
{
// Is this isn't a single culture import/export, then also check for split platform sub-folders
var NewSplitPlatformNames = new List <string>();
if (bUseCultureDirectory)
{
var PlatformSourceDirectory = new DirectoryReference(CommandUtils.CombinePaths(RootWorkingDirectory, DestinationPath, PlatformLocalizationFolderName));
if (DirectoryReference.Exists(PlatformSourceDirectory))
{
foreach (DirectoryReference FoundDirectory in DirectoryReference.EnumerateDirectories(PlatformSourceDirectory, "*", SearchOption.TopDirectoryOnly))
{
string SplitPlatformName = CommandUtils.GetLastDirectoryName(FoundDirectory.FullName);
NewSplitPlatformNames.Add(SplitPlatformName);
}
}
}
SplitPlatformNames = NewSplitPlatformNames;
}
private static string GetLinuxToolchainSettings(TargetPlatformData TargetData)
{
if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Linux)
{
string ExtraSettings = "";
// native builds try to use bundled toolchain
string ToolchainPath = Environment.GetEnvironmentVariable("LINUX_MULTIARCH_ROOT");
if (string.IsNullOrEmpty(ToolchainPath))
{
// try to find bundled toolchain
DirectoryReference ToolchainDir = DirectoryReference.Combine(ThirdPartySourceDirectory, "..", "..", "Extras", "ThirdPartyNotUE", "SDKs", "HostLinux", "Linux_x64");
LogInformation("LINUX_MULTIARCH_ROOT not defined. Looking for Linux toolchain in {0}...", ToolchainDir);
IEnumerable <DirectoryReference> AvailableToolchains = DirectoryReference.EnumerateDirectories(ToolchainDir);
if (AvailableToolchains.Count() > 0)
{
// grab first available
ToolchainPath = AvailableToolchains.First() + "/x86_64-unknown-linux-gnu";
}
}
if (string.IsNullOrEmpty(ToolchainPath))
{
LogInformation("Bundled toolchain not found. Using system clang.");
}
else
{
ExtraSettings = GetBundledLinuxLibCxxFlags(ToolchainPath);
ToolchainPath += "/bin/";
}
LogInformation("Using toolchain: {0}", ToolchainPath);
return(string.Format(" -DCMAKE_C_COMPILER={0}clang -DCMAKE_CXX_COMPILER={0}clang++ ", ToolchainPath) + ExtraSettings);
}
// otherwise, use a per-architecture file.
return(" -DCMAKE_TOOLCHAIN_FILE=\"" + SourceRootDirectory + "\\..\\..\\PhysX3\\Externals\\CMakeModules\\Linux\\LinuxCrossToolchain.multiarch.cmake\"" + " -DARCHITECTURE_TRIPLE=" + TargetData.Architecture);
}
public void StageAppLocalDependencies(ProjectParams Params, DeploymentContext SC, string PlatformDir)
{
Dictionary <string, string> PathVariables = new Dictionary <string, string>();
PathVariables["EngineDir"] = SC.EngineRoot.FullName;
PathVariables["ProjectDir"] = SC.ProjectRoot.FullName;
// support multiple comma-separated paths
string[] AppLocalDirectories = Params.AppLocalDirectory.Split(';');
foreach (string AppLocalDirectory in AppLocalDirectories)
{
string ExpandedAppLocalDir = Utils.ExpandVariables(AppLocalDirectory, PathVariables);
DirectoryReference BaseAppLocalDependenciesPath = Path.IsPathRooted(ExpandedAppLocalDir) ? new DirectoryReference(CombinePaths(ExpandedAppLocalDir, PlatformDir)) : DirectoryReference.Combine(SC.ProjectRoot, ExpandedAppLocalDir, PlatformDir);
if (DirectoryReference.Exists(BaseAppLocalDependenciesPath))
{
StagedDirectoryReference ProjectBinaryPath = new StagedDirectoryReference(SC.ProjectBinariesFolder.MakeRelativeTo(SC.ProjectRoot.ParentDirectory));
StagedDirectoryReference EngineBinaryPath = new StagedDirectoryReference(CombinePaths("Engine", "Binaries", PlatformDir));
Log("Copying AppLocal dependencies from {0} to {1} and {2}", BaseAppLocalDependenciesPath, ProjectBinaryPath, EngineBinaryPath);
// Stage files in subdirs
foreach (DirectoryReference DependencyDirectory in DirectoryReference.EnumerateDirectories(BaseAppLocalDependenciesPath))
{
SC.StageFiles(StagedFileType.NonUFS, DependencyDirectory, StageFilesSearch.TopDirectoryOnly, ProjectBinaryPath);
SC.StageFiles(StagedFileType.NonUFS, DependencyDirectory, StageFilesSearch.TopDirectoryOnly, EngineBinaryPath);
}
// stage loose files here
SC.StageFiles(StagedFileType.NonUFS, BaseAppLocalDependenciesPath, StageFilesSearch.AllDirectories, ProjectBinaryPath);
SC.StageFiles(StagedFileType.NonUFS, BaseAppLocalDependenciesPath, StageFilesSearch.AllDirectories, EngineBinaryPath);
}
else
{
LogWarning("Unable to deploy AppLocalDirectory dependencies. No such path: {0}", BaseAppLocalDependenciesPath);
}
}
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "";
}
var LocalizationStepNames = new List <string>();
{
var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepNamesStr == null)
{
LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepNamesStr.Split(','))
{
LocalizationStepNames.Add(StepName.Trim());
}
}
LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var ShouldGatherPlatforms = ParseParam("IncludePlatforms");
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var EnableParallelGather = ParseParam("ParallelGather");
var StartTime = DateTime.UtcNow;
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for platforms
if (ShouldGatherPlatforms)
{
var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms"));
if (DirectoryReference.Exists(PlatformsRootDirectory))
{
foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory))
{
// Find the localization targets defined for this platform
var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization")));
if (PlatformTargetNames.Count > 0)
{
var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames));
}
}
}
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory));
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
var AvailablePluginNames = new HashSet <string>();
foreach (var PluginInfo in AllPlugins)
{
AvailablePluginNames.Add(PluginInfo.Name);
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
// If we had an explicit list of plugins to include, warn if any were missing
foreach (string PluginName in IncludePlugins)
{
if (!AvailablePluginNames.Contains(PluginName))
{
LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName);
}
}
}
// Create a single changelist to use for all changes
int PendingChangeList = 0;
if (P4Enabled)
{
var ChangeListCommitMessage = String.Format("Localization Automation using CL {0}", P4Env.Changelist);
if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Restricted/NotForLicensees/Build/EpicInternal.txt")))
{
ChangeListCommitMessage += "\n#okforgithub ignore";
}
PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage);
}
// Prepare to process each localization batch
var LocalizationTasks = new List <LocalizationTask>();
foreach (var LocalizationBatch in LocalizationBatches)
{
var LocalizationTask = new LocalizationTask(LocalizationBatch, UEProjectRoot, LocalizationProviderName, PendingChangeList, this);
LocalizationTasks.Add(LocalizationTask);
// Make sure the Localization configs and content is up-to-date to ensure we don't get errors later on
if (P4Enabled)
{
LogInformation("Sync necessary content to head revision");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Config/Localization/...");
P4.Sync(P4Env.Branch + "/" + LocalizationTask.Batch.LocalizationTargetDirectory + "/Content/Localization/...");
}
// Generate the info we need to gather for each project
foreach (var ProjectName in LocalizationTask.Batch.LocalizationProjectNames)
{
LocalizationTask.ProjectInfos.Add(GenerateProjectInfo(LocalizationTask.RootLocalizationTargetDirectory, ProjectName, LocalizationStepNames));
}
}
// Hash the current PO files on disk so we can work out whether they actually change
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Download the latest translations from our localization provider
if (LocalizationStepNames.Contains("Download"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
LocalizationTask.LocProvider.DownloadProjectFromLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ImportInfo);
}
}
}
}
// Begin the gather command for each task
// These can run in parallel when ParallelGather is enabled
{
var EditorExe = CombinePaths(CmdEnv.LocalRoot, @"Engine/Binaries/Win64/UE4Editor-Cmd.exe");
// Set the common basic editor arguments
var EditorArguments = P4Enabled
? String.Format("-SCCProvider=Perforce -P4Port={0} -P4User={1} -P4Client={2} -P4Passwd={3} -P4Changelist={4} -EnableSCC -DisableSCCSubmit", P4Env.ServerAndPort, P4Env.User, P4Env.Client, P4.GetAuthenticationToken(), PendingChangeList)
: "-SCCProvider=None";
if (IsBuildMachine)
{
EditorArguments += " -BuildMachine";
}
EditorArguments += " -Unattended -LogLocalizationConflicts";
if (EnableParallelGather)
{
EditorArguments += " -multiprocess";
}
if (!String.IsNullOrEmpty(AdditionalCommandletArguments))
{
EditorArguments += " " + AdditionalCommandletArguments;
}
// Set the common process run options
var CommandletRunOptions = ERunOptions.Default | ERunOptions.NoLoggingOfRunCommand; // Disable logging of the run command as it will print the exit code which GUBP can pick up as an error (we do that ourselves later)
if (EnableParallelGather)
{
CommandletRunOptions |= ERunOptions.NoWaitForExit;
}
foreach (var LocalizationTask in LocalizationTasks)
{
var ProjectArgument = String.IsNullOrEmpty(UEProjectName) ? "" : String.Format("\"{0}\"", Path.Combine(LocalizationTask.RootWorkingDirectory, String.Format("{0}.uproject", UEProjectName)));
foreach (var ProjectInfo in LocalizationTask.ProjectInfos)
{
var LocalizationConfigFiles = new List <string>();
foreach (var LocalizationStep in ProjectInfo.LocalizationSteps)
{
if (LocalizationStepNames.Contains(LocalizationStep.Name))
{
LocalizationConfigFiles.Add(LocalizationStep.LocalizationConfigFile);
}
}
if (LocalizationConfigFiles.Count > 0)
{
var Arguments = String.Format("{0} -run=GatherText -config=\"{1}\" {2}", ProjectArgument, String.Join(";", LocalizationConfigFiles), EditorArguments);
LogInformation("Running localization commandlet for '{0}': {1}", ProjectInfo.ProjectName, Arguments);
LocalizationTask.GatherProcessResults.Add(Run(EditorExe, Arguments, null, CommandletRunOptions));
}
else
{
LocalizationTask.GatherProcessResults.Add(null);
}
}
}
}
// Wait for each commandlet process to finish and report the result.
// This runs even for non-parallel execution to log the exit state of the process.
foreach (var LocalizationTask in LocalizationTasks)
{
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null)
{
RunResult.WaitForExit();
RunResult.OnProcessExited();
RunResult.DisposeProcess();
if (RunResult.ExitCode == 0)
{
LogInformation("The localization commandlet for '{0}' exited with code 0.", ProjectInfo.ProjectName);
}
else
{
LogWarning("The localization commandlet for '{0}' exited with code {1} which likely indicates a crash.", ProjectInfo.ProjectName, RunResult.ExitCode);
}
}
}
}
// Upload the latest sources to our localization provider
if (LocalizationStepNames.Contains("Upload"))
{
foreach (var LocalizationTask in LocalizationTasks)
{
if (LocalizationTask.LocProvider != null)
{
// Upload all text to our localization provider
for (int ProjectIndex = 0; ProjectIndex < LocalizationTask.ProjectInfos.Count; ++ProjectIndex)
{
var ProjectInfo = LocalizationTask.ProjectInfos[ProjectIndex];
var RunResult = LocalizationTask.GatherProcessResults[ProjectIndex];
if (RunResult != null && RunResult.ExitCode == 0)
{
// Recalculate the split platform paths before doing the upload, as the export may have changed them
ProjectInfo.ExportInfo.CalculateSplitPlatformNames(LocalizationTask.RootLocalizationTargetDirectory);
LocalizationTask.LocProvider.UploadProjectToLocalizationProvider(ProjectInfo.ProjectName, ProjectInfo.ExportInfo);
}
else
{
LogWarning("Skipping upload to the localization provider for '{0}' due to an earlier commandlet failure.", ProjectInfo.ProjectName);
}
}
}
}
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
var RunDuration = (DateTime.UtcNow - StartTime).TotalMilliseconds;
LogInformation("Localize command finished in {0} seconds", RunDuration / 1000);
}
public override void ExecuteBuild()
{
var UEProjectRoot = ParseParamValue("UEProjectRoot");
if (UEProjectRoot == null)
{
UEProjectRoot = CmdEnv.LocalRoot;
}
var UEProjectDirectory = ParseParamValue("UEProjectDirectory");
if (UEProjectDirectory == null)
{
throw new AutomationException("Missing required command line argument: 'UEProjectDirectory'");
}
var UEProjectName = ParseParamValue("UEProjectName");
if (UEProjectName == null)
{
UEProjectName = "";
}
var LocalizationProjectNames = new List <string>();
{
var LocalizationProjectNamesStr = ParseParamValue("LocalizationProjectNames");
if (LocalizationProjectNamesStr != null)
{
foreach (var ProjectName in LocalizationProjectNamesStr.Split(','))
{
LocalizationProjectNames.Add(ProjectName.Trim());
}
}
}
var LocalizationProviderName = ParseParamValue("LocalizationProvider");
if (LocalizationProviderName == null)
{
LocalizationProviderName = "";
}
var LocalizationStepNames = new List <string>();
{
var LocalizationStepNamesStr = ParseParamValue("LocalizationSteps");
if (LocalizationStepNamesStr == null)
{
LocalizationStepNames.AddRange(new string[] { "Download", "Gather", "Import", "Export", "Compile", "GenerateReports", "Upload" });
}
else
{
foreach (var StepName in LocalizationStepNamesStr.Split(','))
{
LocalizationStepNames.Add(StepName.Trim());
}
}
LocalizationStepNames.Add("Monolithic"); // Always allow the monolithic scripts to run as we don't know which steps they do
}
var ShouldGatherPlugins = ParseParam("IncludePlugins");
var IncludePlugins = new List <string>();
var ExcludePlugins = new List <string>();
if (ShouldGatherPlugins)
{
var IncludePluginsStr = ParseParamValue("IncludePlugins");
if (IncludePluginsStr != null)
{
foreach (var PluginName in IncludePluginsStr.Split(','))
{
IncludePlugins.Add(PluginName.Trim());
}
}
var ExcludePluginsStr = ParseParamValue("ExcludePlugins");
if (ExcludePluginsStr != null)
{
foreach (var PluginName in ExcludePluginsStr.Split(','))
{
ExcludePlugins.Add(PluginName.Trim());
}
}
}
var ShouldGatherPlatforms = ParseParam("IncludePlatforms");
var AdditionalCommandletArguments = ParseParamValue("AdditionalCommandletArguments");
if (AdditionalCommandletArguments == null)
{
AdditionalCommandletArguments = "";
}
var LocalizationBatches = new List <LocalizationBatch>();
// Add the static set of localization projects as a batch
if (LocalizationProjectNames.Count > 0)
{
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, UEProjectDirectory, "", LocalizationProjectNames));
}
// Build up any additional batches needed for platforms
if (ShouldGatherPlatforms)
{
var PlatformsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory, "Platforms"));
if (DirectoryReference.Exists(PlatformsRootDirectory))
{
foreach (DirectoryReference PlatformDirectory in DirectoryReference.EnumerateDirectories(PlatformsRootDirectory))
{
// Find the localization targets defined for this platform
var PlatformTargetNames = GetLocalizationTargetsFromDirectory(new DirectoryReference(CombinePaths(PlatformDirectory.FullName, "Config", "Localization")));
if (PlatformTargetNames.Count > 0)
{
var RootRelativePluginPath = PlatformDirectory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, "", PlatformTargetNames));
}
}
}
}
// Build up any additional batches needed for plugins
if (ShouldGatherPlugins)
{
var PluginsRootDirectory = new DirectoryReference(CombinePaths(UEProjectRoot, UEProjectDirectory));
IReadOnlyList <PluginInfo> AllPlugins = Plugins.ReadPluginsFromDirectory(PluginsRootDirectory, "Plugins", UEProjectName.Length == 0 ? PluginType.Engine : PluginType.Project);
// Add a batch for each plugin that meets our criteria
var AvailablePluginNames = new HashSet <string>();
foreach (var PluginInfo in AllPlugins)
{
AvailablePluginNames.Add(PluginInfo.Name);
bool ShouldIncludePlugin = (IncludePlugins.Count == 0 || IncludePlugins.Contains(PluginInfo.Name)) && !ExcludePlugins.Contains(PluginInfo.Name);
if (ShouldIncludePlugin && PluginInfo.Descriptor.LocalizationTargets != null && PluginInfo.Descriptor.LocalizationTargets.Length > 0)
{
var RootRelativePluginPath = PluginInfo.Directory.MakeRelativeTo(new DirectoryReference(UEProjectRoot));
RootRelativePluginPath = RootRelativePluginPath.Replace('\\', '/'); // Make sure we use / as these paths are used with P4
var PluginTargetNames = new List <string>();
foreach (var LocalizationTarget in PluginInfo.Descriptor.LocalizationTargets)
{
PluginTargetNames.Add(LocalizationTarget.Name);
}
LocalizationBatches.Add(new LocalizationBatch(UEProjectDirectory, RootRelativePluginPath, PluginInfo.Name, PluginTargetNames));
}
}
// If we had an explicit list of plugins to include, warn if any were missing
foreach (string PluginName in IncludePlugins)
{
if (!AvailablePluginNames.Contains(PluginName))
{
LogWarning("The plugin '{0}' specified by -IncludePlugins wasn't found and will be skipped.", PluginName);
}
}
}
// Create a single changelist to use for all changes, and hash the current PO files on disk so we can work out whether they actually change
int PendingChangeList = 0;
Dictionary <string, byte[]> InitalPOFileHashes = null;
if (P4Enabled)
{
var ChangeListCommitMessage = "Localization Automation";
if (File.Exists(CombinePaths(CmdEnv.LocalRoot, @"Engine/Build/NotForLicensees/EpicInternal.txt")))
{
ChangeListCommitMessage += "\n#okforgithub ignore";
}
PendingChangeList = P4.CreateChange(P4Env.Client, ChangeListCommitMessage);
InitalPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
}
// Process each localization batch
foreach (var LocalizationBatch in LocalizationBatches)
{
ProcessLocalizationProjects(LocalizationBatch, PendingChangeList, UEProjectRoot, UEProjectName, LocalizationProviderName, LocalizationStepNames, AdditionalCommandletArguments);
}
// Clean-up the changelist so it only contains the changed files, and then submit it (if we were asked to)
if (P4Enabled)
{
// Revert any PO files that haven't changed aside from their header
{
var POFilesToRevert = new List <string>();
var CurrentPOFileHashes = GetPOFileHashes(LocalizationBatches, UEProjectRoot);
foreach (var CurrentPOFileHashPair in CurrentPOFileHashes)
{
byte[] InitialPOFileHash;
if (InitalPOFileHashes.TryGetValue(CurrentPOFileHashPair.Key, out InitialPOFileHash) && InitialPOFileHash.SequenceEqual(CurrentPOFileHashPair.Value))
{
POFilesToRevert.Add(CurrentPOFileHashPair.Key);
}
}
if (POFilesToRevert.Count > 0)
{
var P4RevertArgsFilename = CombinePaths(CmdEnv.LocalRoot, "Engine", "Intermediate", String.Format("LocalizationP4RevertArgs-{0}.txt", Guid.NewGuid().ToString()));
using (StreamWriter P4RevertArgsWriter = File.CreateText(P4RevertArgsFilename))
{
foreach (var POFileToRevert in POFilesToRevert)
{
P4RevertArgsWriter.WriteLine(POFileToRevert);
}
}
P4.LogP4(String.Format("-x{0} revert", P4RevertArgsFilename));
DeleteFile_NoExceptions(P4RevertArgsFilename);
}
}
// Revert any other unchanged files
P4.RevertUnchanged(PendingChangeList);
// Submit that single changelist now
if (AllowSubmit)
{
int SubmittedChangeList;
P4.Submit(PendingChangeList, out SubmittedChangeList);
}
}
}
