/// <summary>
/// Runs the task, filtering the list of build products
/// </summary>
/// <param name="BuildProducts"></param>
/// <returns>True if the task succeeds</returns>
public override bool Execute(List<string> BuildProducts)
{
if(AddFiles != null)
{
BuildProducts.AddRange(AddFiles.ApplyToDirectory(BaseDirectory, true).Select(x => CommandUtils.CombinePaths(BaseDirectory, x)));
}
if(FilterFiles != null)
{
BuildProducts.RemoveAll(x => UnrealBuildTool.Utils.IsFileUnderDirectory(x, BaseDirectory) && !FilterFiles.Matches(UnrealBuildTool.Utils.StripBaseDirectory(x, BaseDirectory)));
}
return true;
}
/// <summary>
/// Resolve a list of files, tag names or file specifications as above, but preserves any directory references for further processing.
/// </summary>
/// <param name="DefaultDirectory">The default directory to resolve relative paths to</param>
/// <param name="FilePatterns">List of files, tag names, or file specifications to include separated by semicolons.</param>
/// <param name="ExcludePatterns">Set of patterns to apply to directory searches. This can greatly speed up enumeration by earlying out of recursive directory searches if large directories are excluded (eg. .../Intermediate/...).</param>
/// <param name="TagNameToFileSet">Mapping of tag name to fileset, as passed to the Execute() method</param>
/// <returns>Set of matching files.</returns>
public static HashSet <FileReference> ResolveFilespecWithExcludePatterns(DirectoryReference DefaultDirectory, List <string> FilePatterns, List <string> ExcludePatterns, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Parse each of the patterns, and add the results into the given sets
HashSet <FileReference> Files = new HashSet <FileReference>();
foreach (string Pattern in FilePatterns)
{
// Check if it's a tag name
if (Pattern.StartsWith("#"))
{
Files.UnionWith(FindOrAddTagSet(TagNameToFileSet, Pattern));
continue;
}
// If it doesn't contain any wildcards, just add the pattern directly
int WildcardIdx = FileFilter.FindWildcardIndex(Pattern);
if (WildcardIdx == -1)
{
Files.Add(FileReference.Combine(DefaultDirectory, Pattern));
continue;
}
// Find the base directory for the search. We construct this in a very deliberate way including the directory separator itself, so matches
// against the OS root directory will resolve correctly both on Mac (where / is the filesystem root) and Windows (where / refers to the current drive).
int LastDirectoryIdx = Pattern.LastIndexOfAny(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, WildcardIdx);
DirectoryReference BaseDir = DirectoryReference.Combine(DefaultDirectory, Pattern.Substring(0, LastDirectoryIdx + 1));
// Construct the absolute include pattern to match against, re-inserting the resolved base directory to construct a canonical path.
string IncludePattern = BaseDir.FullName.TrimEnd(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }) + "/" + Pattern.Substring(LastDirectoryIdx + 1);
// Construct a filter and apply it to the directory
if (DirectoryReference.Exists(BaseDir))
{
FileFilter Filter = new FileFilter();
Filter.AddRule(IncludePattern, FileFilterType.Include);
if (ExcludePatterns != null && ExcludePatterns.Count > 0)
{
Filter.AddRules(ExcludePatterns, FileFilterType.Exclude);
}
Files.UnionWith(Filter.ApplyToDirectory(BaseDir, BaseDir.FullName, true));
}
}
// If we have exclude rules, create and run a filter against all the output files to catch things that weren't added from an include
if (ExcludePatterns != null && ExcludePatterns.Count > 0)
{
FileFilter Filter = new FileFilter(FileFilterType.Include);
Filter.AddRules(ExcludePatterns, FileFilterType.Exclude);
Files.RemoveWhere(x => !Filter.Matches(x.FullName));
}
return(Files);
}
public override void DoBuild(GUBP bp)
{
BuildProducts = new List<string>();
FileFilter Filter = new FileFilter();
// Include all the editor products
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsForCompileNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.RootEditorNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
// Include win64 tools on Mac, to get the win64 build of UBT, UAT and IPP
if (HostPlatform == UnrealTargetPlatform.Mac && bp.HostPlatforms.Contains(UnrealTargetPlatform.Win64))
{
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsNode.StaticGetFullName(UnrealTargetPlatform.Win64), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsForCompileNode.StaticGetFullName(UnrealTargetPlatform.Win64), FileFilterType.Include);
}
// Include the editor headers
UnzipAndAddRuleForHeaders(GUBP.RootEditorNode.StaticGetArchivedHeadersPath(HostPlatform), Filter, FileFilterType.Include);
// Include the build dependencies for every code platform
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if(RocketBuild.IsCodeTargetPlatform(HostPlatform, TargetPlatform))
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
string FileListPath = GUBP.GamePlatformMonolithicsNode.StaticGetBuildDependenciesPath(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform);
Filter.AddRuleForFiles(UnrealBuildTool.Utils.ReadClass<UnrealBuildTool.ExternalFileList>(FileListPath).FileNames, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
UnzipAndAddRuleForHeaders(GUBP.GamePlatformMonolithicsNode.StaticGetArchivedHeadersPath(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform), Filter, FileFilterType.Include);
}
}
// Add the monolithic binaries
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
bool bIsCodeTargetPlatform = RocketBuild.IsCodeTargetPlatform(SourceHostPlatform, TargetPlatform);
AddRuleForBuildProducts(Filter, bp, GUBP.GamePlatformMonolithicsNode.StaticGetFullName(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform, Precompiled: bIsCodeTargetPlatform), FileFilterType.Include);
}
// Include the feature packs
foreach(string CurrentFeaturePack in CurrentFeaturePacks)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(CurrentFeaturePack);
Filter.AddRuleForFile(GUBP.MakeFeaturePacksNode.GetOutputFile(Project), CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
}
// Include all the templates
foreach (string Template in CurrentTemplates)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(Template);
Filter.Include("/" + Utils.StripBaseDirectory(Path.GetDirectoryName(Project.FilePath), CommandUtils.CmdEnv.LocalRoot).Replace('\\', '/') + "/...");
}
// Include all the standard rules
string RulesFileName = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini");
Filter.ReadRulesFromFile(RulesFileName, "CopyEditor", HostPlatform.ToString());
Filter.ReadRulesFromFile(RulesFileName, "CopyTargetPlatforms", HostPlatform.ToString());
// Custom rules for each target platform
foreach(UnrealTargetPlatform TargetPlaform in TargetPlatforms)
{
string SectionName = String.Format("CopyTargetPlatform.{0}", TargetPlaform.ToString());
Filter.ReadRulesFromFile(RulesFileName, SectionName, HostPlatform.ToString());
}
// Add the final exclusions for legal reasons.
Filter.ExcludeConfidentialPlatforms();
Filter.ExcludeConfidentialFolders();
// Run the filter on the stripped symbols, and remove those files from the copy filter
List<string> AllStrippedFiles = new List<string>();
foreach(KeyValuePair<string, string> StrippedNodeManifestPath in StrippedNodeManifestPaths)
{
List<string> StrippedFiles = new List<string>();
StripRocketNode StripNode = (StripRocketNode)bp.FindNode(StrippedNodeManifestPath.Key);
foreach(string BuildProduct in StripNode.BuildProducts)
{
if(Utils.IsFileUnderDirectory(BuildProduct, StripNode.StrippedDir))
{
string RelativePath = CommandUtils.StripBaseDirectory(Path.GetFullPath(BuildProduct), StripNode.StrippedDir);
if(Filter.Matches(RelativePath))
{
StrippedFiles.Add(RelativePath);
AllStrippedFiles.Add(RelativePath);
Filter.Exclude("/" + RelativePath);
}
}
}
WriteManifest(StrippedNodeManifestPath.Value, StrippedFiles);
BuildProducts.Add(StrippedNodeManifestPath.Value);
}
// Write the filtered list of depot files to disk, removing any symlinks
List<string> DepotFiles = Filter.ApplyToDirectory(CommandUtils.CmdEnv.LocalRoot, true).ToList();
WriteManifest(DepotManifestPath, DepotFiles);
BuildProducts.Add(DepotManifestPath);
// Sort the list of output files
SortedDictionary<string, bool> SortedFiles = new SortedDictionary<string,bool>(StringComparer.InvariantCultureIgnoreCase);
foreach(string DepotFile in DepotFiles)
{
SortedFiles.Add(DepotFile, false);
}
foreach(string StrippedFile in AllStrippedFiles)
{
SortedFiles.Add(StrippedFile, true);
}
// Write the list to the log
CommandUtils.Log("Files to be included in Rocket build:");
foreach(KeyValuePair<string, bool> SortedFile in SortedFiles)
{
CommandUtils.Log(" {0}{1}", SortedFile.Key, SortedFile.Value? " (stripped)" : "");
}
}
/// <summary>
/// Creates a file mapping between a set of source patterns and a target pattern. All patterns should have a matching order and number of wildcards.
/// </summary>
/// <param name="Files">Files to use for the mapping</param>
/// <param name="SourcePatterns">List of source patterns</param>
/// <param name="TargetPattern">Matching output pattern</param>
/// <param name="Filter">Filter to apply to source files</param>
/// <param name="TargetFileToSourceFile">Dictionary to receive a mapping from target file to source file. An exception is thrown if multiple source files map to one target file, or a source file is also used as a target file.</param>
public static bool TryCreateMapping(HashSet <FileReference> Files, FilePattern SourcePattern, FilePattern TargetPattern, out Dictionary <FileReference, FileReference> OutTargetFileToSourceFile)
{
bool bResult = true;
// If the source pattern ends in a directory separator, or a set of input files are specified and it doesn't contain wildcards, treat it as a full directory match
if (SourcePattern.EndsWithDirectorySeparator())
{
SourcePattern = new FilePattern(SourcePattern.BaseDirectory, String.Join("", SourcePattern.Tokens) + "...");
}
else if (Files != null)
{
SourcePattern = SourcePattern.AsDirectoryPattern();
}
// If we have multiple potential source files, but no wildcards in the output pattern, assume it's a directory and append the pattern from the source.
if (SourcePattern.ContainsWildcards() && !TargetPattern.ContainsWildcards())
{
StringBuilder NewPattern = new StringBuilder();
foreach (string Token in TargetPattern.Tokens)
{
NewPattern.Append(Token);
}
if (NewPattern.Length > 0 && NewPattern[NewPattern.Length - 1] != Path.DirectorySeparatorChar)
{
NewPattern.Append(Path.DirectorySeparatorChar);
}
foreach (string Token in SourcePattern.Tokens)
{
NewPattern.Append(Token);
}
TargetPattern = new FilePattern(TargetPattern.BaseDirectory, NewPattern.ToString());
}
// If the target pattern ends with a directory separator, treat it as a full directory match if it has wildcards, or a copy of the source pattern if not
if (TargetPattern.EndsWithDirectorySeparator())
{
TargetPattern = new FilePattern(TargetPattern.BaseDirectory, String.Join("", TargetPattern.Tokens) + "...");
}
// Handle the case where source and target pattern are both individual files
Dictionary <FileReference, FileReference> TargetFileToSourceFile = new Dictionary <FileReference, FileReference>();
if (SourcePattern.ContainsWildcards() || TargetPattern.ContainsWildcards())
{
// Check the two patterns are compatible
if (!SourcePattern.IsCompatibleWith(TargetPattern))
{
CommandUtils.LogError("File patterns '{0}' and '{1}' do not have matching wildcards", SourcePattern, TargetPattern);
OutTargetFileToSourceFile = null;
return(false);
}
// Create a filter to match the source files
FileFilter Filter = new FileFilter(FileFilterType.Exclude);
Filter.Include(String.Join("", SourcePattern.Tokens));
// Apply it to the source directory
List <FileReference> SourceFiles;
if (Files == null)
{
SourceFiles = Filter.ApplyToDirectory(SourcePattern.BaseDirectory, true);
}
else
{
SourceFiles = CheckInputFiles(Files, SourcePattern.BaseDirectory, ref bResult);
}
// Map them onto output files
FileReference[] TargetFiles = new FileReference[SourceFiles.Count];
// Get the source and target regexes
string SourceRegex = SourcePattern.GetRegexPattern();
string TargetRegex = TargetPattern.GetRegexReplacementPattern();
for (int Idx = 0; Idx < SourceFiles.Count; Idx++)
{
string SourceRelativePath = SourceFiles[Idx].MakeRelativeTo(SourcePattern.BaseDirectory);
string TargetRelativePath = Regex.Replace(SourceRelativePath, SourceRegex, TargetRegex);
TargetFiles[Idx] = FileReference.Combine(TargetPattern.BaseDirectory, TargetRelativePath);
}
// Add them to the output map
for (int Idx = 0; Idx < TargetFiles.Length; Idx++)
{
FileReference ExistingSourceFile;
if (TargetFileToSourceFile.TryGetValue(TargetFiles[Idx], out ExistingSourceFile) && ExistingSourceFile != SourceFiles[Idx])
{
CommandUtils.LogError("Output file '{0}' is mapped from '{1}' and '{2}'", TargetFiles[Idx], ExistingSourceFile, SourceFiles[Idx]);
bResult = false;
}
TargetFileToSourceFile[TargetFiles[Idx]] = SourceFiles[Idx];
}
}
else
{
// Just copy a single file
FileReference SourceFile = SourcePattern.GetSingleFile();
if (SourceFile.Exists())
{
FileReference TargetFile = TargetPattern.GetSingleFile();
TargetFileToSourceFile[TargetFile] = SourceFile;
}
else
{
CommandUtils.LogError("Source file '{0}' does not exist", SourceFile);
bResult = false;
}
}
// Check that no source file is also destination file
foreach (FileReference SourceFile in TargetFileToSourceFile.Values)
{
if (TargetFileToSourceFile.ContainsKey(SourceFile))
{
CommandUtils.LogError("'{0}' is listed as a source and target file", SourceFile);
bResult = false;
}
}
// Set the output map
if (bResult)
{
OutTargetFileToSourceFile = TargetFileToSourceFile;
return(true);
}
else
{
OutTargetFileToSourceFile = null;
return(false);
}
}
/// <summary>
/// Resolve a list of files, tag names or file specifications as above, but preserves any directory references for further processing.
/// </summary>
/// <param name="DefaultDirectory">The default directory to resolve relative paths to</param>
/// <param name="FilePatterns">List of files, tag names, or file specifications to include separated by semicolons.</param>
/// <param name="ExcludePatterns">Set of patterns to apply to directory searches. This can greatly speed up enumeration by earlying out of recursive directory searches if large directories are excluded (eg. .../Intermediate/...).</param>
/// <param name="TagNameToFileSet">Mapping of tag name to fileset, as passed to the Execute() method</param>
/// <returns>Set of matching files.</returns>
public static HashSet<FileReference> ResolveFilespecWithExcludePatterns(DirectoryReference DefaultDirectory, List<string> FilePatterns, List<string> ExcludePatterns, Dictionary<string, HashSet<FileReference>> TagNameToFileSet)
{
// Parse each of the patterns, and add the results into the given sets
HashSet<FileReference> Files = new HashSet<FileReference>();
foreach(string Pattern in FilePatterns)
{
// Check if it's a tag name
if(Pattern.StartsWith("#"))
{
Files.UnionWith(FindOrAddTagSet(TagNameToFileSet, Pattern));
continue;
}
// If it doesn't contain any wildcards, just add the pattern directly
int WildcardIdx = FileFilter.FindWildcardIndex(Pattern);
if(WildcardIdx == -1)
{
Files.Add(FileReference.Combine(DefaultDirectory, Pattern));
continue;
}
// Find the base directory for the search. We construct this in a very deliberate way including the directory separator itself, so matches
// against the OS root directory will resolve correctly both on Mac (where / is the filesystem root) and Windows (where / refers to the current drive).
int LastDirectoryIdx = Pattern.LastIndexOfAny(new char[]{ Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, WildcardIdx);
DirectoryReference BaseDir = DirectoryReference.Combine(DefaultDirectory, Pattern.Substring(0, LastDirectoryIdx + 1));
// Construct the absolute include pattern to match against, re-inserting the resolved base directory to construct a canonical path.
string IncludePattern = BaseDir.FullName.TrimEnd(new char[]{ Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }) + "/" + Pattern.Substring(LastDirectoryIdx + 1);
// Construct a filter and apply it to the directory
if(BaseDir.Exists())
{
FileFilter Filter = new FileFilter();
Filter.AddRule(IncludePattern, FileFilterType.Include);
Filter.AddRules(ExcludePatterns, FileFilterType.Exclude);
Files.UnionWith(Filter.ApplyToDirectory(BaseDir, BaseDir.FullName, true));
}
}
// If we have exclude rules, create and run a filter against all the output files to catch things that weren't added from an include
if(ExcludePatterns.Count > 0)
{
FileFilter Filter = new FileFilter(FileFilterType.Include);
Filter.AddRules(ExcludePatterns, FileFilterType.Exclude);
Files.RemoveWhere(x => !Filter.Matches(x.FullName));
}
return Files;
}
public static string CopyFilesToZip(string ZipBaseFolder, string BaseDirectory, string InZipDir, FileFilter Filter, bool NoConversion, bool ParseTextOnly = false)
{
string TempFolder = GetRandomFolder(ZipBaseFolder);
string CopyFolder = Path.Combine(TempFolder, InZipDir);
CreateDirectory(CopyFolder);
foreach (string FilteredFile in Filter.ApplyToDirectory(BaseDirectory, true))
{
string srcf = Path.Combine(BaseDirectory, FilteredFile);
string destfo = Path.GetDirectoryName(FilteredFile);
string destfi = Path.Combine(CopyFolder, FilteredFile);
if (!NoConversion && Path.GetExtension(FilteredFile).Equals(".pod", StringComparison.InvariantCultureIgnoreCase))
{
var reader = new StreamReader(srcf);
string content = reader.ReadToEnd();
reader.Dispose();
var parser = Parser.BaseParser.Create<Parser.POD.PODParser>(new StreamReader(srcf));
if (ParseTextOnly)
{
using (var writer = new StreamWriter(destfi + ".txt"))
{
writer.WriteLine(parser.Text());
}
}
else
{
using (var writer = new StreamWriter(destfi + ".htm"))
{
writer.WriteLine(parser.Flatten());
}
}
}
else if (!NoConversion && Path.GetExtension(FilteredFile).Equals(".md", StringComparison.InvariantCultureIgnoreCase))
{
var m = new MarkdownSharp.Markdown();
var reader = new StreamReader(srcf);
string content = reader.ReadToEnd();
reader.Dispose();
var UrlEvaluator = new MatchEvaluator(delegate(Match match)
{
string outstr = match.ToString();
if (match.Groups.Count == 2)
{
string relfile = match.Groups[1].Value;
string MailPattern = @"^(?("")("".+?(?<!\\)""@)|(([0-9a-z]((\.(?!\.))|[-!#\$%&'\*\+/=\?\^`\{\}\|~\w])*)(?<=[0-9a-z])@))";
string DomainPattern = @"(?(\[)(\[(\d{1,3}\.){3}\d{1,3}\])|(([0-9a-z][-\w]*[0-9a-z]*\.)+[a-z0-9][\-a-z0-9]{0,22}[a-z0-9]))";
if (!relfile.Contains("://") && !Regex.Match(relfile, MailPattern + DomainPattern).Success)
{
if (relfile.StartsWith("//"))
{
relfile = "https:" + relfile;
}
else if (relfile.StartsWith("/../../") || relfile.StartsWith("../../"))
{
relfile = relfile.Replace("../../", "");
relfile = URL_REPO_BASE + "/" + relfile.TrimStart('/');
}
else
{
string[] splits = relfile.Split("?|#&".ToCharArray());
if (Regex.Match(relfile, DomainPattern).Success && !File.Exists(Path.Combine(BaseDirectory, relfile)))
{
relfile = "https:" + relfile;
}
else if (splits.Length > 1)
{
if (splits[0].Length >= 260 || !File.Exists(Path.Combine(BaseDirectory, splits[0])))
{
relfile = URL_REPO_BASE + splits[0];
}
}
else
{
string p = Path.Combine(BaseDirectory, relfile);
if (!File.Exists(p) && !Directory.Exists(p))
{
relfile = URL_REPO_BASE + "/" + splits[0].TrimStart('/');
}
}
}
}
outstr = match.Groups[0].Value.Replace(match.Groups[1].Value, relfile);
}
return outstr;
});
var FileEvaluator = new MatchEvaluator(delegate(Match match)
{
string outstr = match.ToString();
if (match.Groups.Count == 6)
{
string seps = "?|#&";
string filename = match.Groups[3].Value;
string[] splits = filename.Split(seps.ToCharArray());
if (splits.Length > 0 && File.Exists(Path.Combine(BaseDirectory, splits[0])))
{
string ext = match.Groups[4].Value;
if (splits.Length > 1) ext = ext.Split(seps.ToCharArray())[0];
filename = filename.Replace(splits[0], splits[0] + ".htm");
}
outstr = string.Format("[{0}{1}{2}{3}", match.Groups[1].Value, match.Groups[2].Value, filename, match.Groups[5].Value);
}
return outstr;
});
if (ParseTextOnly)
{
m.OnlyText = true;
m.AutoHyperlink = false;
content = m.Transform(content);
using (var writer = new StreamWriter(destfi + ".txt"))
{
writer.WriteLine(content.Replace("\n", "\r\n"));
}
}
else
{
// fix relative links
content = Regex.Replace(content, @"\[.*?\]\((.*?)\)", UrlEvaluator);
content = Regex.Replace(content, @"\[\s*[a-zA-Z0-9_-]+\s*\]\s*:\s*(\S+)\s*", UrlEvaluator);
// redirect files in HTML
content = Regex.Replace(content, @"\[(.*?)(\]\()(.*\.(pod|md)\S*)(\))", FileEvaluator, RegexOptions.IgnoreCase);
content = Regex.Replace(content, @"\[(\s*[a-zA-Z0-9_-]+\s*)(\]\s*:\s*)(.*\.(pod|md)\S*)(\s*)", FileEvaluator);
using (var writer = new StreamWriter(destfi + ".htm"))
{
writer.WriteLine(m.Transform(content));
}
}
}
else
{
CopyFile(srcf, destfi);
}
}
return TempFolder;
}
