/// <summary>
/// Copy constructor
/// </summary>
/// <param name="Other">Filter to copy from</param>
public FileFilter(FileFilter Other)
: this(Other.DefaultNode.Type)
{
foreach (FileFilterNode OtherRule in Other.Rules)
{
AddRule(OtherRule.ToString(), OtherRule.Type);
}
}
public override void DoBuild(GUBP bp)
{
// Create a zip file containing the install
string FullZipFileName = Path.Combine(CommandUtils.CmdEnv.LocalRoot, "FullInstall" + StaticGetHostPlatformSuffix(HostPlatform) + ".zip");
CommandUtils.Log("Creating {0}...", FullZipFileName);
CommandUtils.ZipFiles(FullZipFileName, LocalDir, new FileFilter(FileFilterType.Include));
// Create a filter for the files we need just to run the editor
FileFilter EditorFilter = new FileFilter(FileFilterType.Include);
EditorFilter.Exclude("/Engine/Binaries/...");
EditorFilter.Include("/Engine/Binaries/DotNET/...");
EditorFilter.Include("/Engine/Binaries/ThirdParty/...");
EditorFilter.Include("/Engine/Binaries/" + HostPlatform.ToString() + "/...");
EditorFilter.Exclude("/Engine/Binaries/.../*.lib");
EditorFilter.Exclude("/Engine/Binaries/.../*.a");
EditorFilter.Exclude("/Engine/Extras/...");
EditorFilter.Exclude("/Engine/Source/.../Private/...");
EditorFilter.Exclude("/FeaturePacks/...");
EditorFilter.Exclude("/Samples/...");
EditorFilter.Exclude("/Templates/...");
EditorFilter.Exclude("*.pdb");
// Create a zip file containing the editor install
string EditorZipFileName = Path.Combine(CommandUtils.CmdEnv.LocalRoot, "EditorInstall" + StaticGetHostPlatformSuffix(HostPlatform) + ".zip");
CommandUtils.Log("Creating {0}...", EditorZipFileName);
CommandUtils.ZipFiles(EditorZipFileName, LocalDir, EditorFilter);
// Copy the files to their final location
CommandUtils.Log("Copying files to {0}", PublishDir);
CommandUtils.CopyFile(FullZipFileName, Path.Combine(PublishDir, Path.GetFileName(FullZipFileName)));
CommandUtils.CopyFile(EditorZipFileName, Path.Combine(PublishDir, Path.GetFileName(EditorZipFileName)));
CommandUtils.DeleteFile(FullZipFileName);
CommandUtils.DeleteFile(EditorZipFileName);
// Save a record of success
BuildProducts = new List<string>();
SaveRecordOfSuccessAndAddToBuildProducts();
}
public override void DoBuild(GUBP bp)
{
CommandUtils.CreateDirectory(SavedDir);
BuildProducts = new List<string>();
List<string> ManifestFiles = new List<string>();
if(!bp.ParseParam("NoDDC"))
{
// Find all the projects we're interested in
List<BranchInfo.BranchUProject> Projects = new List<BranchInfo.BranchUProject>();
foreach(string ProjectName in ProjectNames)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(ProjectName);
if(!Project.Properties.bIsCodeBasedProject)
{
Projects.Add(Project);
}
}
// Filter out the files we need to build DDC. Removing confidential folders can affect DDC keys, so we want to be sure that we're making DDC with a build that can use it.
FileFilter Filter = new FileFilter(FileFilterType.Exclude);
Filter.AddRuleForFiles(AllDependencyBuildProducts, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
Filter.ReadRulesFromFile(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini"), "CopyEditor", HostPlatform.ToString());
Filter.Exclude("/Engine/Build/...");
Filter.Exclude("/Engine/Extras/...");
Filter.Exclude("/Engine/DerivedDataCache/...");
Filter.Exclude("/Samples/...");
Filter.Exclude("/Templates/...");
Filter.Exclude(".../Source/...");
Filter.Exclude(".../Intermediate/...");
Filter.ExcludeConfidentialPlatforms();
Filter.ExcludeConfidentialFolders();
Filter.Include("/Engine/Build/NotForLicensees/EpicInternal.txt");
Filter.Include("/Engine/Binaries/.../*DDCUtils*"); // Make sure we can use the shared DDC!
// Copy everything to a temporary directory
string TempDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "RocketDDC", CommandUtils.GetGenericPlatformName(HostPlatform));
CommandUtils.DeleteDirectoryContents(TempDir);
CommandUtils.ThreadedCopyFiles(CommandUtils.CmdEnv.LocalRoot, TempDir, Filter, true);
// Get paths to everything within the temporary directory
string EditorExe = CommandUtils.GetEditorCommandletExe(TempDir, HostPlatform);
string RelativePakPath = "Engine/DerivedDataCache/Compressed.ddp";
string OutputPakFile = CommandUtils.CombinePaths(TempDir, RelativePakPath);
string OutputCsvFile = Path.ChangeExtension(OutputPakFile, ".csv");
// Generate DDC for all the non-code projects. We don't necessarily have editor DLLs for the code projects, but they should be the same as their blueprint counterparts.
List<string> ProjectPakFiles = new List<string>();
foreach(BranchInfo.BranchUProject Project in Projects)
{
CommandUtils.Log("Generating DDC data for {0} on {1}", Project.GameName, TargetPlatforms);
CommandUtils.DDCCommandlet(Project.FilePath, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak -ProjectOnly");
string ProjectPakFile = CommandUtils.CombinePaths(Path.GetDirectoryName(OutputPakFile), String.Format("Compressed-{0}.ddp", Project.GameName));
CommandUtils.DeleteFile(ProjectPakFile);
CommandUtils.RenameFile(OutputPakFile, ProjectPakFile);
string ProjectCsvFile = Path.ChangeExtension(ProjectPakFile, ".csv");
CommandUtils.DeleteFile(ProjectCsvFile);
CommandUtils.RenameFile(OutputCsvFile, ProjectCsvFile);
ProjectPakFiles.Add(Path.GetFileName(ProjectPakFile));
}
// Generate DDC for the editor, and merge all the other PAK files in
CommandUtils.Log("Generating DDC data for engine content on {0}", TargetPlatforms);
CommandUtils.DDCCommandlet(null, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak " + CommandUtils.MakePathSafeToUseWithCommandLine("-MergePaks=" + String.Join("+", ProjectPakFiles)));
// Copy the DDP file to the output path
string SavedPakFile = CommandUtils.CombinePaths(SavedDir, RelativePakPath);
CommandUtils.CopyFile(OutputPakFile, SavedPakFile);
BuildProducts.Add(SavedPakFile);
// Add the pak file to the list of files to copy
ManifestFiles.Add(RelativePakPath);
}
CommandUtils.WriteAllLines(SavedManifestPath, ManifestFiles.ToArray());
BuildProducts.Add(SavedManifestPath);
SaveRecordOfSuccessAndAddToBuildProducts();
}
static void UnzipAndAddRuleForHeaders(string ZipFileName, FileFilter Filter, FileFilterType Type)
{
IEnumerable<string> FileNames = CommandUtils.UnzipFiles(ZipFileName, CommandUtils.CmdEnv.LocalRoot);
Filter.AddRuleForFiles(FileNames, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
}
static void AddRuleForBuildProducts(FileFilter Filter, GUBP bp, string NodeName, FileFilterType Type)
{
GUBP.GUBPNode Node = bp.FindNode(NodeName);
if(Node == null)
{
throw new AutomationException("Couldn't find node '{0}'", NodeName);
}
Filter.AddRuleForFiles(Node.BuildProducts, CommandUtils.CmdEnv.LocalRoot, Type);
}
public override void DoBuild(GUBP bp)
{
BuildProducts = new List<string>();
FileFilter Filter = new FileFilter();
// Include all the editor products
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsForCompileNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.RootEditorNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
// Include win64 tools on Mac, to get the win64 build of UBT, UAT and IPP
if (HostPlatform == UnrealTargetPlatform.Mac && bp.HostPlatforms.Contains(UnrealTargetPlatform.Win64))
{
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsNode.StaticGetFullName(UnrealTargetPlatform.Win64), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsForCompileNode.StaticGetFullName(UnrealTargetPlatform.Win64), FileFilterType.Include);
}
// Include the editor headers
UnzipAndAddRuleForHeaders(GUBP.RootEditorNode.StaticGetArchivedHeadersPath(HostPlatform), Filter, FileFilterType.Include);
// Include the build dependencies for every code platform
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if(RocketBuild.IsCodeTargetPlatform(HostPlatform, TargetPlatform))
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
string FileListPath = GUBP.GamePlatformMonolithicsNode.StaticGetBuildDependenciesPath(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform);
Filter.AddRuleForFiles(UnrealBuildTool.Utils.ReadClass<UnrealBuildTool.ExternalFileList>(FileListPath).FileNames, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
UnzipAndAddRuleForHeaders(GUBP.GamePlatformMonolithicsNode.StaticGetArchivedHeadersPath(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform), Filter, FileFilterType.Include);
}
}
// Add the monolithic binaries
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
bool bIsCodeTargetPlatform = RocketBuild.IsCodeTargetPlatform(SourceHostPlatform, TargetPlatform);
AddRuleForBuildProducts(Filter, bp, GUBP.GamePlatformMonolithicsNode.StaticGetFullName(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform, Precompiled: bIsCodeTargetPlatform), FileFilterType.Include);
}
// Include the feature packs
foreach(string CurrentFeaturePack in CurrentFeaturePacks)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(CurrentFeaturePack);
Filter.AddRuleForFile(GUBP.MakeFeaturePacksNode.GetOutputFile(Project), CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
}
// Include all the templates
foreach (string Template in CurrentTemplates)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(Template);
Filter.Include("/" + Utils.StripBaseDirectory(Path.GetDirectoryName(Project.FilePath), CommandUtils.CmdEnv.LocalRoot).Replace('\\', '/') + "/...");
}
// Include all the standard rules
string RulesFileName = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini");
Filter.ReadRulesFromFile(RulesFileName, "CopyEditor", HostPlatform.ToString());
Filter.ReadRulesFromFile(RulesFileName, "CopyTargetPlatforms", HostPlatform.ToString());
// Custom rules for each target platform
foreach(UnrealTargetPlatform TargetPlaform in TargetPlatforms)
{
string SectionName = String.Format("CopyTargetPlatform.{0}", TargetPlaform.ToString());
Filter.ReadRulesFromFile(RulesFileName, SectionName, HostPlatform.ToString());
}
// Add the final exclusions for legal reasons.
Filter.ExcludeConfidentialPlatforms();
Filter.ExcludeConfidentialFolders();
// Run the filter on the stripped symbols, and remove those files from the copy filter
List<string> AllStrippedFiles = new List<string>();
foreach(KeyValuePair<string, string> StrippedNodeManifestPath in StrippedNodeManifestPaths)
{
List<string> StrippedFiles = new List<string>();
StripRocketNode StripNode = (StripRocketNode)bp.FindNode(StrippedNodeManifestPath.Key);
foreach(string BuildProduct in StripNode.BuildProducts)
{
if(Utils.IsFileUnderDirectory(BuildProduct, StripNode.StrippedDir))
{
string RelativePath = CommandUtils.StripBaseDirectory(Path.GetFullPath(BuildProduct), StripNode.StrippedDir);
if(Filter.Matches(RelativePath))
{
StrippedFiles.Add(RelativePath);
AllStrippedFiles.Add(RelativePath);
Filter.Exclude("/" + RelativePath);
}
}
}
WriteManifest(StrippedNodeManifestPath.Value, StrippedFiles);
BuildProducts.Add(StrippedNodeManifestPath.Value);
}
// Write the filtered list of depot files to disk, removing any symlinks
List<string> DepotFiles = Filter.ApplyToDirectory(CommandUtils.CmdEnv.LocalRoot, true).ToList();
WriteManifest(DepotManifestPath, DepotFiles);
BuildProducts.Add(DepotManifestPath);
// Sort the list of output files
SortedDictionary<string, bool> SortedFiles = new SortedDictionary<string,bool>(StringComparer.InvariantCultureIgnoreCase);
foreach(string DepotFile in DepotFiles)
{
SortedFiles.Add(DepotFile, false);
}
foreach(string StrippedFile in AllStrippedFiles)
{
SortedFiles.Add(StrippedFile, true);
}
// Write the list to the log
CommandUtils.Log("Files to be included in Rocket build:");
foreach(KeyValuePair<string, bool> SortedFile in SortedFiles)
{
CommandUtils.Log(" {0}{1}", SortedFile.Key, SortedFile.Value? " (stripped)" : "");
}
}
public override void DoBuild(GUBP bp)
{
// Create a filter for all the promoted binaries
FileFilter PromotableFilter = new FileFilter();
PromotableFilter.AddRuleForFiles(AllDependencyBuildProducts, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
PromotableFilter.ReadRulesFromFile(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, ConfigRelativePath), "promotable");
PromotableFilter.ExcludeConfidentialFolders();
// Copy everything that matches the filter to the promotion folder
string PromotableFolder = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Saved", "GitPromotable");
CommandUtils.DeleteDirectoryContents(PromotableFolder);
string[] PromotableFiles = CommandUtils.ThreadedCopyFiles(CommandUtils.CmdEnv.LocalRoot, PromotableFolder, PromotableFilter, bIgnoreSymlinks: true);
BuildProducts = new List<string>(PromotableFiles);
}
public override void DoBuild(GUBP bp)
{
BuildProducts = new List<string>();
string InputDir = Path.GetFullPath(CommandUtils.CmdEnv.LocalRoot);
string RulesFileName = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini");
// Read the filter for files on this platform
FileFilter StripFilter = new FileFilter();
StripFilter.ReadRulesFromFile(RulesFileName, "StripSymbols." + TargetPlatform.ToString(), HostPlatform.ToString());
// Apply the filter to the build products
List<string> SourcePaths = new List<string>();
List<string> TargetPaths = new List<string>();
foreach(string NodeToStrip in NodesToStrip)
{
GUBP.GUBPNode Node = bp.FindNode(NodeToStrip);
foreach(string DependencyBuildProduct in Node.BuildProducts)
{
string RelativePath = CommandUtils.StripBaseDirectory(Path.GetFullPath(DependencyBuildProduct), InputDir);
if(StripFilter.Matches(RelativePath))
{
SourcePaths.Add(CommandUtils.CombinePaths(InputDir, RelativePath));
TargetPaths.Add(CommandUtils.CombinePaths(StrippedDir, RelativePath));
}
}
}
// Strip the files and add them to the build products
StripSymbols(TargetPlatform, SourcePaths.ToArray(), TargetPaths.ToArray());
BuildProducts.AddRange(TargetPaths);
SaveRecordOfSuccessAndAddToBuildProducts();
}
public override void ExecuteBuild()
{
DLCName = ParseParamValue("Plugin", "");
DLCDir = CombinePaths(CmdEnv.LocalRoot, "UnrealTournament", "Plugins", DLCName);
PluginsDir = CombinePaths(CmdEnv.LocalRoot, "UnrealTournament", "Plugins");
PublishDir = CombinePaths(DLCDir, "Releases");
// Right now all platform asset registries seem to be the exact same, this may change in the future
AssetRegistry = ParseParamValue("ReleaseVersion", "UTVersion0");
VersionString = ParseParamValue("Version", "NOVERSION");
FileVersionName = VERSION_NAME;
// prevent processing automation process for the wrong plugin (if mutliple PluginPackageZip commands exist)
if (!DLCName.Equals(PLUGIN_NAME, StringComparison.InvariantCultureIgnoreCase))
{
// no Expception as other plugins need to be processed
return;
}
// Check parms
if (!DirectoryExists(DLCDir))
{
throw new AutomationException("Plugin folder '{0}' does not exist. Verify that '-plugin={1}' is set up properly.", DLCDir, DLCName);
}
if (new DirectoryInfo(PluginsDir).FullName.Equals(new DirectoryInfo(DLCDir).FullName, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("No Plugin given. Verify that '-plugin=PLUGINNAME' is set.");
}
// Check for changed files in the git repo and abort
if (!ParseParam("NoGitCheck"))
{
string GitFolder = CombinePaths(DLCDir, GIT_FOLDER_NAME);
if (DirectoryExists(GitFolder))
{
using (var repo = new LibGit2Sharp.Repository(GitFolder))
{
LibGit2Sharp.StatusOptions check = new LibGit2Sharp.StatusOptions();
check.ExcludeSubmodules = true;
LibGit2Sharp.RepositoryStatus status = repo.RetrieveStatus(check);
if (status.IsDirty)
{
throw new AutomationException("Git code repository has uncommited changes.");
}
foreach (LibGit2Sharp.Submodule submodule in repo.Submodules)
{
LibGit2Sharp.SubmoduleStatus substatus = submodule.RetrieveStatus();
if (!substatus.HasFlag(LibGit2Sharp.SubmoduleStatus.Unmodified))
{
throw new AutomationException("Git sub module '{0}' has uncommited changes.", submodule.Name);
}
}
}
}
}
// Check if binaries exist
{
string BinariesFolder = CombinePaths(DLCDir, "Binaries");
if (!DirectoryExists(BinariesFolder))
{
throw new AutomationException("No Binaries folder. Please, recompile the plugin from source files");
}
// TODO: Check for specific files (SO for linux and DLL for windows)
if (FindFiles("*", true, BinariesFolder).Length == 0)
{
throw new AutomationException("No binary files. Please, recompile the plugin from source files");
}
}
//
// Create filter list to exclude folder/files from zip
//
FileFilter Filter = new FileFilter();
Filter.ExcludeConfidentialFolders();
Filter.ExcludeConfidentialPlatforms();
// read git ignore file and add these to the filder
var FilterInclude = new List<string>();
var FilterExclude = new List<string>();
{
string GitFolder = CombinePaths(DLCDir, GIT_FOLDER_NAME);
if (DirectoryExists(GitFolder))
{
using (var repo = new LibGit2Sharp.Repository(GitFolder))
{
var subrepos = new List<LibGit2Sharp.Repository>();
foreach (LibGit2Sharp.Submodule submodule in repo.Submodules)
{
string SubFolder = CombinePaths(DLCDir, submodule.Path);
if (DirectoryExists(SubFolder))
{
string relpath = SubFolder.Substring(DLCDir.Length).Replace("\\", "/").Trim('/');
FilterExclude.Add("/" + relpath + "/**");
subrepos.Add(new LibGit2Sharp.Repository(SubFolder));
}
}
foreach (var file in FindFiles("*", true, DLCDir))
{
bool subuntracked = false;
foreach (var subrepo in subrepos)
{
if (file.StartsWith(subrepo.Info.WorkingDirectory))
{
LibGit2Sharp.FileStatus substatus = subrepo.RetrieveStatus(file);
if (substatus == LibGit2Sharp.FileStatus.Untracked)
{
subuntracked = true;
string relpath = file.Substring(DLCDir.Length).Replace("\\", "/").TrimStart('/');
FilterInclude.Add(relpath);
break;
}
}
}
if (!subuntracked)
{
LibGit2Sharp.FileStatus status = repo.RetrieveStatus(file);
if (status == LibGit2Sharp.FileStatus.Untracked)
{
string relpath = file.Substring(DLCDir.Length).Replace("\\", "/").TrimStart('/');
FilterInclude.Add(relpath);
}
}
}
foreach (var subrepo in subrepos)
{
subrepo.Dispose();
}
subrepos.Clear();
}
}
string GitIgnoreFile = CombinePaths(DLCDir, GIT_IGNORE_FILE);
if (FileExists(GitIgnoreFile))
{
//HasGitIgnore = true;
var GitIgnores = GitIgnoreParser.ReadFromFile(GitIgnoreFile);
foreach (var entry in GitIgnores.Entries)
{
string pattern = entry.Pattern;
if (entry.Pattern.EndsWith("/")) pattern += "**";
if (entry.Type == GitIgnoreParser.GitIgnoreTypInfo.Include)
{
FilterInclude.Add(pattern);
}
else if (entry.Type == GitIgnoreParser.GitIgnoreTypInfo.Exclude)
{
FilterExclude.Add(pattern);
}
}
}
}
// TODO: Add build file for includes
// include bin and plugin content but only valid editor files
foreach (var f in FilterExclude)
{
Filter.Include(f.Replace("**", "..."));
}
if (FilterInclude.Count == 0)
{
Filter.Include("Readme*");
Filter.Include("*.txt");
Filter.Include("*.md");
Filter.Include("*.pod");
}
else
{
Filter.Include("/*.*");
}
// Add source if required
if (ParseParam("source"))
{
Filter.Include("/Source/...");
Filter.Include("/Build/...");
Filter.Exclude("/Build/.../Obj/...");
}
else
{
Filter.Exclude("/Build/...");
}
foreach (var f in FilterInclude)
{
Filter.Exclude(f.Replace("**", "..."));
}
Filter.Include("/Binaries/...");
// ignore debug symbols
Filter.Exclude("/Binaries/.../*.pdb");
// ignore git files
Filter.Exclude(".git*");
Filter.Exclude(".git/...");
// always include plugin descriptor file
Filter.Include("*.uplugin");
// exclude releaes foldre
Filter.Exclude("/Releases/...");
Filter.Exclude("/Releases/.../...");
// read current version info from file
CreateVersionInfo();
// create meta info which is stored in BuildMeta
CreateMetaInfo();
// create file name
string ZipFileName = GenerateFileName() + ".zip";
string ZipFile = CombinePaths(PublishDir, ZipFileName);
string ZipDir = CombinePaths(CmdEnv.LocalRoot, "UnrealTournament", "Plugins", DLCName);
// check if file exists and abort...
if (FileExists(ZipFile) && !ParseParam("overwrite"))
{
throw new AutomationException("Release file {0} already exists.", ZipFileName);
}
string zipfolder = Packager.CopyFilesToZip(PublishDir, ZipDir, DLCName, Filter, NoConversion: HasParam("source"), ParseTextOnly: HasParam("ParseTextOnly"));
ZipFiles(ZipFile, zipfolder, new FileFilter(FileFilterType.Include));
if (ParseParam("publish") && BuildVersion.IsValid())
{
bool IsInternal = false;
if (ParseParam("release"))
{
// remove internal build (public release) ... just in case
DataWriteToFile(FileVersionName, BuildVersion.ToData(true));
}
else
{
IsInternal = true;
// increase internal build
BuildVersion.BuildAsNumber += 1;
DataWriteToFile(FileVersionName, BuildVersion.ToData());
}
string GitFolder = CombinePaths(DLCDir, GIT_FOLDER_NAME);
if (ParseParam("commit") && DirectoryExists(GitFolder))
{
using (var repo = new LibGit2Sharp.Repository(GitFolder))
{
string commmessage;
if (IsInternal)
commmessage = string.Format(COMMIT_TITLE_VERSION_INTERNAL, BuildVersion.ToString());
else
commmessage = string.Format(COMMIT_TITLE_VERSION_PUBLISH, BuildVersion.ToShortVersion());
commmessage += Environment.NewLine;
commmessage += Environment.NewLine;
commmessage += BuildMeta.ToExtendedString();
// Stage the file
LibGit2Sharp.StageOptions stageoptions = new LibGit2Sharp.StageOptions();
repo.Stage(FileVersionName, stageoptions);
// Create the committer's signature and commit
LibGit2Sharp.Signature author = repo.Config.BuildSignature(DateTime.Now);
LibGit2Sharp.Signature committer = author;
// Commit to the repository
LibGit2Sharp.CommitOptions options = new LibGit2Sharp.CommitOptions();
LibGit2Sharp.Commit commit = repo.Commit(commmessage, author, committer, options);
// Commit new tag
LibGit2Sharp.Tag t = repo.Tags.Add(string.Format("v{0}", BuildVersion.ToShortVersion()), repo.Head.Tip);
}
}
}
}
/// <summary>
/// Resolve a list of files, tag names or file specifications as above, but preserves any directory references for further processing.
/// </summary>
/// <param name="DefaultDirectory">The default directory to resolve relative paths to</param>
/// <param name="FilePatterns">List of files, tag names, or file specifications to include separated by semicolons.</param>
/// <param name="ExcludePatterns">Set of patterns to apply to directory searches. This can greatly speed up enumeration by earlying out of recursive directory searches if large directories are excluded (eg. .../Intermediate/...).</param>
/// <param name="TagNameToFileSet">Mapping of tag name to fileset, as passed to the Execute() method</param>
/// <returns>Set of matching files.</returns>
public static HashSet<FileReference> ResolveFilespecWithExcludePatterns(DirectoryReference DefaultDirectory, List<string> FilePatterns, List<string> ExcludePatterns, Dictionary<string, HashSet<FileReference>> TagNameToFileSet)
{
// Parse each of the patterns, and add the results into the given sets
HashSet<FileReference> Files = new HashSet<FileReference>();
foreach(string Pattern in FilePatterns)
{
// Check if it's a tag name
if(Pattern.StartsWith("#"))
{
Files.UnionWith(FindOrAddTagSet(TagNameToFileSet, Pattern));
continue;
}
// If it doesn't contain any wildcards, just add the pattern directly
int WildcardIdx = FileFilter.FindWildcardIndex(Pattern);
if(WildcardIdx == -1)
{
Files.Add(FileReference.Combine(DefaultDirectory, Pattern));
continue;
}
// Find the base directory for the search. We construct this in a very deliberate way including the directory separator itself, so matches
// against the OS root directory will resolve correctly both on Mac (where / is the filesystem root) and Windows (where / refers to the current drive).
int LastDirectoryIdx = Pattern.LastIndexOfAny(new char[]{ Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, WildcardIdx);
DirectoryReference BaseDir = DirectoryReference.Combine(DefaultDirectory, Pattern.Substring(0, LastDirectoryIdx + 1));
// Construct the absolute include pattern to match against, re-inserting the resolved base directory to construct a canonical path.
string IncludePattern = BaseDir.FullName.TrimEnd(new char[]{ Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }) + "/" + Pattern.Substring(LastDirectoryIdx + 1);
// Construct a filter and apply it to the directory
if(BaseDir.Exists())
{
FileFilter Filter = new FileFilter();
Filter.AddRule(IncludePattern, FileFilterType.Include);
Filter.AddRules(ExcludePatterns, FileFilterType.Exclude);
Files.UnionWith(Filter.ApplyToDirectory(BaseDir, BaseDir.FullName, true));
}
}
// If we have exclude rules, create and run a filter against all the output files to catch things that weren't added from an include
if(ExcludePatterns.Count > 0)
{
FileFilter Filter = new FileFilter(FileFilterType.Include);
Filter.AddRules(ExcludePatterns, FileFilterType.Exclude);
Files.RemoveWhere(x => !Filter.Matches(x.FullName));
}
return Files;
}
public override void DoBuild(GUBP bp)
{
BuildProducts = new List<string>();
string InputDir = Path.GetFullPath(CommandUtils.CmdEnv.LocalRoot);
// Read the filter for files on this platform
FileFilter SignFilter = new FileFilter();
if (HostPlatform == UnrealTargetPlatform.Mac)
{
SignFilter.AddRule("*.dylib");
SignFilter.AddRule("*.app");
}
else
{
SignFilter.AddRule("*.exe");
SignFilter.AddRule("*.dll");
}
// Apply the filter to the build products
List<string> SourcePaths = new List<string>();
List<string> TargetPaths = new List<string>();
foreach (string NodeToSign in NodesToSign)
{
GUBP.GUBPNode Node = BranchConfig.FindNode(NodeToSign);
foreach (string DependencyBuildProduct in Node.BuildProducts)
{
string RelativePath = CommandUtils.StripBaseDirectory(Path.GetFullPath(DependencyBuildProduct), InputDir);
if (SignFilter.Matches(RelativePath))
{
SourcePaths.Add(CommandUtils.CombinePaths(InputDir, RelativePath));
TargetPaths.Add(CommandUtils.CombinePaths(SignedDir, RelativePath));
}
}
}
// Strip the files and add them to the build products
SignFiles(bp, SourcePaths.ToArray(), TargetPaths.ToArray());
BuildProducts.AddRange(TargetPaths);
SaveRecordOfSuccessAndAddToBuildProducts();
}
public static string CopyFilesToZip(string ZipBaseFolder, string BaseDirectory, string InZipDir, FileFilter Filter, bool NoConversion, bool ParseTextOnly = false)
{
string TempFolder = GetRandomFolder(ZipBaseFolder);
string CopyFolder = Path.Combine(TempFolder, InZipDir);
CreateDirectory(CopyFolder);
foreach (string FilteredFile in Filter.ApplyToDirectory(BaseDirectory, true))
{
string srcf = Path.Combine(BaseDirectory, FilteredFile);
string destfo = Path.GetDirectoryName(FilteredFile);
string destfi = Path.Combine(CopyFolder, FilteredFile);
if (!NoConversion && Path.GetExtension(FilteredFile).Equals(".pod", StringComparison.InvariantCultureIgnoreCase))
{
var reader = new StreamReader(srcf);
string content = reader.ReadToEnd();
reader.Dispose();
var parser = Parser.BaseParser.Create<Parser.POD.PODParser>(new StreamReader(srcf));
if (ParseTextOnly)
{
using (var writer = new StreamWriter(destfi + ".txt"))
{
writer.WriteLine(parser.Text());
}
}
else
{
using (var writer = new StreamWriter(destfi + ".htm"))
{
writer.WriteLine(parser.Flatten());
}
}
}
else if (!NoConversion && Path.GetExtension(FilteredFile).Equals(".md", StringComparison.InvariantCultureIgnoreCase))
{
var m = new MarkdownSharp.Markdown();
var reader = new StreamReader(srcf);
string content = reader.ReadToEnd();
reader.Dispose();
var UrlEvaluator = new MatchEvaluator(delegate(Match match)
{
string outstr = match.ToString();
if (match.Groups.Count == 2)
{
string relfile = match.Groups[1].Value;
string MailPattern = @"^(?("")("".+?(?<!\\)""@)|(([0-9a-z]((\.(?!\.))|[-!#\$%&'\*\+/=\?\^`\{\}\|~\w])*)(?<=[0-9a-z])@))";
string DomainPattern = @"(?(\[)(\[(\d{1,3}\.){3}\d{1,3}\])|(([0-9a-z][-\w]*[0-9a-z]*\.)+[a-z0-9][\-a-z0-9]{0,22}[a-z0-9]))";
if (!relfile.Contains("://") && !Regex.Match(relfile, MailPattern + DomainPattern).Success)
{
if (relfile.StartsWith("//"))
{
relfile = "https:" + relfile;
}
else if (relfile.StartsWith("/../../") || relfile.StartsWith("../../"))
{
relfile = relfile.Replace("../../", "");
relfile = URL_REPO_BASE + "/" + relfile.TrimStart('/');
}
else
{
string[] splits = relfile.Split("?|#&".ToCharArray());
if (Regex.Match(relfile, DomainPattern).Success && !File.Exists(Path.Combine(BaseDirectory, relfile)))
{
relfile = "https:" + relfile;
}
else if (splits.Length > 1)
{
if (splits[0].Length >= 260 || !File.Exists(Path.Combine(BaseDirectory, splits[0])))
{
relfile = URL_REPO_BASE + splits[0];
}
}
else
{
string p = Path.Combine(BaseDirectory, relfile);
if (!File.Exists(p) && !Directory.Exists(p))
{
relfile = URL_REPO_BASE + "/" + splits[0].TrimStart('/');
}
}
}
}
outstr = match.Groups[0].Value.Replace(match.Groups[1].Value, relfile);
}
return outstr;
});
var FileEvaluator = new MatchEvaluator(delegate(Match match)
{
string outstr = match.ToString();
if (match.Groups.Count == 6)
{
string seps = "?|#&";
string filename = match.Groups[3].Value;
string[] splits = filename.Split(seps.ToCharArray());
if (splits.Length > 0 && File.Exists(Path.Combine(BaseDirectory, splits[0])))
{
string ext = match.Groups[4].Value;
if (splits.Length > 1) ext = ext.Split(seps.ToCharArray())[0];
filename = filename.Replace(splits[0], splits[0] + ".htm");
}
outstr = string.Format("[{0}{1}{2}{3}", match.Groups[1].Value, match.Groups[2].Value, filename, match.Groups[5].Value);
}
return outstr;
});
if (ParseTextOnly)
{
m.OnlyText = true;
m.AutoHyperlink = false;
content = m.Transform(content);
using (var writer = new StreamWriter(destfi + ".txt"))
{
writer.WriteLine(content.Replace("\n", "\r\n"));
}
}
else
{
// fix relative links
content = Regex.Replace(content, @"\[.*?\]\((.*?)\)", UrlEvaluator);
content = Regex.Replace(content, @"\[\s*[a-zA-Z0-9_-]+\s*\]\s*:\s*(\S+)\s*", UrlEvaluator);
// redirect files in HTML
content = Regex.Replace(content, @"\[(.*?)(\]\()(.*\.(pod|md)\S*)(\))", FileEvaluator, RegexOptions.IgnoreCase);
content = Regex.Replace(content, @"\[(\s*[a-zA-Z0-9_-]+\s*)(\]\s*:\s*)(.*\.(pod|md)\S*)(\s*)", FileEvaluator);
using (var writer = new StreamWriter(destfi + ".htm"))
{
writer.WriteLine(m.Transform(content));
}
}
}
else
{
CopyFile(srcf, destfi);
}
}
return TempFolder;
}
