public override void DoBuild(GUBP bp)
{
// Create a zip file containing the install
string FullZipFileName = Path.Combine(CommandUtils.CmdEnv.LocalRoot, "FullInstall" + StaticGetHostPlatformSuffix(HostPlatform) + ".zip");
CommandUtils.Log("Creating {0}...", FullZipFileName);
CommandUtils.ZipFiles(FullZipFileName, LocalDir, new FileFilter(FileFilterType.Include));
// Create a filter for the files we need just to run the editor
FileFilter EditorFilter = new FileFilter(FileFilterType.Include);
EditorFilter.Exclude("/Engine/Binaries/...");
EditorFilter.Include("/Engine/Binaries/DotNET/...");
EditorFilter.Include("/Engine/Binaries/ThirdParty/...");
EditorFilter.Include("/Engine/Binaries/" + HostPlatform.ToString() + "/...");
EditorFilter.Exclude("/Engine/Binaries/.../*.lib");
EditorFilter.Exclude("/Engine/Binaries/.../*.a");
EditorFilter.Exclude("/Engine/Extras/...");
EditorFilter.Exclude("/Engine/Source/.../Private/...");
EditorFilter.Exclude("/FeaturePacks/...");
EditorFilter.Exclude("/Samples/...");
EditorFilter.Exclude("/Templates/...");
EditorFilter.Exclude("*.pdb");
// Create a zip file containing the editor install
string EditorZipFileName = Path.Combine(CommandUtils.CmdEnv.LocalRoot, "EditorInstall" + StaticGetHostPlatformSuffix(HostPlatform) + ".zip");
CommandUtils.Log("Creating {0}...", EditorZipFileName);
CommandUtils.ZipFiles(EditorZipFileName, LocalDir, EditorFilter);
// Copy the files to their final location
CommandUtils.Log("Copying files to {0}", PublishDir);
CommandUtils.CopyFile(FullZipFileName, Path.Combine(PublishDir, Path.GetFileName(FullZipFileName)));
CommandUtils.CopyFile(EditorZipFileName, Path.Combine(PublishDir, Path.GetFileName(EditorZipFileName)));
CommandUtils.DeleteFile(FullZipFileName);
CommandUtils.DeleteFile(EditorZipFileName);
// Save a record of success
BuildProducts = new List<string>();
SaveRecordOfSuccessAndAddToBuildProducts();
}
public override void DoBuild(GUBP bp)
{
BuildProducts = new List<string>();
FileFilter Filter = new FileFilter();
// Include all the editor products
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsForCompileNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.RootEditorNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsNode.StaticGetFullName(HostPlatform), FileFilterType.Include);
// Include win64 tools on Mac, to get the win64 build of UBT, UAT and IPP
if (HostPlatform == UnrealTargetPlatform.Mac && bp.HostPlatforms.Contains(UnrealTargetPlatform.Win64))
{
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsNode.StaticGetFullName(UnrealTargetPlatform.Win64), FileFilterType.Include);
AddRuleForBuildProducts(Filter, bp, GUBP.ToolsForCompileNode.StaticGetFullName(UnrealTargetPlatform.Win64), FileFilterType.Include);
}
// Include the editor headers
UnzipAndAddRuleForHeaders(GUBP.RootEditorNode.StaticGetArchivedHeadersPath(HostPlatform), Filter, FileFilterType.Include);
// Include the build dependencies for every code platform
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
if(RocketBuild.IsCodeTargetPlatform(HostPlatform, TargetPlatform))
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
string FileListPath = GUBP.GamePlatformMonolithicsNode.StaticGetBuildDependenciesPath(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform);
Filter.AddRuleForFiles(UnrealBuildTool.Utils.ReadClass<UnrealBuildTool.ExternalFileList>(FileListPath).FileNames, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
UnzipAndAddRuleForHeaders(GUBP.GamePlatformMonolithicsNode.StaticGetArchivedHeadersPath(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform), Filter, FileFilterType.Include);
}
}
// Add the monolithic binaries
foreach(UnrealTargetPlatform TargetPlatform in TargetPlatforms)
{
UnrealTargetPlatform SourceHostPlatform = RocketBuild.GetSourceHostPlatform(bp, HostPlatform, TargetPlatform);
bool bIsCodeTargetPlatform = RocketBuild.IsCodeTargetPlatform(SourceHostPlatform, TargetPlatform);
AddRuleForBuildProducts(Filter, bp, GUBP.GamePlatformMonolithicsNode.StaticGetFullName(SourceHostPlatform, bp.Branch.BaseEngineProject, TargetPlatform, Precompiled: bIsCodeTargetPlatform), FileFilterType.Include);
}
// Include the feature packs
foreach(string CurrentFeaturePack in CurrentFeaturePacks)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(CurrentFeaturePack);
Filter.AddRuleForFile(GUBP.MakeFeaturePacksNode.GetOutputFile(Project), CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
}
// Include all the templates
foreach (string Template in CurrentTemplates)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(Template);
Filter.Include("/" + Utils.StripBaseDirectory(Path.GetDirectoryName(Project.FilePath), CommandUtils.CmdEnv.LocalRoot).Replace('\\', '/') + "/...");
}
// Include all the standard rules
string RulesFileName = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini");
Filter.ReadRulesFromFile(RulesFileName, "CopyEditor", HostPlatform.ToString());
Filter.ReadRulesFromFile(RulesFileName, "CopyTargetPlatforms", HostPlatform.ToString());
// Custom rules for each target platform
foreach(UnrealTargetPlatform TargetPlaform in TargetPlatforms)
{
string SectionName = String.Format("CopyTargetPlatform.{0}", TargetPlaform.ToString());
Filter.ReadRulesFromFile(RulesFileName, SectionName, HostPlatform.ToString());
}
// Add the final exclusions for legal reasons.
Filter.ExcludeConfidentialPlatforms();
Filter.ExcludeConfidentialFolders();
// Run the filter on the stripped symbols, and remove those files from the copy filter
List<string> AllStrippedFiles = new List<string>();
foreach(KeyValuePair<string, string> StrippedNodeManifestPath in StrippedNodeManifestPaths)
{
List<string> StrippedFiles = new List<string>();
StripRocketNode StripNode = (StripRocketNode)bp.FindNode(StrippedNodeManifestPath.Key);
foreach(string BuildProduct in StripNode.BuildProducts)
{
if(Utils.IsFileUnderDirectory(BuildProduct, StripNode.StrippedDir))
{
string RelativePath = CommandUtils.StripBaseDirectory(Path.GetFullPath(BuildProduct), StripNode.StrippedDir);
if(Filter.Matches(RelativePath))
{
StrippedFiles.Add(RelativePath);
AllStrippedFiles.Add(RelativePath);
Filter.Exclude("/" + RelativePath);
}
}
}
WriteManifest(StrippedNodeManifestPath.Value, StrippedFiles);
BuildProducts.Add(StrippedNodeManifestPath.Value);
}
// Write the filtered list of depot files to disk, removing any symlinks
List<string> DepotFiles = Filter.ApplyToDirectory(CommandUtils.CmdEnv.LocalRoot, true).ToList();
WriteManifest(DepotManifestPath, DepotFiles);
BuildProducts.Add(DepotManifestPath);
// Sort the list of output files
SortedDictionary<string, bool> SortedFiles = new SortedDictionary<string,bool>(StringComparer.InvariantCultureIgnoreCase);
foreach(string DepotFile in DepotFiles)
{
SortedFiles.Add(DepotFile, false);
}
foreach(string StrippedFile in AllStrippedFiles)
{
SortedFiles.Add(StrippedFile, true);
}
// Write the list to the log
CommandUtils.Log("Files to be included in Rocket build:");
foreach(KeyValuePair<string, bool> SortedFile in SortedFiles)
{
CommandUtils.Log(" {0}{1}", SortedFile.Key, SortedFile.Value? " (stripped)" : "");
}
}
public override void DoBuild(GUBP bp)
{
CommandUtils.CreateDirectory(SavedDir);
BuildProducts = new List<string>();
List<string> ManifestFiles = new List<string>();
if(!bp.ParseParam("NoDDC"))
{
// Find all the projects we're interested in
List<BranchInfo.BranchUProject> Projects = new List<BranchInfo.BranchUProject>();
foreach(string ProjectName in ProjectNames)
{
BranchInfo.BranchUProject Project = bp.Branch.FindGameChecked(ProjectName);
if(!Project.Properties.bIsCodeBasedProject)
{
Projects.Add(Project);
}
}
// Filter out the files we need to build DDC. Removing confidential folders can affect DDC keys, so we want to be sure that we're making DDC with a build that can use it.
FileFilter Filter = new FileFilter(FileFilterType.Exclude);
Filter.AddRuleForFiles(AllDependencyBuildProducts, CommandUtils.CmdEnv.LocalRoot, FileFilterType.Include);
Filter.ReadRulesFromFile(CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "Engine", "Build", "InstalledEngineFilters.ini"), "CopyEditor", HostPlatform.ToString());
Filter.Exclude("/Engine/Build/...");
Filter.Exclude("/Engine/Extras/...");
Filter.Exclude("/Engine/DerivedDataCache/...");
Filter.Exclude("/Samples/...");
Filter.Exclude("/Templates/...");
Filter.Exclude(".../Source/...");
Filter.Exclude(".../Intermediate/...");
Filter.ExcludeConfidentialPlatforms();
Filter.ExcludeConfidentialFolders();
Filter.Include("/Engine/Build/NotForLicensees/EpicInternal.txt");
Filter.Include("/Engine/Binaries/.../*DDCUtils*"); // Make sure we can use the shared DDC!
// Copy everything to a temporary directory
string TempDir = CommandUtils.CombinePaths(CommandUtils.CmdEnv.LocalRoot, "LocalBuilds", "RocketDDC", CommandUtils.GetGenericPlatformName(HostPlatform));
CommandUtils.DeleteDirectoryContents(TempDir);
CommandUtils.ThreadedCopyFiles(CommandUtils.CmdEnv.LocalRoot, TempDir, Filter, true);
// Get paths to everything within the temporary directory
string EditorExe = CommandUtils.GetEditorCommandletExe(TempDir, HostPlatform);
string RelativePakPath = "Engine/DerivedDataCache/Compressed.ddp";
string OutputPakFile = CommandUtils.CombinePaths(TempDir, RelativePakPath);
string OutputCsvFile = Path.ChangeExtension(OutputPakFile, ".csv");
// Generate DDC for all the non-code projects. We don't necessarily have editor DLLs for the code projects, but they should be the same as their blueprint counterparts.
List<string> ProjectPakFiles = new List<string>();
foreach(BranchInfo.BranchUProject Project in Projects)
{
CommandUtils.Log("Generating DDC data for {0} on {1}", Project.GameName, TargetPlatforms);
CommandUtils.DDCCommandlet(Project.FilePath, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak -ProjectOnly");
string ProjectPakFile = CommandUtils.CombinePaths(Path.GetDirectoryName(OutputPakFile), String.Format("Compressed-{0}.ddp", Project.GameName));
CommandUtils.DeleteFile(ProjectPakFile);
CommandUtils.RenameFile(OutputPakFile, ProjectPakFile);
string ProjectCsvFile = Path.ChangeExtension(ProjectPakFile, ".csv");
CommandUtils.DeleteFile(ProjectCsvFile);
CommandUtils.RenameFile(OutputCsvFile, ProjectCsvFile);
ProjectPakFiles.Add(Path.GetFileName(ProjectPakFile));
}
// Generate DDC for the editor, and merge all the other PAK files in
CommandUtils.Log("Generating DDC data for engine content on {0}", TargetPlatforms);
CommandUtils.DDCCommandlet(null, EditorExe, null, TargetPlatforms, "-fill -DDC=CreateInstalledEnginePak " + CommandUtils.MakePathSafeToUseWithCommandLine("-MergePaks=" + String.Join("+", ProjectPakFiles)));
// Copy the DDP file to the output path
string SavedPakFile = CommandUtils.CombinePaths(SavedDir, RelativePakPath);
CommandUtils.CopyFile(OutputPakFile, SavedPakFile);
BuildProducts.Add(SavedPakFile);
// Add the pak file to the list of files to copy
ManifestFiles.Add(RelativePakPath);
}
CommandUtils.WriteAllLines(SavedManifestPath, ManifestFiles.ToArray());
BuildProducts.Add(SavedManifestPath);
SaveRecordOfSuccessAndAddToBuildProducts();
}
public override void ExecuteBuild()
{
DLCName = ParseParamValue("Plugin", "");
DLCDir = CombinePaths(CmdEnv.LocalRoot, "UnrealTournament", "Plugins", DLCName);
PluginsDir = CombinePaths(CmdEnv.LocalRoot, "UnrealTournament", "Plugins");
PublishDir = CombinePaths(DLCDir, "Releases");
// Right now all platform asset registries seem to be the exact same, this may change in the future
AssetRegistry = ParseParamValue("ReleaseVersion", "UTVersion0");
VersionString = ParseParamValue("Version", "NOVERSION");
FileVersionName = VERSION_NAME;
// prevent processing automation process for the wrong plugin (if mutliple PluginPackageZip commands exist)
if (!DLCName.Equals(PLUGIN_NAME, StringComparison.InvariantCultureIgnoreCase))
{
// no Expception as other plugins need to be processed
return;
}
// Check parms
if (!DirectoryExists(DLCDir))
{
throw new AutomationException("Plugin folder '{0}' does not exist. Verify that '-plugin={1}' is set up properly.", DLCDir, DLCName);
}
if (new DirectoryInfo(PluginsDir).FullName.Equals(new DirectoryInfo(DLCDir).FullName, StringComparison.InvariantCultureIgnoreCase))
{
throw new AutomationException("No Plugin given. Verify that '-plugin=PLUGINNAME' is set.");
}
// Check for changed files in the git repo and abort
if (!ParseParam("NoGitCheck"))
{
string GitFolder = CombinePaths(DLCDir, GIT_FOLDER_NAME);
if (DirectoryExists(GitFolder))
{
using (var repo = new LibGit2Sharp.Repository(GitFolder))
{
LibGit2Sharp.StatusOptions check = new LibGit2Sharp.StatusOptions();
check.ExcludeSubmodules = true;
LibGit2Sharp.RepositoryStatus status = repo.RetrieveStatus(check);
if (status.IsDirty)
{
throw new AutomationException("Git code repository has uncommited changes.");
}
foreach (LibGit2Sharp.Submodule submodule in repo.Submodules)
{
LibGit2Sharp.SubmoduleStatus substatus = submodule.RetrieveStatus();
if (!substatus.HasFlag(LibGit2Sharp.SubmoduleStatus.Unmodified))
{
throw new AutomationException("Git sub module '{0}' has uncommited changes.", submodule.Name);
}
}
}
}
}
// Check if binaries exist
{
string BinariesFolder = CombinePaths(DLCDir, "Binaries");
if (!DirectoryExists(BinariesFolder))
{
throw new AutomationException("No Binaries folder. Please, recompile the plugin from source files");
}
// TODO: Check for specific files (SO for linux and DLL for windows)
if (FindFiles("*", true, BinariesFolder).Length == 0)
{
throw new AutomationException("No binary files. Please, recompile the plugin from source files");
}
}
//
// Create filter list to exclude folder/files from zip
//
FileFilter Filter = new FileFilter();
Filter.ExcludeConfidentialFolders();
Filter.ExcludeConfidentialPlatforms();
// read git ignore file and add these to the filder
var FilterInclude = new List<string>();
var FilterExclude = new List<string>();
{
string GitFolder = CombinePaths(DLCDir, GIT_FOLDER_NAME);
if (DirectoryExists(GitFolder))
{
using (var repo = new LibGit2Sharp.Repository(GitFolder))
{
var subrepos = new List<LibGit2Sharp.Repository>();
foreach (LibGit2Sharp.Submodule submodule in repo.Submodules)
{
string SubFolder = CombinePaths(DLCDir, submodule.Path);
if (DirectoryExists(SubFolder))
{
string relpath = SubFolder.Substring(DLCDir.Length).Replace("\\", "/").Trim('/');
FilterExclude.Add("/" + relpath + "/**");
subrepos.Add(new LibGit2Sharp.Repository(SubFolder));
}
}
foreach (var file in FindFiles("*", true, DLCDir))
{
bool subuntracked = false;
foreach (var subrepo in subrepos)
{
if (file.StartsWith(subrepo.Info.WorkingDirectory))
{
LibGit2Sharp.FileStatus substatus = subrepo.RetrieveStatus(file);
if (substatus == LibGit2Sharp.FileStatus.Untracked)
{
subuntracked = true;
string relpath = file.Substring(DLCDir.Length).Replace("\\", "/").TrimStart('/');
FilterInclude.Add(relpath);
break;
}
}
}
if (!subuntracked)
{
LibGit2Sharp.FileStatus status = repo.RetrieveStatus(file);
if (status == LibGit2Sharp.FileStatus.Untracked)
{
string relpath = file.Substring(DLCDir.Length).Replace("\\", "/").TrimStart('/');
FilterInclude.Add(relpath);
}
}
}
foreach (var subrepo in subrepos)
{
subrepo.Dispose();
}
subrepos.Clear();
}
}
string GitIgnoreFile = CombinePaths(DLCDir, GIT_IGNORE_FILE);
if (FileExists(GitIgnoreFile))
{
//HasGitIgnore = true;
var GitIgnores = GitIgnoreParser.ReadFromFile(GitIgnoreFile);
foreach (var entry in GitIgnores.Entries)
{
string pattern = entry.Pattern;
if (entry.Pattern.EndsWith("/")) pattern += "**";
if (entry.Type == GitIgnoreParser.GitIgnoreTypInfo.Include)
{
FilterInclude.Add(pattern);
}
else if (entry.Type == GitIgnoreParser.GitIgnoreTypInfo.Exclude)
{
FilterExclude.Add(pattern);
}
}
}
}
// TODO: Add build file for includes
// include bin and plugin content but only valid editor files
foreach (var f in FilterExclude)
{
Filter.Include(f.Replace("**", "..."));
}
if (FilterInclude.Count == 0)
{
Filter.Include("Readme*");
Filter.Include("*.txt");
Filter.Include("*.md");
Filter.Include("*.pod");
}
else
{
Filter.Include("/*.*");
}
// Add source if required
if (ParseParam("source"))
{
Filter.Include("/Source/...");
Filter.Include("/Build/...");
Filter.Exclude("/Build/.../Obj/...");
}
else
{
Filter.Exclude("/Build/...");
}
foreach (var f in FilterInclude)
{
Filter.Exclude(f.Replace("**", "..."));
}
Filter.Include("/Binaries/...");
// ignore debug symbols
Filter.Exclude("/Binaries/.../*.pdb");
// ignore git files
Filter.Exclude(".git*");
Filter.Exclude(".git/...");
// always include plugin descriptor file
Filter.Include("*.uplugin");
// exclude releaes foldre
Filter.Exclude("/Releases/...");
Filter.Exclude("/Releases/.../...");
// read current version info from file
CreateVersionInfo();
// create meta info which is stored in BuildMeta
CreateMetaInfo();
// create file name
string ZipFileName = GenerateFileName() + ".zip";
string ZipFile = CombinePaths(PublishDir, ZipFileName);
string ZipDir = CombinePaths(CmdEnv.LocalRoot, "UnrealTournament", "Plugins", DLCName);
// check if file exists and abort...
if (FileExists(ZipFile) && !ParseParam("overwrite"))
{
throw new AutomationException("Release file {0} already exists.", ZipFileName);
}
string zipfolder = Packager.CopyFilesToZip(PublishDir, ZipDir, DLCName, Filter, NoConversion: HasParam("source"), ParseTextOnly: HasParam("ParseTextOnly"));
ZipFiles(ZipFile, zipfolder, new FileFilter(FileFilterType.Include));
if (ParseParam("publish") && BuildVersion.IsValid())
{
bool IsInternal = false;
if (ParseParam("release"))
{
// remove internal build (public release) ... just in case
DataWriteToFile(FileVersionName, BuildVersion.ToData(true));
}
else
{
IsInternal = true;
// increase internal build
BuildVersion.BuildAsNumber += 1;
DataWriteToFile(FileVersionName, BuildVersion.ToData());
}
string GitFolder = CombinePaths(DLCDir, GIT_FOLDER_NAME);
if (ParseParam("commit") && DirectoryExists(GitFolder))
{
using (var repo = new LibGit2Sharp.Repository(GitFolder))
{
string commmessage;
if (IsInternal)
commmessage = string.Format(COMMIT_TITLE_VERSION_INTERNAL, BuildVersion.ToString());
else
commmessage = string.Format(COMMIT_TITLE_VERSION_PUBLISH, BuildVersion.ToShortVersion());
commmessage += Environment.NewLine;
commmessage += Environment.NewLine;
commmessage += BuildMeta.ToExtendedString();
// Stage the file
LibGit2Sharp.StageOptions stageoptions = new LibGit2Sharp.StageOptions();
repo.Stage(FileVersionName, stageoptions);
// Create the committer's signature and commit
LibGit2Sharp.Signature author = repo.Config.BuildSignature(DateTime.Now);
LibGit2Sharp.Signature committer = author;
// Commit to the repository
LibGit2Sharp.CommitOptions options = new LibGit2Sharp.CommitOptions();
LibGit2Sharp.Commit commit = repo.Commit(commmessage, author, committer, options);
// Commit new tag
LibGit2Sharp.Tag t = repo.Tags.Add(string.Format("v{0}", BuildVersion.ToShortVersion()), repo.Head.Tip);
}
}
}
}
