public override void ExecuteBuild()
{
FileReference InputFile = ParseRequiredFileReferenceParam("File");
string[] Lines = FileReference.ReadAllLines(InputFile);
for (int LineIdx = 0; LineIdx < Lines.Length;)
{
string Line = Lines[LineIdx];
if (Line.StartsWith("Include Headers:", StringComparison.Ordinal))
{
LineIdx = ParseIncludeHeaders(Lines, LineIdx + 1, InputFile.ChangeExtension(".json"));
}
else if (Line.StartsWith("Class Definitions:", StringComparison.Ordinal))
{
LineIdx = ParseDefinitions(Lines, LineIdx + 1, InputFile.ChangeExtension(".classes.txt"));
}
else if (Line.StartsWith("Function Definitions:", StringComparison.Ordinal))
{
LineIdx = ParseDefinitions(Lines, LineIdx + 1, InputFile.ChangeExtension(".functions.txt"));
}
else
{
LineIdx++;
}
}
}
/// <summary>
/// Execute the task.
/// </summary>
/// <param name="Job">Information about the current job</param>
/// <param name="BuildProducts">Set of build products produced by this node.</param>
/// <param name="TagNameToFileSet">Mapping from tag names to the set of files they include</param>
public override void Execute(JobContext Job, HashSet <FileReference> BuildProducts, Dictionary <string, HashSet <FileReference> > TagNameToFileSet)
{
// Get the base directory
DirectoryReference BaseDir = Parameters.BaseDir ?? CommandUtils.RootDirectory;
// Parse all the exclude rules
List <string> ExcludeRules = ParseRules(BaseDir, Parameters.Except ?? "", TagNameToFileSet);
// Resolve the input list
HashSet <FileReference> Files = new HashSet <FileReference>();
if (!String.IsNullOrEmpty(Parameters.Files))
{
Files.UnionWith(ResolveFilespecWithExcludePatterns(BaseDir, Parameters.Files, ExcludeRules, TagNameToFileSet));
}
// Resolve the input file lists
if (!String.IsNullOrEmpty(Parameters.FileLists))
{
HashSet <FileReference> FileLists = ResolveFilespec(BaseDir, Parameters.FileLists, TagNameToFileSet);
foreach (FileReference FileList in FileLists)
{
if (!FileReference.Exists(FileList))
{
throw new AutomationException("Specified file list '{0}' does not exist", FileList);
}
string[] Lines = FileReference.ReadAllLines(FileList);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim();
if (TrimLine.Length > 0)
{
Files.Add(FileReference.Combine(BaseDir, TrimLine));
}
}
}
}
// Limit to matches against the 'Filter' parameter, if set
if (Parameters.Filter != null)
{
FileFilter Filter = new FileFilter();
Filter.AddRules(ParseRules(BaseDir, Parameters.Filter, TagNameToFileSet));
Files.RemoveWhere(x => !Filter.Matches(x.FullName));
}
// Apply the tag to all the matching files
foreach (string TagName in FindTagNamesFromList(Parameters.With))
{
FindOrAddTagSet(TagNameToFileSet, TagName).UnionWith(Files);
}
}
protected override void Execute(Repository Repo)
{
List <string> ExpandedFilters = ExpandFilters(Filters);
if (FilterFile != null)
{
if (!FileReference.Exists(FilterFile))
{
throw new FileNotFoundException(string.Format("Filter file '{0}' could not be found!", FilterFile), FilterFile.FullName);
}
ExpandedFilters.AddRange(FileReference.ReadAllLines(FilterFile));
}
List <KeyValuePair <string, string> > ClientAndStreams = ParseClientAndStreams(ClientAndStreamParams);
List <string> DistinctFilters = ExpandedFilters.Distinct(StringComparer.InvariantCultureIgnoreCase).ToList();
Repo.Populate(ClientAndStreams, DistinctFilters, bFakeSync);
}
private void ApplyBuildVersion()
{
// Read from ProjectR/Build/Build.version if exists
string BuildVersionPath = Path.Combine(ModuleDirectory, "..", "..", "Build", "Build.version");
string BuildVersion;
if (File.Exists(BuildVersionPath))
{
PublicDefinitions.Add("PROJECTR_NOPATCH_BUILD=0");
FileReference BuildVersionFile = new FileReference(BuildVersionPath);
string[] Lines = FileReference.ReadAllLines(BuildVersionFile);
if (Lines.Length >= 1)
{
BuildVersion = Lines[0];
}
else
{
BuildVersion = "FailedToReadBuildVersion";
}
}
else
{
PublicDefinitions.Add("PROJECTR_NOPATCH_BUILD=1");
BuildVersion = "c-live-1";
//BuildVersion = "c-dev-1";
//if (Target.Platform == UnrealTargetPlatform.Android)
//{
//BuildVersion = "c-live-1";
//}
//else
//{
//BuildVersion = "c-dev-1";
//}
}
PublicDefinitions.Add("PROJECTR_BUILD_VERSION=" + BuildVersion);
System.Console.WriteLine("PROJECTR_BUILD_VERSION=" + BuildVersion);
}
public override int Execute()
{
int ExitCode = 0;
// Auto-register all the known matchers in this assembly
List <IErrorMatcher> Matchers = new List <IErrorMatcher>();
foreach (Type Type in Assembly.GetExecutingAssembly().GetTypes())
{
if (Type.GetCustomAttribute <AutoRegisterAttribute>() != null)
{
object Instance = Activator.CreateInstance(Type);
if (typeof(IErrorMatcher).IsAssignableFrom(Type))
{
Matchers.Add((IErrorMatcher)Instance);
}
else
{
throw new Exception(String.Format("Unable to auto-register object of type {0}", Type.Name));
}
}
}
// Read all the ignore patterns
List <string> IgnorePatterns = new List <string>();
if (IgnorePatternsFile != null)
{
if (!FileReference.Exists(IgnorePatternsFile))
{
throw new FatalErrorException("Unable to read '{0}", IgnorePatternsFile);
}
// Read all the ignore patterns
string[] Lines = FileReference.ReadAllLines(IgnorePatternsFile);
foreach (string Line in Lines)
{
string TrimLine = Line.Trim();
if (TrimLine.Length > 0 && !TrimLine.StartsWith("#"))
{
IgnorePatterns.Add(TrimLine);
}
}
}
// Create the output listeners
List <IErrorListener> Listeners = new List <IErrorListener>();
try
{
if (bDebugListener)
{
Listeners.Add(new DebugOutputListener());
}
if (bElectricCommanderListener)
{
Listeners.Add(new ElectricCommanderListener());
}
if (IssuesOutputFile != null)
{
Listeners.Add(new IssuesListener(Stream, Change, JobName, JobUrl, JobStepName, JobStepUrl, LineUrl, BaseDir, IssuesOutputFile));
}
// Process the input
if (InputFile != null)
{
if (!FileReference.Exists(InputFile))
{
throw new FatalErrorException("Specified input file '{0}' does not exist", InputFile);
}
using (StreamReader Reader = new StreamReader(InputFile.FullName))
{
LineFilter Filter = new LineFilter(() => Reader.ReadLine());
ProcessErrors(Filter.ReadLine, Matchers, IgnorePatterns, Listeners, bNoWarnings);
}
}
else
{
CancellationTokenSource CancellationTokenSource = new CancellationTokenSource();
if (Timeout.HasValue)
{
CancellationTokenSource.CancelAfter(Timeout.Value);
}
CancellationToken CancellationToken = CancellationTokenSource.Token;
using (ManagedProcess Process = new ManagedProcess(null, Program.FullName, CommandLineArguments.Join(ProgramArguments), null, null, null, ProcessPriorityClass.Normal))
{
Func <string> ReadLine = new LineFilter(() => ReadProcessLine(Process, CancellationToken)).ReadLine;
ProcessErrors(ReadLine, Matchers, IgnorePatterns, Listeners, bNoWarnings);
ExitCode = Process.ExitCode;
}
}
}
finally
{
foreach (IErrorListener Listener in Listeners)
{
Listener.Dispose();
}
}
// Kill off any remaining child processes
ProcessUtils.TerminateChildProcesses();
return(ExitCode);
}
public override void ExecuteBuild()
{
CommandUtils.Log("************************* List Third Party Software");
string ProjectPath = ParseParamValue("Project", String.Empty);
//Add quotes to avoid issues with spaces in project path
if (ProjectPath != String.Empty)
{
ProjectPath = "\"" + ProjectPath + "\"";
}
// Parse the list of targets to list TPS for. Each target is specified by -Target="Name|Configuration|Platform" on the command line.
HashSet <FileReference> TpsFiles = new HashSet <FileReference>();
foreach (string Target in ParseParamValues(Params, "Target"))
{
// Get the path to store the exported JSON target data
FileReference OutputFile = FileReference.Combine(CommandUtils.EngineDirectory, "Intermediate", "Build", "ThirdParty.json");
IProcessResult Result;
Result = Run(UE4Build.GetUBTExecutable(), String.Format("{0} {1} -jsonexport=\"{2}\" -skipbuild", Target.Replace('|', ' '), ProjectPath, OutputFile.FullName), Options: ERunOptions.Default);
if (Result.ExitCode != 0)
{
throw new AutomationException("Failed to run UBT");
}
// Read the exported target info back in
JsonObject Object = JsonObject.Read(OutputFile);
// Get the project file if there is one
FileReference ProjectFile = null;
string ProjectFileName;
if (Object.TryGetStringField("ProjectFile", out ProjectFileName))
{
ProjectFile = new FileReference(ProjectFileName);
}
// Get the default paths to search
HashSet <DirectoryReference> DirectoriesToScan = new HashSet <DirectoryReference>();
DirectoriesToScan.Add(DirectoryReference.Combine(CommandUtils.EngineDirectory, "Shaders"));
DirectoriesToScan.Add(DirectoryReference.Combine(CommandUtils.EngineDirectory, "Content"));
if (ProjectFile != null)
{
DirectoriesToScan.Add(DirectoryReference.Combine(ProjectFile.Directory, "Content"));
}
// Get the variables to be expanded in any runtime dependencies variables
Dictionary <string, string> Variables = new Dictionary <string, string>();
Variables.Add("EngineDir", CommandUtils.EngineDirectory.FullName);
if (ProjectFile != null)
{
Variables.Add("ProjectDir", ProjectFile.Directory.FullName);
}
// Add all the paths for each module, and its runtime dependencies
JsonObject Modules = Object.GetObjectField("Modules");
foreach (string ModuleName in Modules.KeyNames)
{
JsonObject Module = Modules.GetObjectField(ModuleName);
DirectoriesToScan.Add(new DirectoryReference(Module.GetStringField("Directory")));
foreach (JsonObject RuntimeDependency in Module.GetObjectArrayField("RuntimeDependencies"))
{
string RuntimeDependencyPath = RuntimeDependency.GetStringField("Path");
RuntimeDependencyPath = Utils.ExpandVariables(RuntimeDependencyPath, Variables);
DirectoriesToScan.Add(new FileReference(RuntimeDependencyPath).Directory);
}
}
// Remove any directories that are under other directories, and sort the output list
List <DirectoryReference> SortedDirectoriesToScan = new List <DirectoryReference>();
foreach (DirectoryReference DirectoryToScan in DirectoriesToScan.OrderBy(x => x.FullName))
{
if (SortedDirectoriesToScan.Count == 0 || !DirectoryToScan.IsUnderDirectory(SortedDirectoriesToScan[SortedDirectoriesToScan.Count - 1]))
{
SortedDirectoriesToScan.Add(DirectoryToScan);
}
}
// Get the platforms to exclude
List <UnrealTargetPlatform> SupportedPlatforms = new List <UnrealTargetPlatform> {
(UnrealTargetPlatform)Enum.Parse(typeof(UnrealTargetPlatform), Object.GetStringField("Platform"))
};
FileSystemName[] ExcludePlatformNames = Utils.MakeListOfUnsupportedPlatforms(SupportedPlatforms).Select(x => new FileSystemName(x)).ToArray();
// Find all the TPS files under the engine directory which match
foreach (DirectoryReference DirectoryToScan in SortedDirectoriesToScan)
{
foreach (FileReference TpsFile in DirectoryReference.EnumerateFiles(DirectoryToScan, "*.tps", SearchOption.AllDirectories))
{
if (!TpsFile.ContainsAnyNames(ExcludePlatformNames, DirectoryToScan))
{
TpsFiles.Add(TpsFile);
}
}
}
}
// Also add any redirects
List <string> OutputMessages = new List <string>();
foreach (FileReference TpsFile in TpsFiles)
{
string Message = TpsFile.FullName;
string[] Lines = FileReference.ReadAllLines(TpsFile);
foreach (string Line in Lines)
{
const string RedirectPrefix = "Redirect:";
int Idx = Line.IndexOf(RedirectPrefix, StringComparison.InvariantCultureIgnoreCase);
if (Idx >= 0)
{
FileReference RedirectTpsFile = FileReference.Combine(TpsFile.Directory, Line.Substring(Idx + RedirectPrefix.Length).Trim());
Message = String.Format("{0} (redirect from {1})", RedirectTpsFile.FullName, TpsFile.FullName);
break;
}
}
OutputMessages.Add(Message);
}
OutputMessages.Sort();
// Print them all out
foreach (string OutputMessage in OutputMessages)
{
CommandUtils.Log(OutputMessage);
}
}
public override void ExecuteBuild()
{
string[] ProjectParams = ParseParamValues("Project");
string UpdateDirParam = ParseParamValue("UpdateDir", null);
if (UpdateDirParam == null)
{
throw new AutomationException("Missing -UpdateDir=... parameter");
}
DirectoryReference UpdateDir = new DirectoryReference(UpdateDirParam);
bool bWrite = ParseParam("Write");
// Get all the root dirs
HashSet <DirectoryReference> RootDirs = new HashSet <DirectoryReference>();
RootDirs.Add(EngineDirectory);
// Add the enterprise edirectory
DirectoryReference EnterpriseDirectory = DirectoryReference.Combine(RootDirectory, "Enterprise");
if (DirectoryReference.Exists(EnterpriseDirectory))
{
RootDirs.Add(EnterpriseDirectory);
}
// Add the project directories
foreach (string ProjectParam in ProjectParams)
{
FileReference ProjectLocation = new FileReference(ProjectParam);
if (!FileReference.Exists(ProjectLocation))
{
throw new AutomationException("Unable to find project '{0}'", ProjectLocation);
}
RootDirs.Add(ProjectLocation.Directory);
}
// Find all the modules
HashSet <DirectoryReference> ModuleDirs = new HashSet <DirectoryReference>();
foreach (DirectoryReference RootDir in RootDirs)
{
// Find all the modules from the source folder
DirectoryReference SourceDir = DirectoryReference.Combine(RootDir, "Source");
if (DirectoryReference.Exists(SourceDir))
{
foreach (FileReference ModuleFile in DirectoryReference.EnumerateFiles(SourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(ModuleFile.Directory);
}
}
// Find all the modules under the plugins folder
DirectoryReference PluginsDir = DirectoryReference.Combine(RootDir, "Plugins");
foreach (FileReference PluginFile in DirectoryReference.EnumerateFiles(PluginsDir, "*.uplugin", SearchOption.AllDirectories))
{
DirectoryReference PluginSourceDir = DirectoryReference.Combine(PluginFile.Directory, "Source");
if (DirectoryReference.Exists(PluginSourceDir))
{
foreach (FileReference PluginModuleFile in DirectoryReference.EnumerateFiles(PluginSourceDir, "*.Build.cs", SearchOption.AllDirectories))
{
ModuleDirs.Add(PluginModuleFile.Directory);
}
}
}
}
// Find a mapping from old to new include paths
Dictionary <string, Tuple <string, FileReference> > RemapIncludePaths = new Dictionary <string, Tuple <string, FileReference> >(StringComparer.InvariantCultureIgnoreCase);
foreach (DirectoryReference ModuleDir in ModuleDirs)
{
DirectoryReference ModulePublicDir = DirectoryReference.Combine(ModuleDir, "Public");
if (DirectoryReference.Exists(ModulePublicDir))
{
foreach (FileReference HeaderFile in DirectoryReference.EnumerateFiles(ModulePublicDir, "*.h", SearchOption.AllDirectories))
{
string BaseIncludeFile = HeaderFile.GetFileName();
Tuple <string, FileReference> ExistingIncludeName;
if (RemapIncludePaths.TryGetValue(BaseIncludeFile, out ExistingIncludeName))
{
LogWarning("Multiple include paths for {0}: {1}, {2}", BaseIncludeFile, ExistingIncludeName.Item2, HeaderFile);
}
else
{
RemapIncludePaths.Add(BaseIncludeFile, Tuple.Create(HeaderFile.MakeRelativeTo(ModulePublicDir).Replace('\\', '/'), HeaderFile));
}
}
}
}
// List of folders to exclude from updates
string[] ExcludeFoldersFromUpdate =
{
"Intermediate",
"ThirdParty"
};
// Enumerate all the files to update
HashSet <FileReference> UpdateFiles = new HashSet <FileReference>();
foreach (FileReference UpdateFile in DirectoryReference.EnumerateFiles(UpdateDir, "*", SearchOption.AllDirectories))
{
if (!UpdateFile.ContainsAnyNames(ExcludeFoldersFromUpdate, UpdateDir))
{
if (UpdateFile.HasExtension(".cpp") | UpdateFile.HasExtension(".h") || UpdateFile.HasExtension(".inl"))
{
UpdateFiles.Add(UpdateFile);
}
}
}
// Process all the source files
Dictionary <FileReference, string[]> ModifiedFiles = new Dictionary <FileReference, string[]>();
foreach (FileReference UpdateFile in UpdateFiles)
{
bool bModifiedFile = false;
string[] Lines = FileReference.ReadAllLines(UpdateFile);
for (int Idx = 0; Idx < Lines.Length; Idx++)
{
Match Match = Regex.Match(Lines[Idx], "^(\\s*#\\s*include\\s+\\\")([^\"]+)(\\\".*)$");
if (Match.Success)
{
string IncludePath = Match.Groups[2].Value;
Tuple <string, FileReference> NewIncludePath;
if (RemapIncludePaths.TryGetValue(IncludePath, out NewIncludePath))
{
if (IncludePath != NewIncludePath.Item1)
{
// Log("{0}: Changing {1} -> {2}", UpdateFile, IncludePath, NewIncludePath.Item1);
Lines[Idx] = String.Format("{0}{1}{2}", Match.Groups[1].Value, NewIncludePath.Item1, Match.Groups[3].Value);
bModifiedFile = true;
}
}
}
}
if (bModifiedFile)
{
ModifiedFiles.Add(UpdateFile, Lines);
}
}
// Output them all to disk
if (bWrite && ModifiedFiles.Count > 0)
{
LogInformation("Updating {0} files...", ModifiedFiles.Count);
List <FileReference> FilesToCheckOut = new List <FileReference>();
foreach (FileReference ModifiedFile in ModifiedFiles.Keys)
{
if ((FileReference.GetAttributes(ModifiedFile) & FileAttributes.ReadOnly) != 0)
{
FilesToCheckOut.Add(ModifiedFile);
}
}
if (FilesToCheckOut.Count > 0)
{
if (!P4Enabled)
{
throw new AutomationException("{0} files have been modified, but are read only. Run with -P4 to enable Perforce checkout.\n{1}", FilesToCheckOut.Count, String.Join("\n", FilesToCheckOut.Select(x => " " + x)));
}
LogInformation("Checking out files from Perforce");
int ChangeNumber = P4.CreateChange(Description: "Updating source files");
P4.Edit(ChangeNumber, FilesToCheckOut.Select(x => x.FullName).ToList(), false);
}
foreach (KeyValuePair <FileReference, string[]> FileToWrite in ModifiedFiles)
{
LogInformation("Writing {0}", FileToWrite.Key);
FileReference.WriteAllLines(FileToWrite.Key, FileToWrite.Value);
}
}
}
public List <ISharedCookedBuild> FindBestBuilds()
{
// Attempt manifest searching first
ConfigHierarchy Hierarchy = ConfigCache.ReadHierarchy(ConfigHierarchyType.Engine, DirectoryReference.FromFile(ProjectFile), UnrealTargetPlatform.Win64);
IEnumerable <string> RawSharedCookedSources = null;
Hierarchy.TryGetValues("SharedCookedBuildSettings", "SharedCookedSources", out RawSharedCookedSources);
if (RawSharedCookedSources == null)
{
throw new AutomationException("Unable to locate shared cooked builds. SharedCookedSources not set in Engine.ini [SharedCookedBuildSettings]");
}
List <Dictionary <string, string> > ParsedSharedCookSources = new List <Dictionary <string, string> >();
foreach (string RawConfig in RawSharedCookedSources)
{
Dictionary <string, string> ParsedSource = null;
if (ConfigHierarchy.TryParse(RawConfig, out ParsedSource))
{
ParsedSharedCookSources.Add(ParsedSource);
}
}
List <ISharedCookedBuild> CandidateBuilds = new List <ISharedCookedBuild>();
// If existing sync is present, stick to it. Read version out of sync file
foreach (string Platform in TargetPlatforms)
{
FileReference SyncedBuildFile = new FileReference(CommandUtils.CombinePaths(InstallPath.FullName, Platform, SyncedBuildFileName));
if (FileReference.Exists(SyncedBuildFile))
{
string[] SyncedBuildInfo = FileReference.ReadAllLines(SyncedBuildFile);
int SyncedCL = int.Parse(SyncedBuildInfo[0]);
if (IsValidCL(SyncedCL, BuildType, LocalSync))
{
CandidateBuilds.Add(new ExistingSharedCookedBuild()
{
CL = SyncedCL, Platform = Platform
});
}
}
}
foreach (Dictionary <string, string> Source in ParsedSharedCookSources)
{
SharedCookSource SourceType = (SharedCookSource)Enum.Parse(typeof(SharedCookSource), Source["Type"], true);
foreach (string Platform in TargetPlatforms)
{
if (SourceType == SharedCookSource.Manifest)
{
CandidateBuilds.AddRange(FindValidManifestBuilds(Source["Path"], Platform));
}
else if (SourceType == SharedCookSource.LooseFiles)
{
CandidateBuilds.AddRange(FindValidLooseBuilds(Source["Path"], Platform));
}
}
}
// Strip all failed searches
CandidateBuilds.RemoveAll(x => x == null);
// Make sure we have a matching CL for all target platforms, regardless of source
List <int> OrderedDistinctCLs = CandidateBuilds.Select(x => x.CL).Distinct().OrderByDescending(i => i).ToList();
int BestCL = -1;
foreach (int CL in OrderedDistinctCLs)
{
// Ensure we have a platform for each
HashSet <string> CLPlatforms = new HashSet <string>(CandidateBuilds.Where(x => x.CL == CL).Select(x => x.Platform).ToList());
if (CLPlatforms.SetEquals(TargetPlatforms))
{
BestCL = CL;
break;
}
}
if (BestCL < 0)
{
CommandUtils.LogError("Could not locate valid shared cooked build for all target platforms");
CommandUtils.LogError("Current CL: {0}, Current Code CL: {1}", LocalSync.Changelist, LocalSync.CompatibleChangelist);
}
return(CandidateBuilds.Where(x => x.CL == BestCL).ToList());
}