/// <summary> /// Callback from the thread pool worker /// </summary> /// <param name="Queue">The queue calling this method</param> public void Run(ThreadPoolWorkQueue Queue) { // Add all the files in this directory foreach (WorkspaceFile File in ParentDirectory.Files) { if (Rules.IsSourceFile(File.NormalizedPathFromBranchRoot)) { // Read the contents of the file if (!Rules.IsExternalHeaderPath(File.NormalizedPathFromBranchRoot)) { File.ReadSourceFile(); } // Add it to the output list of files FoundFiles.Add(File); } } // Add everything in subdirectories foreach (WorkspaceDirectory Directory in ParentDirectory.Directories) { if (Rules.SearchDirectoryForSource(Directory.NormalizedPathFromBranchRoot)) { Queue.Enqueue(new WorkspaceReferenceScanner(Directory, FoundFiles)); } } }
/// <summary> /// Callback from the thread pool worker /// </summary> /// <param name="Queue">The queue calling this method</param> public void Run(ThreadPoolWorkQueue Queue) { WorkspaceDirectory CurrentDirectory = Directory; while (CurrentDirectory != null) { CurrentDirectory.CacheFiles(); CurrentDirectory.CacheDirectories(); if (CurrentDirectory.NormalizedPathFromBranchRoot != null && !Rules.SearchDirectoryForSource(Directory.NormalizedPathFromBranchRoot)) { break; } WorkspaceDirectory NextDirectory = null; foreach (WorkspaceDirectory ChildDirectory in CurrentDirectory.Directories) { if (NextDirectory == null) { NextDirectory = ChildDirectory; } else { Queue.Enqueue(new WorkspaceScanner(ChildDirectory)); } } CurrentDirectory = NextDirectory; } }
/// <summary> /// Finds all the rules of the given type under a given directory /// </summary> /// <param name="Directory">Directory to search</param> /// <param name="Type">Type of rules to return</param> /// <returns>List of rules files of the given type</returns> private static IReadOnlyList <FileReference> FindAllRulesFiles(DirectoryReference Directory, RulesFileType Type) { // Check to see if we've already cached source files for this folder RulesFileCache Cache; if (!RootFolderToRulesFileCache.TryGetValue(Directory, out Cache)) { Cache = new RulesFileCache(); using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue()) { DirectoryItem BaseDirectory = DirectoryItem.GetItemByDirectoryReference(Directory); Queue.Enqueue(() => FindAllRulesFilesRecursively(BaseDirectory, Cache, Queue)); } RootFolderToRulesFileCache[Directory] = Cache; } // Get the list of files of the type we're looking for if (Type == RulesCompiler.RulesFileType.Module) { return(Cache.ModuleRules); } else if (Type == RulesCompiler.RulesFileType.Target) { return(Cache.TargetRules); } else if (Type == RulesCompiler.RulesFileType.AutomationModule) { return(Cache.AutomationModules); } else { throw new BuildException("Unhandled rules type: {0}", Type); } }
/// <summary> /// Caches all the files under the given directory /// </summary> /// <param name="Directory">The directory to cache</param> public static void CacheFiles(WorkspaceDirectory Directory) { using (ThreadPoolWorkQueue WorkQueue = new ThreadPoolWorkQueue()) { WorkQueue.Enqueue(new WorkspaceScanner(Directory)); } }
void AddSubDirectory(WorkspaceDirectoryInfo NewWorkspaceDir, StreamDirectoryInfo StreamSubDir, ThreadPoolWorkQueue Queue) { WorkspaceDirectoryInfo NewWorkspaceSubDir = new WorkspaceDirectoryInfo(NewWorkspaceDir, StreamSubDir.Name); NewWorkspaceDir.NameToSubDirectory.Add(StreamSubDir.Name, NewWorkspaceSubDir); Queue.Enqueue(() => AddDirectory(NewWorkspaceSubDir, StreamSubDir, Queue)); }
/// <summary> /// Enqueue a task which checks for the cancellation token first /// </summary> /// <param name="Action">Action to enqueue</param> static void Enqueue(System.Action Action) { Queue.Enqueue(() => { if (!CancelToken.IsCancellationRequested) { Action(); } }); }
public string[] FindDifferences() { ConcurrentBag <string> Paths = new ConcurrentBag <string>(); using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue()) { Queue.Enqueue(() => FindDifferences(new DirectoryInfo(GetFullName()), "/", Paths, Queue)); } return(Paths.OrderBy(x => x).ToArray()); }
public WorkspaceTransactionRemove(WorkspaceDirectoryInfo WorkspaceRootDir, StreamDirectoryInfo StreamRootDir, Dictionary <FileContentId, TrackedFileInfo> ContentIdToTrackedFile) { this.NewWorkspaceRootDir = new WorkspaceDirectoryInfo(WorkspaceRootDir.GetLocation()); this.ContentIdToTrackedFile = ContentIdToTrackedFile; using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue()) { Queue.Enqueue(() => Merge(WorkspaceRootDir, NewWorkspaceRootDir, StreamRootDir, Queue)); } }
public void Refresh(out FileInfo[] FilesToDelete, out DirectoryInfo[] DirectoriesToDelete) { ConcurrentBag <FileInfo> ConcurrentFilesToDelete = new ConcurrentBag <FileInfo>(); ConcurrentBag <DirectoryInfo> ConcurrentDirectoriesToDelete = new ConcurrentBag <DirectoryInfo>(); using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue()) { Queue.Enqueue(() => Refresh(new DirectoryInfo(GetFullName()), ConcurrentFilesToDelete, ConcurrentDirectoriesToDelete, Queue)); } DirectoriesToDelete = ConcurrentDirectoriesToDelete.ToArray(); FilesToDelete = ConcurrentFilesToDelete.ToArray(); }
void RemoveDirectory(WorkspaceDirectoryInfo WorkspaceDir, ThreadPoolWorkQueue Queue) { DirectoriesToDelete.Add(WorkspaceDir); foreach (WorkspaceDirectoryInfo WorkspaceSubDir in WorkspaceDir.NameToSubDirectory.Values) { Queue.Enqueue(() => RemoveDirectory(WorkspaceSubDir, Queue)); } foreach (WorkspaceFileInfo WorkspaceFile in WorkspaceDir.NameToFile.Values) { RemoveFile(WorkspaceFile); } }
private void FindDifferences(DirectoryInfo Directory, string Path, ConcurrentBag <string> Paths, ThreadPoolWorkQueue Queue) { // Recurse through subdirectories HashSet <string> RemainingSubDirectoryNames = new HashSet <string>(NameToSubDirectory.Keys); foreach (DirectoryInfo SubDirectory in Directory.EnumerateDirectories()) { WorkspaceDirectoryInfo StagedSubDirectory; if (NameToSubDirectory.TryGetValue(SubDirectory.Name, out StagedSubDirectory)) { RemainingSubDirectoryNames.Remove(SubDirectory.Name); Queue.Enqueue(() => StagedSubDirectory.FindDifferences(SubDirectory, String.Format("{0}{1}/", Path, SubDirectory.Name), Paths, Queue)); continue; } Paths.Add(String.Format("+{0}{1}/...", Path, SubDirectory.Name)); } foreach (string RemainingSubDirectoryName in RemainingSubDirectoryNames) { Paths.Add(String.Format("-{0}{1}/...", Path, RemainingSubDirectoryName)); } // Search through files HashSet <string> RemainingFileNames = new HashSet <string>(NameToFile.Keys); foreach (FileInfo File in Directory.EnumerateFiles()) { WorkspaceFileInfo StagedFile; if (!NameToFile.TryGetValue(File.Name, out StagedFile)) { Paths.Add(String.Format("+{0}{1}", Path, File.Name)); } else if (!StagedFile.MatchesAttributes(File)) { Paths.Add(String.Format("!{0}{1}", Path, File.Name)); RemainingFileNames.Remove(File.Name); } else { RemainingFileNames.Remove(File.Name); } } foreach (string RemainingFileName in RemainingFileNames) { Paths.Add(String.Format("-{0}{1}", Path, RemainingFileName)); } }
public WorkspaceTransactionAdd(WorkspaceDirectoryInfo WorkspaceRootDir, StreamDirectoryInfo StreamRootDir, Dictionary <FileContentId, TrackedFileInfo> ContentIdToTrackedFile) { this.NewWorkspaceRootDir = new WorkspaceDirectoryInfo(WorkspaceRootDir.GetLocation()); this.ContentIdToTrackedFile = ContentIdToTrackedFile; List <WorkspaceFileInfo> WorkspaceFiles = WorkspaceRootDir.GetFiles(); foreach (WorkspaceFileInfo WorkspaceFile in WorkspaceFiles) { ContentIdToWorkspaceFile[WorkspaceFile.ContentId] = WorkspaceFile; } using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue()) { Queue.Enqueue(() => MergeDirectory(WorkspaceRootDir, NewWorkspaceRootDir, StreamRootDir, Queue)); } }
/// <summary> /// Finds all the source files under a directory that contain reflection markup /// </summary> /// <param name="Directory">The directory to search</param> /// <param name="MetadataCache">Cache of source file metadata</param> /// <param name="ExcludedFolderNames">Set of folder names to ignore when recursing the directory tree</param> /// <param name="FilesWithMarkup">Receives the set of files which contain reflection markup</param> /// <param name="Queue">Queue to add sub-tasks to</param> static void FindFilesWithMarkup(DirectoryItem Directory, SourceFileMetadataCache MetadataCache, ReadOnlyHashSet <string> ExcludedFolderNames, ConcurrentBag <FileItem> FilesWithMarkup, ThreadPoolWorkQueue Queue) { // Search through all the subfolders foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories()) { if (!ExcludedFolderNames.Contains(SubDirectory.Name)) { Queue.Enqueue(() => FindFilesWithMarkup(SubDirectory, MetadataCache, ExcludedFolderNames, FilesWithMarkup, Queue)); } } // Check for all the headers in this folder foreach (FileItem File in Directory.EnumerateFiles()) { if (File.HasExtension(".h") && MetadataCache.ContainsReflectionMarkup(File)) { FilesWithMarkup.Add(File); } } }
/// <summary> /// Search through a directory tree for any rules files /// </summary> /// <param name="Directory">The root directory to search from</param> /// <param name="Cache">Receives all the discovered rules files</param> /// <param name="Queue">Queue for adding additional tasks to</param> private static void FindAllRulesFilesRecursively(DirectoryItem Directory, RulesFileCache Cache, ThreadPoolWorkQueue Queue) { // Scan all the files in this directory bool bSearchSubFolders = true; foreach (FileItem File in Directory.EnumerateFiles()) { if (File.HasExtension(".build.cs")) { lock (Cache.ModuleRules) { Cache.ModuleRules.Add(File.Location); } bSearchSubFolders = false; } else if (File.HasExtension(".target.cs")) { lock (Cache.TargetRules) { Cache.TargetRules.Add(File.Location); } } else if (File.HasExtension(".automation.csproj")) { lock (Cache.AutomationModules) { Cache.AutomationModules.Add(File.Location); } bSearchSubFolders = false; } } // If we didn't find anything to stop the search, search all the subdirectories too if (bSearchSubFolders) { foreach (DirectoryItem SubDirectory in Directory.EnumerateDirectories()) { Queue.Enqueue(() => FindAllRulesFilesRecursively(SubDirectory, Cache, Queue)); } } }
/// <summary> /// Find paths to all the plugins under a given parent directory (recursively) /// </summary> /// <param name="ParentDirectory">Parent directory to look in. Plugins will be found in any *subfolders* of this directory.</param> /// <param name="FileNames">List of filenames. Will have all the discovered .uplugin files appended to it.</param> /// <param name="Queue">Queue for tasks to be executed</param> static void EnumeratePluginsInternal(DirectoryItem ParentDirectory, List <FileReference> FileNames, ThreadPoolWorkQueue Queue) { foreach (DirectoryItem ChildDirectory in ParentDirectory.EnumerateDirectories()) { bool bSearchSubDirectories = true; foreach (FileItem PluginFile in ChildDirectory.EnumerateFiles()) { if (PluginFile.HasExtension(".uplugin")) { lock (FileNames) { FileNames.Add(PluginFile.Location); } bSearchSubDirectories = false; } } if (bSearchSubDirectories) { Queue.Enqueue(() => EnumeratePluginsInternal(ChildDirectory, FileNames, Queue)); } } }
private void Refresh(DirectoryInfo Info, ConcurrentBag <FileInfo> FilesToDelete, ConcurrentBag <DirectoryInfo> DirectoriesToDelete, ThreadPoolWorkQueue Queue) { // Recurse through subdirectories Dictionary <string, WorkspaceDirectoryInfo> NewNameToSubDirectory = new Dictionary <string, WorkspaceDirectoryInfo>(NameToSubDirectory.Count, NameToSubDirectory.Comparer); foreach (DirectoryInfo SubDirectoryInfo in Info.EnumerateDirectories()) { WorkspaceDirectoryInfo SubDirectory; if (NameToSubDirectory.TryGetValue(SubDirectoryInfo.Name, out SubDirectory)) { NewNameToSubDirectory.Add(SubDirectory.Name, SubDirectory); Queue.Enqueue(() => SubDirectory.Refresh(SubDirectoryInfo, FilesToDelete, DirectoriesToDelete, Queue)); } else { DirectoriesToDelete.Add(SubDirectoryInfo); } } NameToSubDirectory = NewNameToSubDirectory; // Figure out which files have changed. Dictionary <string, WorkspaceFileInfo> NewNameToFile = new Dictionary <string, WorkspaceFileInfo>(NameToFile.Count, NameToFile.Comparer); foreach (FileInfo File in Info.EnumerateFiles()) { WorkspaceFileInfo StagedFile; if (NameToFile.TryGetValue(File.Name, out StagedFile) && StagedFile.MatchesAttributes(File)) { NewNameToFile.Add(StagedFile.Name, StagedFile); } else { FilesToDelete.Add(File); } } NameToFile = NewNameToFile; }
/// <summary> /// Checks if the makefile is valid for the current set of source files. This is done separately to the Load() method to allow pre-build steps to modify source files. /// </summary> /// <param name="Makefile">The makefile that has been loaded</param> /// <param name="ProjectFile">Path to the project file</param> /// <param name="Platform">The platform being built</param> /// <param name="WorkingSet">The current working set of source files</param> /// <param name="ReasonNotLoaded">If the makefile is not valid, is set to a message describing why</param> /// <returns>True if the makefile is valid, false otherwise</returns> public static bool IsValidForSourceFiles(TargetMakefile Makefile, FileReference ProjectFile, UnrealTargetPlatform Platform, ISourceFileWorkingSet WorkingSet, out string ReasonNotLoaded) { using (Timeline.ScopeEvent("TargetMakefile.IsValidForSourceFiles()")) { // Get the list of excluded folder names for this platform ReadOnlyHashSet <string> ExcludedFolderNames = UEBuildPlatform.GetBuildPlatform(Platform).GetExcludedFolderNames(); // Check if any source files have been added or removed foreach (KeyValuePair <DirectoryItem, FileItem[]> Pair in Makefile.DirectoryToSourceFiles) { DirectoryItem InputDirectory = Pair.Key; if (!InputDirectory.Exists || InputDirectory.LastWriteTimeUtc > Makefile.CreateTimeUtc) { FileItem[] SourceFiles = UEBuildModuleCPP.GetSourceFiles(InputDirectory); if (SourceFiles.Length < Pair.Value.Length) { ReasonNotLoaded = "source file removed"; return(false); } else if (SourceFiles.Length > Pair.Value.Length) { ReasonNotLoaded = "source file added"; return(false); } else if (SourceFiles.Intersect(Pair.Value).Count() != SourceFiles.Length) { ReasonNotLoaded = "source file modified"; return(false); } foreach (DirectoryItem Directory in InputDirectory.EnumerateDirectories()) { if (!Makefile.DirectoryToSourceFiles.ContainsKey(Directory) && ContainsSourceFiles(Directory, ExcludedFolderNames)) { ReasonNotLoaded = "directory added"; return(false); } } } } // Check if any external dependencies have changed. These comparisons are done against the makefile creation time. foreach (FileItem ExternalDependency in Makefile.ExternalDependencies) { if (!ExternalDependency.Exists) { Log.TraceLog("{0} has been deleted since makefile was built.", ExternalDependency.Location); ReasonNotLoaded = string.Format("{0} deleted", ExternalDependency.Location.GetFileName()); return(false); } if (ExternalDependency.LastWriteTimeUtc > Makefile.CreateTimeUtc) { Log.TraceLog("{0} has been modified since makefile was built.", ExternalDependency.Location); ReasonNotLoaded = string.Format("{0} modified", ExternalDependency.Location.GetFileName()); return(false); } } // Check if any internal dependencies has changed. These comparisons are done against the makefile modified time. foreach (FileItem InternalDependency in Makefile.InternalDependencies) { if (!InternalDependency.Exists) { Log.TraceLog("{0} has been deleted since makefile was written.", InternalDependency.Location); ReasonNotLoaded = string.Format("{0} deleted", InternalDependency.Location.GetFileName()); return(false); } if (InternalDependency.LastWriteTimeUtc > Makefile.ModifiedTimeUtc) { Log.TraceLog("{0} has been modified since makefile was written.", InternalDependency.Location); ReasonNotLoaded = string.Format("{0} modified", InternalDependency.Location.GetFileName()); return(false); } } // Check that no new plugins have been added foreach (FileReference PluginFile in Plugins.EnumeratePlugins(ProjectFile)) { FileItem PluginFileItem = FileItem.GetItemByFileReference(PluginFile); if (!Makefile.PluginFiles.Contains(PluginFileItem)) { Log.TraceLog("{0} has been added", PluginFile.GetFileName()); ReasonNotLoaded = string.Format("{0} has been added", PluginFile.GetFileName()); return(false); } } // Load the metadata cache SourceFileMetadataCache MetadataCache = SourceFileMetadataCache.CreateHierarchy(ProjectFile); // Find the set of files that contain reflection markup ConcurrentBag <FileItem> NewFilesWithMarkupBag = new ConcurrentBag <FileItem>(); using (ThreadPoolWorkQueue Queue = new ThreadPoolWorkQueue()) { foreach (DirectoryItem SourceDirectory in Makefile.SourceDirectories) { Queue.Enqueue(() => FindFilesWithMarkup(SourceDirectory, MetadataCache, ExcludedFolderNames, NewFilesWithMarkupBag, Queue)); } } // Check whether the list has changed List <FileItem> PrevFilesWithMarkup = Makefile.UObjectModuleHeaders.Where(x => !x.bUsePrecompiled).SelectMany(x => x.HeaderFiles).ToList(); List <FileItem> NextFilesWithMarkup = NewFilesWithMarkupBag.ToList(); if (NextFilesWithMarkup.Count != PrevFilesWithMarkup.Count || NextFilesWithMarkup.Intersect(PrevFilesWithMarkup).Count() != PrevFilesWithMarkup.Count) { ReasonNotLoaded = "UHT files changed"; return(false); } // If adaptive unity build is enabled, do a check to see if there are any source files that became part of the // working set since the Makefile was created (or, source files were removed from the working set.) If anything // changed, then we'll force a new Makefile to be created so that we have fresh unity build blobs. We always // want to make sure that source files in the working set are excluded from those unity blobs (for fastest possible // iteration times.) // Check if any source files in the working set no longer belong in it foreach (FileItem SourceFile in Makefile.WorkingSet) { if (!WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc) { Log.TraceLog("{0} was part of source working set and now is not; invalidating makefile", SourceFile.AbsolutePath); ReasonNotLoaded = string.Format("working set of source files changed"); return(false); } } // Check if any source files that are eligible for being in the working set have been modified foreach (FileItem SourceFile in Makefile.CandidatesForWorkingSet) { if (WorkingSet.Contains(SourceFile) && SourceFile.LastWriteTimeUtc > Makefile.CreateTimeUtc) { Log.TraceLog("{0} was part of source working set and now is not", SourceFile.AbsolutePath); ReasonNotLoaded = string.Format("working set of source files changed"); return(false); } } } ReasonNotLoaded = null; return(true); }