//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected ITaskItem [] GetOutOfDateSourcesFromCmdLineChanges(string expectedCommandLine, ITaskItem [] uncheckedSources) { // // Evaluate which (if any) of the `uncheckedSources` are already present in the command TLog. // If cached entries are found, identify whether this source should be considered "out of date". // if (TLogCommandFiles?.Length == 0) { return(Array.Empty <ITaskItem>()); } Dictionary <string, ITaskItem> uncheckedSourcesRooted = new Dictionary <string, ITaskItem>(); foreach (var uncheckedSource in uncheckedSources) { uncheckedSourcesRooted.Add(TrackedFileManager.ConvertToTrackerFormat(uncheckedSource), uncheckedSource); } var outOfDateSources = new HashSet <ITaskItem>(); using StreamReader reader = File.OpenText(TLogCommandFiles[0].ItemSpec); for (string line = reader.ReadLine(); !string.IsNullOrEmpty(line); line = reader.ReadLine()) { if (line.StartsWith("^")) { var trackedFiles = line.Substring(1).Split('|'); string trackedCommandLine = reader.ReadLine(); foreach (string trackedFile in trackedFiles) { if (uncheckedSourcesRooted.TryGetValue(trackedFile, out ITaskItem match) && !string.Equals(trackedCommandLine, expectedCommandLine)) { #if DEBUG Log.LogMessageFromText($"[{GetType().Name}] Out of date source identified: {trackedFile}. Command lines differed.", MessageImportance.Low); Log.LogMessageFromText($"[{GetType().Name}] Out of date source identified: {trackedFile}. Expected: {expectedCommandLine} ", MessageImportance.Low); Log.LogMessageFromText($"[{GetType().Name}] Out of date source identified: {trackedFile}. Cached: {trackedCommandLine} ", MessageImportance.Low); #endif outOfDateSources.Add(match); } } } } return(outOfDateSources.ToArray()); }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected override void AddTaskSpecificOutputFiles (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { // // Collate the output files using 'GetOutputFilesFromPath' helper. This handles singular and directory output(s). // string fullOutputPath = OutputPath.GetMetadata ("FullPath"); ITaskItem [] outputFiles = GetOutputFilesFromPath (fullOutputPath); if (outputFiles != null) { trackedFileManager.AddDependencyForSources (outputFiles, sources); } }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected override void AddTaskSpecificDependencies (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { foreach (ITaskItem source in sources) { string fullPath = source.GetMetadata ("FullPath"); // // Mark additional dependencies for .class files contained within specified folder class paths. // if (Directory.Exists (fullPath)) { string [] classPathFiles = Directory.GetFiles (fullPath, "*.class", SearchOption.AllDirectories); List<ITaskItem> classPathFileItems = new List<ITaskItem> (classPathFiles.Length); foreach (string classpath in classPathFiles) { classPathFileItems.Add (new TaskItem (classpath)); } trackedFileManager.AddDependencyForSources (classPathFileItems.ToArray (), new ITaskItem [] { source }); } // // Ensure configuration file(s) for MultiDex output are flagged as dependencies. // bool multiDex = (source.GetMetadata ("MultiDex") == "true"); string multiDexMainList = source.GetMetadata ("MultiDexMainList"); if (multiDex && !string.IsNullOrWhiteSpace (multiDexMainList) && File.Exists (multiDexMainList)) { ITaskItem multiDexMainListItem = new TaskItem (multiDexMainList); trackedFileManager.AddDependencyForSources (new ITaskItem [] { multiDexMainListItem }, new ITaskItem [] { source }); } } }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected override void AddTaskSpecificDependencies (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { // // Register additional 'forced include' usage for each of the sources. // foreach (ITaskItem source in sources) { try { if (!string.IsNullOrWhiteSpace (source.GetMetadata ("ForcedIncludeFiles"))) { string [] forcedIncludeFiles = source.GetMetadata ("ForcedIncludeFiles").Split (new char [] { ';' }, StringSplitOptions.RemoveEmptyEntries); List<ITaskItem> forcedIncludeItems = new List<ITaskItem> (); foreach (string file in forcedIncludeFiles) { // // Supports including pre-compiled headers via '-include' when they need to be referenced without '.pch'/'.gch'. Fix this. // string fileFullPath = Path.GetFullPath (file); if ((ToolExe.StartsWith ("clang")) && (File.Exists (fileFullPath + ".pch"))) { fileFullPath = fileFullPath + ".pch"; } else if (File.Exists (fileFullPath + ".gch")) { fileFullPath = fileFullPath + ".gch"; } // // Also validate that we don't try adding dependencies to missing files, as this breaks tracking. // if (!File.Exists (fileFullPath)) { throw new FileNotFoundException ("Could not find 'forced include' dependency: " + fileFullPath); } forcedIncludeItems.Add (new TaskItem (fileFullPath)); } trackedFileManager.AddDependencyForSources (forcedIncludeItems.ToArray (), new ITaskItem [] { source }); } } catch (Exception e) { Log.LogWarningFromException (e, false); } } }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected override void AddTaskSpecificOutputFiles (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { trackedFileManager.AddDependencyForSources (OutputFiles, sources); }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected override void AddTaskSpecificDependencies (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { if (ManifestFile != null) { trackedFileManager.AddDependencyForSources (new ITaskItem [] { ManifestFile }, sources); } }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected override void AddTaskSpecificOutputFiles (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { trackedFileManager.AddDependencyForSources (m_outputClassSourceFiles.ToArray (), sources); }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// #if false protected override void OutputCommandTLog(ITaskItem commandFile, string responseFileCommands, string commandLineCommands) { // // Output a tracking file for each of the commands used in the previous build, and target sources to which they relate. // // *Keeps existing entries in the command log. Updates entries for sources which have been compiled/modified.* // if (TLogCommandFiles?.Length == 0) { throw new InvalidOperationException("TLogCommandFile is invalid"); } // // Merge existing and new dictionaries. This is quite expensive, but means we can utilise a more simple base export implementation. // Dictionary <string, List <ITaskItem> > cachedCommandLogDictionary = GenerateCommandLinesFromTlogs(TLogCommandFiles); Dictionary <string, List <ITaskItem> > mergedCommandLogDictionary = new Dictionary <string, List <ITaskItem> > (); // // Add recently changed sources first, ensuring these take precedence. // foreach (KeyValuePair <string, List <ITaskItem> > entry in commandDictionary) { HashSet <string> mergedLogDictionaryListAsFullPaths = new HashSet <string> (); if (mergedCommandLogDictionary.TryGetValue(entry.Key, out List <ITaskItem> mergedLogDictionaryList)) { foreach (ITaskItem source in mergedLogDictionaryList) { string trackerFormat = TrackedFileManager.ConvertToTrackerFormat(source.GetMetadata("FullPath")); if (!mergedLogDictionaryListAsFullPaths.Contains(trackerFormat)) { mergedLogDictionaryListAsFullPaths.Add(trackerFormat); } } } else { mergedLogDictionaryList = new List <ITaskItem> (); } foreach (ITaskItem source in entry.Value) { string trackerFormat = TrackedFileManager.ConvertToTrackerFormat(source.GetMetadata("FullPath")); if (!mergedLogDictionaryListAsFullPaths.Contains(trackerFormat)) { mergedLogDictionaryList.Add(source); mergedLogDictionaryListAsFullPaths.Add(trackerFormat); } } mergedCommandLogDictionary [entry.Key] = mergedLogDictionaryList; } // // Continue by adding the remaining cached source commands, if they won't overwrite any existing entries. // foreach (KeyValuePair <string, List <ITaskItem> > entry in cachedCommandLogDictionary) { HashSet <string> mergedLogDictionaryListAsFullPaths = new HashSet <string> (); if (mergedCommandLogDictionary.TryGetValue(entry.Key, out List <ITaskItem> mergedLogDictionaryList)) { foreach (ITaskItem source in mergedLogDictionaryList) { string trackerFormat = TrackedFileManager.ConvertToTrackerFormat(source.GetMetadata("FullPath")); if (!mergedLogDictionaryListAsFullPaths.Contains(trackerFormat)) { mergedLogDictionaryListAsFullPaths.Add(trackerFormat); } } } else { mergedLogDictionaryList = new List <ITaskItem> (); } foreach (ITaskItem source in entry.Value) { string trackerFormat = TrackedFileManager.ConvertToTrackerFormat(source.GetMetadata("FullPath")); if (!mergedLogDictionaryListAsFullPaths.Contains(trackerFormat)) { mergedLogDictionaryList.Add(source); mergedLogDictionaryListAsFullPaths.Add(trackerFormat); } } mergedCommandLogDictionary [entry.Key] = mergedLogDictionaryList; } base.OutputCommandTLog(mergedCommandLogDictionary); }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected virtual void OutputReadTLog (Dictionary<string, List<ITaskItem>> commandDictionary, ITaskItem [] sources) { // // Output a tracking file detailing which files were read (or are dependencies) for the source files built. Changes in these files will invoke recompilation. // try { if (!TrackFileAccess) { throw new InvalidOperationException ("'TrackFileAccess' is not set. Should not be attempting to output read TLog."); } if (commandDictionary == null) { throw new ArgumentNullException ("commandDictionary"); } if (sources == null) { throw new ArgumentNullException ("sources"); } if ((TLogReadFiles == null) || (TLogReadFiles.Length != 1)) { throw new InvalidOperationException ("TLogReadFiles is missing or does not have a length of 1"); } TrackedFileManager trackedFileManager = new TrackedFileManager (); if (trackedFileManager != null) { // // Clear any old entries to sources which have just been processed. // trackedFileManager.ImportFromExistingTLog (TLogReadFiles [0]); trackedFileManager.RemoveSourcesFromTable (sources); trackedFileManager.AddSourcesToTable (sources); // // Add any explicit inputs registered by parent task. // AddTaskSpecificDependencies (ref trackedFileManager, sources); // // Create dependency mappings for 'global' task outputs. Assume these relate to all processed sources. // // Dependency files are exported/collated in various different ways: // // - C/C++ uses <filename>.d (alongside .o/.obj output) // - Java uses: <filename>.java.d // - Unix uses: <filename>.<ext>.d // // // Check alongside the known output files for a dependency file. // { Dictionary<string, string> outputDependencyFilePermutations = new Dictionary<string, string> (OutputFiles.Length * 2); foreach (ITaskItem outputFile in OutputFiles) { string outputFileFullPath = outputFile.GetMetadata ("FullPath"); if (!string.IsNullOrWhiteSpace (outputFileFullPath)) { string [] permutations = new string [] { Path.ChangeExtension (outputFileFullPath, ".d"), outputFileFullPath + ".d" }; for (int i = 0; i < permutations.Length; ++i) { if (!outputDependencyFilePermutations.ContainsKey (permutations [i])) { outputDependencyFilePermutations.Add (permutations [i], outputFileFullPath); } } } } foreach (KeyValuePair<string, string> dependencyKeyPair in outputDependencyFilePermutations) { // // Validate this permutation is something we'd expect. // string dependencyFile = dependencyKeyPair.Key; if (string.IsNullOrWhiteSpace (dependencyFile)) { continue; } else if (string.IsNullOrWhiteSpace (Path.GetFileNameWithoutExtension (dependencyFile))) { continue; } else if (!File.Exists (dependencyFile)) { continue; } // // Probe the dependency file. Evaluate & find parent source item and add dependencies. // GccUtilities.DependencyParser parser = new GccUtilities.DependencyParser (); parser.Parse (dependencyFile); #if DEBUG Log.LogMessageFromText (string.Format ("[{0}] --> Dependencies (Read) : {1} (Entries: {2})", ToolName, dependencyFile, parser.Dependencies.Count), MessageImportance.Low); Log.LogMessageFromText (string.Format ("[{0}] --> Dependencies (Read) : [{1}] '{2}'", ToolName, "+", parser.OutputFile.GetMetadata ("FullPath")), MessageImportance.Low); int index = 0; foreach (var dependency in parser.Dependencies) { Log.LogMessageFromText (string.Format ("[{0}] --> Dependencies (Read) : [{1}] '{2}'", ToolName, ++index, dependency), MessageImportance.Low); } #endif if (parser.Dependencies.Count > 0) { Dictionary<string, ITaskItem> collatedFullPathSources = new Dictionary<string, ITaskItem> (sources.Length); foreach (ITaskItem source in sources) { collatedFullPathSources [source.GetMetadata ("FullPath")] = source; } ITaskItem parentSourceItem = null; if (collatedFullPathSources.TryGetValue (parser.OutputFile.GetMetadata ("FullPath"), out parentSourceItem)) { // // We managed to find a parent source file (the master file from which the output was generated), // just add of the evaluated dependencies as they are all relevant in this case. // ITaskItem [] dependenciesItemArray = new ITaskItem [parser.Dependencies.Count]; parser.Dependencies.CopyTo (dependenciesItemArray, 0); trackedFileManager.AddDependencyForSources (dependenciesItemArray, new ITaskItem [] { parentSourceItem }); } else { // // If we can't determine a parent source file (the master file from which the output was generated), // we'll want to add entries for all the dependencies which aren't already flagged as sources. // NOTE: This isn't ideal, but should reduce likelyhood of required dependencies being missed. // Dictionary<string, ITaskItem> nonSourceDependencies = new Dictionary<string, ITaskItem> (parser.Dependencies.Count); foreach (ITaskItem dependency in parser.Dependencies) { string dependencyFullPath = dependency.GetMetadata ("FullPath"); if (collatedFullPathSources.ContainsKey (dependencyFullPath)) { continue; } if (nonSourceDependencies.ContainsKey (dependencyFullPath)) { continue; } nonSourceDependencies.Add (dependencyFullPath, dependency); } ITaskItem [] nonSourceDependenciesArray = new ITaskItem [nonSourceDependencies.Count]; nonSourceDependencies.Values.CopyTo (nonSourceDependenciesArray, 0); trackedFileManager.AddDependencyForSources (nonSourceDependenciesArray, sources); } } } } // // In some instances, we need to use metadata to search for dependency files in alternative locations. // foreach (KeyValuePair<string, List<ITaskItem>> commandKeyPair in commandDictionary) { foreach (ITaskItem source in commandKeyPair.Value) { Dictionary<string, string> alternateDependencyFilePermutations = new Dictionary<string, string> (4); string dependentOutputFile = source.GetMetadata ("OutputFile"); string dependentObjectFileName = source.GetMetadata ("ObjectFileName"); if (!string.IsNullOrWhiteSpace (dependentOutputFile)) { string [] permutations = new string [] { Path.ChangeExtension (dependentOutputFile, ".d"), dependentOutputFile + ".d" }; for (int i = 0; i < permutations.Length; ++i) { if (!alternateDependencyFilePermutations.ContainsKey (permutations [i])) { alternateDependencyFilePermutations.Add (permutations [i], dependentOutputFile); } } } if (!string.IsNullOrWhiteSpace (dependentObjectFileName)) { string [] permutations = new string [] { Path.ChangeExtension (dependentObjectFileName, ".d"), dependentObjectFileName + ".d" }; for (int i = 0; i < permutations.Length; ++i) { if (!alternateDependencyFilePermutations.ContainsKey (permutations [i])) { alternateDependencyFilePermutations.Add (permutations [i], dependentObjectFileName); } } } // // Iterate through each possible dependency file. Cache listings so that similar outputs aren't re-parsed (i.e. static/shared libraries from object files) // foreach (KeyValuePair<string, string> dependencyKeyPair in alternateDependencyFilePermutations) { // // Validate this permutation is something we'd expect. // string dependencyFile = dependencyKeyPair.Key; if (string.IsNullOrWhiteSpace (dependencyFile)) { continue; } else if (string.IsNullOrWhiteSpace (Path.GetFileNameWithoutExtension (dependencyFile))) { continue; } else if (!File.Exists (dependencyFile)) { continue; } // // Probe and cache each dependency file. Saves re-parsing identical file references each time. // GccUtilities.DependencyParser parser = new GccUtilities.DependencyParser (); parser.Parse (dependencyFile); #if DEBUG Log.LogMessageFromText (string.Format ("[{0}] --> Dependencies (Read) : {1} (Entries: {2})", ToolName, dependencyFile, parser.Dependencies.Count), MessageImportance.Low); Log.LogMessageFromText (string.Format ("[{0}] --> Dependencies (Read) : [{1}] '{2}'", ToolName, "+", parser.OutputFile.GetMetadata ("FullPath")), MessageImportance.Low); int index = 0; foreach (var dependency in parser.Dependencies) { Log.LogMessageFromText (string.Format ("[{0}] --> Dependencies (Read) : [{1}] '{2}'", ToolName, ++index, dependency), MessageImportance.Low); } #endif ITaskItem [] dependenciesItemArray = new ITaskItem [parser.Dependencies.Count]; parser.Dependencies.CopyTo (dependenciesItemArray, 0); trackedFileManager.AddDependencyForSources (dependenciesItemArray, new ITaskItem [] { source }); } } } trackedFileManager.Save (TLogReadFiles [0]); } } catch (Exception e) { Log.LogErrorFromException (e, true, true, null); } }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected virtual void AddTaskSpecificOutputFiles (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected virtual void AddTaskSpecificDependencies (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected virtual void OutputWriteTLog (Dictionary<string, List<ITaskItem>> commandDictionary, ITaskItem [] sources) { try { if (!TrackFileAccess) { throw new InvalidOperationException ("'TrackFileAccess' is not set. Should not be attempting to output write TLog."); } if (commandDictionary == null) { throw new ArgumentNullException ("commandDictionary"); } if (sources == null) { throw new ArgumentNullException ("sources"); } if ((TLogWriteFiles == null) || (TLogWriteFiles.Length != 1)) { throw new InvalidOperationException ("TLogWriteFiles is missing or does not have a length of 1"); } TrackedFileManager trackedFileManager = new TrackedFileManager (); if (trackedFileManager != null) { // // Clear any old entries to sources which have just been processed. // trackedFileManager.ImportFromExistingTLog (TLogWriteFiles [0]); trackedFileManager.RemoveSourcesFromTable (sources); trackedFileManager.AddSourcesToTable (sources); // // Add any explicit outputs registered by parent task. // AddTaskSpecificOutputFiles (ref trackedFileManager, sources); // // Create dependency mappings between source and explicit output file (object-file type relationship). // Dictionary<string, ITaskItem> dependantFiles = new Dictionary<string, ITaskItem> (5); foreach (KeyValuePair<string, List<ITaskItem>> keyPair in commandDictionary) { foreach (ITaskItem source in keyPair.Value) { dependantFiles.Clear (); string outputFile = source.GetMetadata ("OutputFile"); string objectFileName = source.GetMetadata ("ObjectFileName"); if (!string.IsNullOrWhiteSpace (outputFile)) { string key = Path.GetFullPath (outputFile); if (!dependantFiles.ContainsKey (key)) { dependantFiles.Add (key, new TaskItem (key)); } } if (!string.IsNullOrWhiteSpace (objectFileName)) { string key = Path.GetFullPath (objectFileName); if (!dependantFiles.ContainsKey (key)) { dependantFiles.Add (key, new TaskItem (key)); } } if (!string.IsNullOrWhiteSpace (source.GetMetadata ("OutputFiles"))) { string [] files = source.GetMetadata ("OutputFiles").Split (new char [] { ';' }, StringSplitOptions.RemoveEmptyEntries); foreach (string file in files) { string key = Path.GetFullPath (objectFileName); if (!dependantFiles.ContainsKey (key)) { dependantFiles.Add (key, new TaskItem (key)); } } } ITaskItem [] dependencies = new ITaskItem [dependantFiles.Count]; dependantFiles.Values.CopyTo (dependencies, 0); trackedFileManager.AddDependencyForSources (dependencies, new ITaskItem [] { source }); } } trackedFileManager.Save (TLogWriteFiles [0]); } } catch (Exception e) { Log.LogErrorFromException (e, true, true, null); } }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected override void AddTaskSpecificOutputFiles (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { if (m_qualifiedOutputJars.Count > 0) { ITaskItem [] outputFiles = new ITaskItem [m_qualifiedOutputJars.Count]; m_qualifiedOutputJars.Values.CopyTo (outputFiles, 0); trackedFileManager.AddDependencyForSources (outputFiles, sources); } }
//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// protected override void AddTaskSpecificDependencies (ref TrackedFileManager trackedFileManager, ITaskItem [] sources) { // // Mark additional dependencies for .class files contained within specified class paths. // foreach (ITaskItem source in sources) { string sourceFullPath = source.GetMetadata ("FullPath"); if (Directory.Exists (sourceFullPath)) { string [] classPathFiles = Directory.GetFiles (sourceFullPath, "*.class", SearchOption.AllDirectories); List<ITaskItem> classPathFileItems = new List<ITaskItem> (classPathFiles.Length); foreach (string classpath in classPathFiles) { classPathFileItems.Add (new TaskItem (classpath)); } trackedFileManager.AddDependencyForSources (classPathFileItems.ToArray (), new ITaskItem [] { source }); } } }