Example #1
0
        /// <summary>
        /// Creates an output file which represents the same includes as the inpu tfile
        /// </summary>
        /// <param name="InputFile">The input file that this output file corresponds to</param>
        /// <param name="Includes">The active set of includes parsed for this file</param>
        /// <param name="Log">Writer for log messages</param>
        public static OutputFile CreatePassthroughOutputFile(SourceFile InputFile, List <OutputFileInclude> Includes, LineBasedTextWriter Log)
        {
            // Write the state
            InputFile.LogVerbose("InputFile={0}", InputFile.Location.FullName);
            InputFile.LogVerbose("Duplicate.");

            // Reduce the list of includes to those that are required.
            HashSet <SourceFragment> Dependencies = new HashSet <SourceFragment>();

            for (int FragmentIdx = InputFile.Fragments.Length - 1, IncludeIdx = Includes.Count - 1; FragmentIdx >= 0; FragmentIdx--)
            {
                // Update the dependency lists for this fragment
                SourceFragment InputFragment = InputFile.Fragments[FragmentIdx];
                if (InputFragment.Dependencies != null)
                {
                    Dependencies.UnionWith(InputFragment.Dependencies);
                }
                Dependencies.Remove(InputFragment);

                // Scan backwards through the list of includes, expanding each include to those which are required
                int MarkupMin = (FragmentIdx == 0)? -1 : InputFragment.MarkupMin;
                for (; IncludeIdx >= 0 && Includes[IncludeIdx].MarkupIdx >= MarkupMin; IncludeIdx--)
                {
                    OutputFileInclude Include = Includes[IncludeIdx];
                    Include.FinalFiles.Add(Include.TargetFile);
                    Dependencies.ExceptWith(Include.TargetFile.IncludedFragments);
                    Dependencies.UnionWith(Include.TargetFile.Dependencies);
                }
            }

            // Create the optimized file
            return(new OutputFile(InputFile, Includes, new HashSet <SourceFragment>(), new List <Symbol>()));
        }
Example #2
0
        /// <summary>
        /// Creates an optimized output file
        /// </summary>
        /// <param name="InputFile">The input file that this output file corresponds to</param>
        /// <param name="HeaderFile">The corresponding header file</param>
        /// <param name="PreviousFiles">List of files parsed before this one</param>
        /// <param name="Includes">The active set of includes parsed for this file</param>
        /// <param name="InputFileStack">The active include stack</param>
        /// <param name="FwdSymbolToHeader"></param>
        /// <param name="bMakeStandalone">Whether to make this output file standalone</param>
        /// <param name="Log">Writer for log messages</param>
        public static OutputFile CreateOptimizedOutputFile(SourceFile InputFile, OutputFile HeaderFile, List <OutputFile> PreviousFiles, List <OutputFileInclude> Includes, List <SourceFile> InputFileStack, Dictionary <Symbol, OutputFile> FwdSymbolToHeader, bool bMakeStandalone, LineBasedTextWriter Log)
        {
            Debug.Assert(HeaderFile == null || (InputFile.Flags & SourceFileFlags.TranslationUnit) != 0);

            // Write the state
            InputFile.LogVerbose("InputFile={0}", InputFile.Location.FullName);
            InputFile.LogVerbose("InputFile.Flags={0}", InputFile.Flags.ToString());
            if (HeaderFile != null)
            {
                InputFile.LogVerbose("HeaderFile={0}", HeaderFile.InputFile.Location.FullName);
                InputFile.LogVerbose("HeaderFile.Flags={0}", HeaderFile.InputFile.Flags.ToString());
            }
            InputFile.LogVerbose("");
            for (int Idx = 0; Idx < InputFileStack.Count; Idx++)
            {
                InputFile.LogVerbose("InputFileStack[{0}]={1}", Idx, InputFileStack[Idx].Location.FullName);
            }
            InputFile.LogVerbose("");
            for (int Idx = 0; Idx < PreviousFiles.Count; Idx++)
            {
                InputFile.LogVerbose("PreviousFiles[{0}]={1}", Idx, PreviousFiles[Idx].InputFile.Location.FullName);
            }
            InputFile.LogVerbose("");
            for (int Idx = 0; Idx < Includes.Count; Idx++)
            {
            }
            InputFile.LogVerbose("");
            for (int Idx = 0; Idx < Includes.Count; Idx++)
            {
                OutputFileInclude Include = Includes[Idx];
                InputFile.LogVerbose("Includes[{0}]={1}", Idx, Includes[Idx].TargetFile.InputFile.Location.FullName);
                foreach (SourceFragment Fragment in Include.FinalFiles.SelectMany(x => x.IncludedFragments))
                {
                    InputFile.LogVerbose("Includes[{0}].FinalFiles.IncludedFragments={1}", Idx, Fragment);
                }
            }

            // Traverse through all the included headers, figuring out the first unique include for each file and fragment
            HashSet <OutputFile>     VisitedFiles     = new HashSet <OutputFile>();
            HashSet <SourceFragment> VisitedFragments = new HashSet <SourceFragment>();

            // Go through the standalone headers first
            OutputFile MonolithicHeader = null;

            if (HeaderFile == null && (InputFile.Flags & SourceFileFlags.Standalone) != 0 && (InputFile.Flags & SourceFileFlags.External) == 0 && (InputFile.Flags & SourceFileFlags.Aggregate) == 0)
            {
                // Insert a dummy include to receive all the inserted headers
                OutputFileInclude ImplicitInclude = new OutputFileInclude(-1, null);
                ImplicitInclude.ExpandedReferences = new List <OutputFileReference>();
                Includes.Insert(0, ImplicitInclude);

                // Determine which monolithic header to use
                IEnumerable <OutputFile> PotentialMonolithicHeaders = PreviousFiles.Union(Includes.Select(x => x.TargetFile).Where(x => x != null).SelectMany(x => x.IncludedFiles));
                if (InputFile.Module != null && InputFile.Module.PublicDependencyModules.Union(InputFile.Module.PrivateDependencyModules).Any(x => x.Name == "Core"))
                {
                    MonolithicHeader = PotentialMonolithicHeaders.FirstOrDefault(x => (x.InputFile.Flags & SourceFileFlags.IsCoreMinimal) != 0);
                }
                else
                {
                    MonolithicHeader = PotentialMonolithicHeaders.FirstOrDefault(x => (x.InputFile.Flags & SourceFileFlags.IsCoreTypes) != 0);
                }

                // Update the dependencies to treat all the contents of a monolithic header as pinned
                if (MonolithicHeader != null)
                {
                    SourceFragment[] UniqueFragments = MonolithicHeader.IncludedFragments.Except(VisitedFragments).ToArray();
                    ImplicitInclude.ExpandedReferences.Add(new OutputFileReference(MonolithicHeader, UniqueFragments));
                    VisitedFragments.UnionWith(UniqueFragments);
                    VisitedFiles.Add(MonolithicHeader);
                }

                // Insert all the forward declaration headers, but only treat them as supplying the forward declarations themselves. They may happen to include
                // some utility classes (eg. TSharedPtr), and we don't want to include an unrelated header to satisfy that dependency.
                foreach (OutputFile FwdHeader in FwdSymbolToHeader.Values)
                {
                    FindExpandedReferences(FwdHeader, ImplicitInclude.ExpandedReferences, VisitedFiles, VisitedFragments, true);
                }

                // Add all the other files
                if (bMakeStandalone)
                {
                    foreach (OutputFile PreviousFile in PreviousFiles)
                    {
                        if ((InputFile.Flags & SourceFileFlags.Standalone) != 0 && (PreviousFile.InputFile.Flags & SourceFileFlags.Inline) == 0 && (PreviousFile.InputFile.Flags & SourceFileFlags.Pinned) == 0 && VisitedFiles.Add(PreviousFile))
                        {
                            SourceFragment[] UniqueFragments = PreviousFile.IncludedFragments.Except(VisitedFragments).ToArray();
                            ImplicitInclude.ExpandedReferences.Add(new OutputFileReference(PreviousFile, UniqueFragments));
                            VisitedFragments.UnionWith(UniqueFragments);
                        }
                    }
                }
            }

            // Figure out a list of files which are uniquely reachable through each include. Force an include of the matching header as the first thing.
            OutputFileReference ForcedHeaderFileReference = null;

            foreach (OutputFileInclude Include in Includes)
            {
                if (Include.ExpandedReferences == null)
                {
                    Include.ExpandedReferences = new List <OutputFileReference>();
                    if (Include == Includes[0] && HeaderFile != null)
                    {
                        ForcedHeaderFileReference = new OutputFileReference(HeaderFile, HeaderFile.IncludedFragments);
                        Include.ExpandedReferences.Add(ForcedHeaderFileReference);
                        VisitedFragments.UnionWith(HeaderFile.IncludedFragments);
                    }
                    FindExpandedReferences(Include.TargetFile, Include.ExpandedReferences, VisitedFiles, VisitedFragments, true);
                }
            }

            // Find all the symbols which are referenced by this file
            HashSet <SourceFragment> FragmentsWithReferencedSymbols = new HashSet <SourceFragment>();

            foreach (SourceFragment Fragment in InputFile.Fragments)
            {
                foreach (KeyValuePair <Symbol, SymbolReferenceType> ReferencedSymbol in Fragment.ReferencedSymbols)
                {
                    if (ReferencedSymbol.Value == SymbolReferenceType.RequiresDefinition)
                    {
                        FragmentsWithReferencedSymbols.Add(ReferencedSymbol.Key.Fragment);
                    }
                }
            }

            // Aggregate headers are designed to explicitly include headers from the current module. Expand out a list of them, so they can be included when encountered.
            HashSet <OutputFile> ExplicitIncludes = new HashSet <OutputFile>();

            if ((InputFile.Flags & SourceFileFlags.Aggregate) != 0)
            {
                foreach (OutputFileInclude Include in Includes)
                {
                    ExplicitIncludes.UnionWith(Include.ExpandedReferences.Where(x => x.File.InputFile.Location.IsUnderDirectory(InputFile.Location.Directory)).Select(x => x.File));
                }
                foreach (OutputFileInclude Include in Includes)
                {
                    ExplicitIncludes.Remove(Include.TargetFile);
                }
            }

            // Create the list of remaining dependencies for this file, and add any forward declarations
            HashSet <SourceFragment> Dependencies        = new HashSet <SourceFragment>();
            List <Symbol>            ForwardDeclarations = new List <Symbol>();

            AddForwardDeclarations(InputFile, ForwardDeclarations, Dependencies, FwdSymbolToHeader);

            // Reduce the list of includes to those that are required.
            for (int FragmentIdx = InputFile.Fragments.Length - 1, IncludeIdx = Includes.Count - 1; FragmentIdx >= 0; FragmentIdx--)
            {
                // Update the dependency lists for this fragment
                SourceFragment InputFragment = InputFile.Fragments[FragmentIdx];
                if (InputFragment.Dependencies != null)
                {
                    Dependencies.UnionWith(InputFragment.Dependencies);
                }
                Dependencies.Remove(InputFragment);

                // Scan backwards through the list of includes, expanding each include to those which are required
                int MarkupMin = (FragmentIdx == 0)? -1 : InputFragment.MarkupMin;
                for (; IncludeIdx >= 0 && Includes[IncludeIdx].MarkupIdx >= MarkupMin; IncludeIdx--)
                {
                    OutputFileInclude Include = Includes[IncludeIdx];

                    // Always include the same header for aggregates
                    if ((InputFile.Flags & SourceFileFlags.Aggregate) != 0)
                    {
                        Include.FinalFiles.Insert(0, Include.TargetFile);
                        Dependencies.ExceptWith(Include.TargetFile.IncludedFragments);
                        Dependencies.UnionWith(Include.TargetFile.Dependencies);
                    }

                    // Include any indirectly included files
                    for (int Idx = Include.ExpandedReferences.Count - 1; Idx >= 0; Idx--)
                    {
                        // Make sure we haven't already added it above
                        OutputFileReference Reference = Include.ExpandedReferences[Idx];
                        if (!Include.FinalFiles.Contains(Reference.File))
                        {
                            if (Dependencies.Any(x => Reference.UniqueFragments.Contains(x)) ||
                                (Reference.File.InputFile.Flags & SourceFileFlags.Pinned) != 0 ||
                                Reference == ForcedHeaderFileReference ||
                                Reference.File == MonolithicHeader ||
                                ExplicitIncludes.Contains(Reference.File) ||
                                ((InputFile.Flags & SourceFileFlags.Aggregate) != 0 && Reference.File == Include.TargetFile) ||                                 // Always include the original header for aggregates. They are written explicitly to include certain files.
                                Reference.UniqueFragments.Any(x => FragmentsWithReferencedSymbols.Contains(x)))
                            {
                                Include.FinalFiles.Insert(0, Reference.File);
                                Dependencies.ExceptWith(Reference.File.IncludedFragments);
                                Dependencies.UnionWith(Reference.File.Dependencies);
                            }
                        }
                    }
                }
            }

            // Remove any includes that are already included by the matching header
            if (HeaderFile != null)
            {
                HashSet <OutputFile> HeaderIncludedFiles = new HashSet <OutputFile>(HeaderFile.Includes.SelectMany(x => x.FinalFiles));
                foreach (OutputFileInclude Include in Includes)
                {
                    Include.FinalFiles.RemoveAll(x => HeaderIncludedFiles.Contains(x));
                }
            }

            // Check that all the dependencies have been satisfied
            if (Dependencies.Count > 0)
            {
                // Find those which are completely invalid
                List <SourceFragment> InvalidDependencies = Dependencies.Where(x => !InputFileStack.Contains(x.File)).ToList();
                if (InvalidDependencies.Count > 0)
                {
                    Log.WriteLine("warning: {0} does not include {1}{2}; may have missing dependencies.", InputFile, String.Join(", ", InvalidDependencies.Select(x => x.Location.FullName).Take(3)), (InvalidDependencies.Count > 3)? String.Format(" and {0} others", InvalidDependencies.Count - 3) : "");
                }
                Dependencies.ExceptWith(InvalidDependencies);

                // Otherwise warn about those which were not pinned
                foreach (SourceFile DependencyFile in Dependencies.Select(x => x.File))
                {
                    Log.WriteLine("warning: {0} is included by {1} ({2}), but depends on it and should be marked as pinned.", InputFile, DependencyFile, String.Join(" -> ", InputFileStack.SkipWhile(x => x != DependencyFile).Select(x => x.Location.GetFileName())));
                }

                // Mark it as non-standalone and pinned
                InputFile.Flags = (InputFile.Flags | SourceFileFlags.Pinned) & ~SourceFileFlags.Standalone;
            }

            // Do one more forward pass through all the headers, and remove anything that's included more than once. That can happen if we have a referenced symbol as well as
            // an explicit include, for example.
            HashSet <OutputFile> FinalIncludes = new HashSet <OutputFile>();

            foreach (OutputFileInclude Include in Includes)
            {
                for (int Idx = 0; Idx < Include.FinalFiles.Count; Idx++)
                {
                    if (!FinalIncludes.Add(Include.FinalFiles[Idx]))
                    {
                        Include.FinalFiles.RemoveAt(Idx);
                        Idx--;
                    }
                }
            }

            // Create the optimized file
            OutputFile OptimizedFile = new OutputFile(InputFile, Includes, Dependencies, ForwardDeclarations);

            // Write the verbose log
            InputFile.LogVerbose("");
            foreach (OutputFile IncludedFile in OptimizedFile.IncludedFiles)
            {
                InputFile.LogVerbose("Output: {0}", IncludedFile.InputFile.Location.FullName);
            }

            // Return the optimized file
            return(OptimizedFile);
        }