示例#1
0
 public bool HasChanged(IncrementalCheck check, bool careMSBuildProperties)
 {
     return(check.BuildInfo.Options == null ||
            check.BuildInfo.Options.ShouldSkipMarkup != ShouldSkipMarkup ||
            check.BuildInfo.Options.PreserveRawInlineComments != PreserveRawInlineComments ||
            check.BuildInfo.Options.FilterConfigFile != FilterConfigFile ||
            check.IsFileModified(FilterConfigFile) ||
            (careMSBuildProperties && check.MSBuildPropertiesUpdated(MSBuildProperties)));
 }
示例#2
0
        private async Task SaveAllMembersFromCacheAsync()
        {
            var forceRebuild = _rebuild;
            var outputFolder = _outputFolder;

            var projectCache = new ConcurrentDictionary <string, AbstractProject>();

            // Project<=>Documents
            var      documentCache          = new ProjectDocumentCache();
            var      projectDependencyGraph = new ConcurrentDictionary <string, List <string> >();
            DateTime triggeredTime          = DateTime.UtcNow;

            // Exclude not supported files from inputs
            var cacheKey = GetCacheKey(_files.SelectMany(s => s.Value));

            Logger.LogInfo("Loading projects...");
            if (_files.TryGetValue(FileType.Solution, out var sln))
            {
                var solutions = sln.Select(s => s.NormalizedPath);
                // No matter is incremental or not, we have to load solutions into memory
                foreach (var path in solutions)
                {
                    using (new LoggerFileScope(path))
                    {
                        documentCache.AddDocument(path, path);
                        var solution = await GetSolutionAsync(path);

                        if (solution != null)
                        {
                            foreach (var project in solution.Projects)
                            {
                                var projectFile = new FileInformation(project.FilePath);

                                // If the project is supported, add to project dictionary, otherwise, ignore
                                if (projectFile.Type == FileType.Project)
                                {
                                    projectCache.GetOrAdd(projectFile.NormalizedPath,
                                                          s => _loader.Load(projectFile.NormalizedPath));
                                }
                                else
                                {
                                    Logger.LogWarning($"Project {projectFile.RawPath} inside solution {path} is ignored, supported projects are csproj, fsproj and vbproj.");
                                }
                            }
                        }
                    }
                }
            }

            if (_files.TryGetValue(FileType.Project, out var p))
            {
                foreach (var pp in p)
                {
                    GetProject(projectCache, pp.NormalizedPath);
                }
            }

            foreach (var item in projectCache)
            {
                var path    = item.Key;
                var project = item.Value;
                documentCache.AddDocument(path, path);
                if (project.HasDocuments)
                {
                    documentCache.AddDocuments(path, project.Documents.Select(s => s.FilePath));
                }
                else
                {
                    Logger.Log(LogLevel.Warning, $"Project '{project.FilePath}' does not contain any documents.");
                }
                documentCache.AddDocuments(path, project.PortableExecutableMetadataReferences);
                FillProjectDependencyGraph(projectCache, projectDependencyGraph, project);
            }

            var csFiles       = new List <string>();
            var vbFiles       = new List <string>();
            var assemblyFiles = new List <string>();

            if (_files.TryGetValue(FileType.CSSourceCode, out var cs))
            {
                csFiles.AddRange(cs.Select(s => s.NormalizedPath));
                documentCache.AddDocuments(csFiles);
            }

            if (_files.TryGetValue(FileType.VBSourceCode, out var vb))
            {
                vbFiles.AddRange(vb.Select(s => s.NormalizedPath));
                documentCache.AddDocuments(vbFiles);
            }

            if (_files.TryGetValue(FileType.Assembly, out var asm))
            {
                assemblyFiles.AddRange(asm.Select(s => s.NormalizedPath));
                documentCache.AddDocuments(assemblyFiles);
            }

            // Incremental check for inputs as a whole:
            var applicationCache = ApplicationLevelCache.Get(cacheKey);

            var options = _options;

            if (!forceRebuild)
            {
                var buildInfo = applicationCache.GetValidConfig(cacheKey);
                if (buildInfo != null)
                {
                    IncrementalCheck check = new IncrementalCheck(buildInfo);
                    if (!options.HasChanged(check, true))
                    {
                        // 1. Check if sln files/ project files and its contained documents/ source files are modified
                        var projectModified = check.AreFilesModified(documentCache.Documents);

                        if (!projectModified)
                        {
                            // 2. Check if documents/ assembly references are changed in a project
                            // e.g. <Compile Include="*.cs* /> and file added/deleted
                            foreach (var project in projectCache.Values)
                            {
                                var key = StringExtension.ToNormalizedFullPath(project.FilePath);
                                IEnumerable <string> currentContainedFiles = documentCache.GetDocuments(project.FilePath);
                                var previousDocumentCache = new ProjectDocumentCache(buildInfo.ContainedFiles);

                                IEnumerable <string> previousContainedFiles = previousDocumentCache.GetDocuments(project.FilePath);
                                if (previousContainedFiles != null && currentContainedFiles != null)
                                {
                                    projectModified = !previousContainedFiles.SequenceEqual(currentContainedFiles);
                                }
                                else
                                {
                                    // When one of them is not null, project is modified
                                    if (!object.Equals(previousContainedFiles, currentContainedFiles))
                                    {
                                        projectModified = true;
                                    }
                                }
                                if (projectModified)
                                {
                                    break;
                                }
                            }
                        }

                        if (!projectModified)
                        {
                            // Nothing modified, use the result in cache
                            try
                            {
                                CopyFromCachedResult(buildInfo, cacheKey, outputFolder);
                                return;
                            }
                            catch (Exception e)
                            {
                                Logger.Log(LogLevel.Warning, $"Unable to copy results from cache: {e.Message}. Rebuild starts.");
                            }
                        }
                    }
                }
            }

            Logger.LogInfo("Generating metadata for each project...");

            // Build all the projects to get the output and save to cache
            List <MetadataItem> projectMetadataList = new List <MetadataItem>();
            ConcurrentDictionary <string, bool> projectRebuildInfo = new ConcurrentDictionary <string, bool>();
            ConcurrentDictionary <string, AbstractCompilation> compilationCache =
                await GetProjectCompilationAsync(projectCache);

            var roslynProjects = compilationCache.Values.OfType <RoslynCompilation>().Select(rc => rc.Compilation);

            options.RoslynExtensionMethods =
                RoslynIntermediateMetadataExtractor.GetAllExtensionMethodsFromCompilation(roslynProjects);
            foreach (var key in GetTopologicalSortedItems(projectDependencyGraph))
            {
                var dependencyRebuilt = projectDependencyGraph[key].Any(r => projectRebuildInfo[r]);
                var k          = documentCache.GetDocuments(key);
                var input      = new ProjectFileInputParameters(options, k, key, dependencyRebuilt);
                var controller = compilationCache[key].GetBuildController();

                var projectMetadataResult = GetMetadataFromProjectLevelCache(controller, input);
                var projectMetadata       = projectMetadataResult.Item1;
                if (projectMetadata != null)
                {
                    projectMetadataList.Add(projectMetadata);
                }
                projectRebuildInfo[key] = projectMetadataResult.Item2;
            }

            if (csFiles.Count > 0)
            {
                var csContent     = string.Join(Environment.NewLine, csFiles.Select(s => File.ReadAllText(s)));
                var csCompilation = CompilationUtility.CreateCompilationFromCsharpCode(csContent);
                if (csCompilation != null)
                {
                    var input      = new SourceFileInputParameters(options, csFiles);
                    var controller = new RoslynSourceFileBuildController(csCompilation);

                    var csMetadata = GetMetadataFromProjectLevelCache(controller, input);
                    if (csMetadata != null)
                    {
                        projectMetadataList.Add(csMetadata.Item1);
                    }
                }
            }

            if (vbFiles.Count > 0)
            {
                var vbContent     = string.Join(Environment.NewLine, vbFiles.Select(s => File.ReadAllText(s)));
                var vbCompilation = CompilationUtility.CreateCompilationFromVBCode(vbContent);
                if (vbCompilation != null)
                {
                    var input      = new SourceFileInputParameters(options, vbFiles);
                    var controller = new RoslynSourceFileBuildController(vbCompilation);

                    var vbMetadata = GetMetadataFromProjectLevelCache(controller, input);
                    if (vbMetadata != null)
                    {
                        projectMetadataList.Add(vbMetadata.Item1);
                    }
                }
            }

            if (assemblyFiles.Count > 0)
            {
                var assemblyCompilation = CompilationUtility.CreateCompilationFromAssembly(
                    assemblyFiles.Concat(_references ?? Enumerable.Empty <string>()));
                if (assemblyCompilation != null)
                {
                    var commentFiles = (from file in assemblyFiles
                                        select Path.ChangeExtension(file, XmlCommentFileExtension) into xmlFile
                                        where File.Exists(xmlFile)
                                        select xmlFile).ToList();

                    var referencedAssemblyList = CompilationUtility.GetAssemblyFromAssemblyComplation(assemblyCompilation, assemblyFiles).ToList();
                    // TODO: why not merge with compilation's extension methods?
                    var assemblyExtension = RoslynIntermediateMetadataExtractor.GetAllExtensionMethodsFromAssembly(assemblyCompilation, referencedAssemblyList.Select(s => s.assembly));
                    options.RoslynExtensionMethods = assemblyExtension;
                    foreach (var(reference, assembly) in referencedAssemblyList)
                    {
                        var input      = new AssemblyFileInputParameters(options, reference.Display);
                        var controller = new RoslynSourceFileBuildController(assemblyCompilation, assembly);

                        var mta = GetMetadataFromProjectLevelCache(controller, input);

                        if (mta != null)
                        {
                            MergeCommentsHelper.MergeComments(mta.Item1, commentFiles);
                            projectMetadataList.Add(mta.Item1);
                        }
                    }
                }
            }

            Dictionary <string, MetadataItem>  allMembers;
            Dictionary <string, ReferenceItem> allReferences;

            using (new PerformanceScope("MergeMetadata"))
            {
                allMembers = MergeYamlProjectMetadata(projectMetadataList);
            }

            using (new PerformanceScope("MergeReference"))
            {
                allReferences = MergeYamlProjectReferences(projectMetadataList);
            }

            if (allMembers == null || allMembers.Count == 0)
            {
                var value = StringExtension.ToDelimitedString(projectMetadataList.Select(s => s.Name));
                Logger.Log(LogLevel.Warning, $"No metadata is generated for {value}.");
                applicationCache.SaveToCache(cacheKey, null, triggeredTime, outputFolder, null, options);
            }
            else
            {
                // TODO: need an intermediate folder? when to clean it up?
                // Save output to output folder
                List <string> outputFiles;
                using (new PerformanceScope("ResolveAndExport"))
                {
                    outputFiles = ResolveAndExportYamlMetadata(allMembers, allReferences, outputFolder, options.PreserveRawInlineComments, options.ShouldSkipMarkup, _useCompatibilityFileName).ToList();
                }

                applicationCache.SaveToCache(cacheKey, documentCache.Cache, triggeredTime, outputFolder, outputFiles, options);
            }
        }
示例#3
0
        private static async Task <Tuple <MetadataItem, bool> > GetMetadataFromProjectLevelCacheAsync <T>(
            T input,
            IEnumerable <string> inputKey,
            Func <IncrementalCheck, Task <bool> > rebuildChecker,
            Func <T, Task <Compilation> > compilationProvider,
            Func <T, IDictionary <string, List <string> > > containedFilesProvider,
            string outputFolder,
            bool preserveRawInlineComments,
            bool shouldSkipMarkup,
            string filterConfigFile,
            IReadOnlyDictionary <Compilation, IEnumerable <IMethodSymbol> > extensionMethods)
        {
            DateTime triggeredTime     = DateTime.UtcNow;
            var      projectLevelCache = ProjectLevelCache.Get(inputKey);
            var      projectConfig     = projectLevelCache.GetValidConfig(inputKey);
            var      rebuildProject    = true;

            if (projectConfig != null)
            {
                var projectCheck = new IncrementalCheck(projectConfig);
                rebuildProject = await rebuildChecker(projectCheck);
            }

            MetadataItem projectMetadata;

            if (!rebuildProject)
            {
                // Load from cache
                var cacheFile = Path.Combine(projectConfig.OutputFolder, projectConfig.RelatvieOutputFiles.First());
                Logger.Log(LogLevel.Info, $"'{projectConfig.InputFilesKey}' keep up-to-date since '{projectConfig.TriggeredUtcTime.ToString()}', cached intermediate result '{cacheFile}' is used.");
                if (TryParseYamlMetadataFile(cacheFile, out projectMetadata))
                {
                    return(Tuple.Create(projectMetadata, rebuildProject));
                }
                else
                {
                    Logger.Log(LogLevel.Info, $"'{projectConfig.InputFilesKey}' is invalid, rebuild needed.");
                }
            }

            var compilation = await compilationProvider(input);

            projectMetadata = GenerateYamlMetadata(compilation, preserveRawInlineComments, filterConfigFile, extensionMethods);
            var file = Path.GetRandomFileName();
            var cacheOutputFolder = projectLevelCache.OutputFolder;
            var path = Path.Combine(cacheOutputFolder, file);

            YamlUtility.Serialize(path, projectMetadata);
            Logger.Log(LogLevel.Verbose, $"Successfully generated metadata {cacheOutputFolder} for {projectMetadata.Name}");

            IDictionary <string, List <string> > containedFiles = null;

            if (containedFilesProvider != null)
            {
                containedFiles = containedFilesProvider(input);
            }

            // Save to cache
            projectLevelCache.SaveToCache(inputKey, containedFiles, triggeredTime, cacheOutputFolder, new List <string>()
            {
                file
            }, shouldSkipMarkup);

            return(Tuple.Create(projectMetadata, rebuildProject));
        }
示例#4
0
        private async Task SaveAllMembersFromCacheAsync(IEnumerable <string> inputs, string outputFolder, bool forceRebuild)
        {
            var projectCache = new ConcurrentDictionary <string, Project>();
            // Project<=>Documents
            var      documentCache          = new ProjectDocumentCache();
            var      projectDependencyGraph = new ConcurrentDictionary <string, List <string> >();
            DateTime triggeredTime          = DateTime.UtcNow;
            var      solutions = inputs.Where(s => IsSupportedSolution(s));
            var      projects  = inputs.Where(s => IsSupportedProject(s));

            var sourceFiles = inputs.Where(s => IsSupportedSourceFile(s));

            // Exclude not supported files from inputs
            inputs = solutions.Concat(projects).Concat(sourceFiles);

            // Add filter config file into inputs and cache
            if (!string.IsNullOrEmpty(_filterConfigFile))
            {
                inputs = inputs.Concat(new string[] { _filterConfigFile });
                documentCache.AddDocument(_filterConfigFile, _filterConfigFile);
            }

            // No matter is incremental or not, we have to load solutions into memory
            await solutions.ForEachInParallelAsync(async path =>
            {
                documentCache.AddDocument(path, path);
                var solution = await GetSolutionAsync(path);
                if (solution != null)
                {
                    foreach (var project in solution.Projects)
                    {
                        var filePath = project.FilePath;

                        // If the project is csproj/vbproj, add to project dictionary, otherwise, ignore
                        if (IsSupportedProject(filePath))
                        {
                            projectCache.GetOrAdd(TypeForwardedToStringExtension.ToNormalizedFullPath(project.FilePath), s => project);
                        }
                        else
                        {
                            var value = string.Join(",", SupportedExtensions);
                            Logger.Log(LogLevel.Warning, $"Project {filePath} inside solution {path} is not supported, supported file extension are: {value}. The project will be ignored.");
                        }
                    }
                }
            }, 60);

            // Load additional projects out if it is not contained in expanded solution
            projects = projects.Except(projectCache.Keys).Distinct();

            await projects.ForEachInParallelAsync(async path =>
            {
                var project = await GetProjectAsync(path);
                if (project != null)
                {
                    projectCache.GetOrAdd(path, s => project);
                }
            }, 60);

            foreach (var item in projectCache)
            {
                var path    = item.Key;
                var project = item.Value;
                documentCache.AddDocument(path, path);
                documentCache.AddDocuments(path, project.Documents.Select(s => s.FilePath));
                documentCache.AddDocuments(path, project.MetadataReferences
                                           .Where(s => s is PortableExecutableReference)
                                           .Select(s => ((PortableExecutableReference)s).FilePath));
                FillProjectDependencyGraph(projectCache, projectDependencyGraph, project);
            }

            documentCache.AddDocuments(sourceFiles);

            // Incremental check for inputs as a whole:
            var applicationCache = ApplicationLevelCache.Get(inputs);

            if (!forceRebuild)
            {
                BuildInfo buildInfo = applicationCache.GetValidConfig(inputs);
                if (buildInfo != null && buildInfo.ShouldSkipMarkup == _shouldSkipMarkup)
                {
                    IncrementalCheck check = new IncrementalCheck(buildInfo);
                    // 1. Check if sln files/ project files and its contained documents/ source files are modified
                    var projectModified = check.AreFilesModified(documentCache.Documents);

                    if (!projectModified)
                    {
                        // 2. Check if documents/ assembly references are changed in a project
                        // e.g. <Compile Include="*.cs* /> and file added/deleted
                        foreach (var project in projectCache.Values)
                        {
                            var key = TypeForwardedToStringExtension.ToNormalizedFullPath(project.FilePath);
                            IEnumerable <string> currentContainedFiles = documentCache.GetDocuments(project.FilePath);
                            var previousDocumentCache = new ProjectDocumentCache(buildInfo.ContainedFiles);

                            IEnumerable <string> previousContainedFiles = previousDocumentCache.GetDocuments(project.FilePath);
                            if (previousContainedFiles != null && currentContainedFiles != null)
                            {
                                projectModified = !previousContainedFiles.SequenceEqual(currentContainedFiles);
                            }
                            else
                            {
                                // When one of them is not null, project is modified
                                if (!object.Equals(previousContainedFiles, currentContainedFiles))
                                {
                                    projectModified = true;
                                }
                            }
                            if (projectModified)
                            {
                                break;
                            }
                        }
                    }

                    if (!projectModified)
                    {
                        // Nothing modified, use the result in cache
                        try
                        {
                            CopyFromCachedResult(buildInfo, inputs, outputFolder);
                            return;
                        }
                        catch (Exception e)
                        {
                            Logger.Log(LogLevel.Warning, $"Unable to copy results from cache: {e.Message}. Rebuild starts.");
                        }
                    }
                }
            }

            // Build all the projects to get the output and save to cache
            List <MetadataItem> projectMetadataList = new List <MetadataItem>();
            ConcurrentDictionary <string, bool>        projectRebuildInfo = new ConcurrentDictionary <string, bool>();
            ConcurrentDictionary <string, Compilation> compilationCache   = await GetProjectCompilationAsync(projectCache);

            var extensionMethods = GetAllExtensionMethods(compilationCache.Values);

            foreach (var key in GetTopologicalSortedItems(projectDependencyGraph))
            {
                var dependencyRebuilt     = projectDependencyGraph[key].Any(r => projectRebuildInfo[r]);
                var projectMetadataResult = await GetProjectMetadataFromCacheAsync(projectCache[key], compilationCache[key], outputFolder, documentCache, forceRebuild, _shouldSkipMarkup, _preserveRawInlineComments, _filterConfigFile, extensionMethods, dependencyRebuilt);

                var projectMetadata = projectMetadataResult.Item1;
                if (projectMetadata != null)
                {
                    projectMetadataList.Add(projectMetadata);
                }
                projectRebuildInfo[key] = projectMetadataResult.Item2;
            }

            var csFiles = sourceFiles.Where(s => IsSupportedCSSourceFile(s));

            if (csFiles.Any())
            {
                var csContent     = string.Join(Environment.NewLine, csFiles.Select(s => File.ReadAllText(s)));
                var csCompilation = CompilationUtility.CreateCompilationFromCsharpCode(csContent);
                if (csCompilation != null)
                {
                    var csMetadata = await GetFileMetadataFromCacheAsync(csFiles, csCompilation, outputFolder, forceRebuild, _shouldSkipMarkup, _preserveRawInlineComments, _filterConfigFile, extensionMethods);

                    if (csMetadata != null)
                    {
                        projectMetadataList.Add(csMetadata.Item1);
                    }
                }
            }

            var vbFiles = sourceFiles.Where(s => IsSupportedVBSourceFile(s));

            if (vbFiles.Any())
            {
                var vbContent     = string.Join(Environment.NewLine, vbFiles.Select(s => File.ReadAllText(s)));
                var vbCompilation = CompilationUtility.CreateCompilationFromVBCode(vbContent);
                if (vbCompilation != null)
                {
                    var vbMetadata = await GetFileMetadataFromCacheAsync(vbFiles, vbCompilation, outputFolder, forceRebuild, _preserveRawInlineComments, _shouldSkipMarkup, _filterConfigFile, extensionMethods);

                    if (vbMetadata != null)
                    {
                        projectMetadataList.Add(vbMetadata.Item1);
                    }
                }
            }

            var allMemebers   = MergeYamlProjectMetadata(projectMetadataList);
            var allReferences = MergeYamlProjectReferences(projectMetadataList);

            if (allMemebers == null || allMemebers.Count == 0)
            {
                var value = TypeForwardedToStringExtension.ToDelimitedString(projectMetadataList.Select(s => s.Name));
                Logger.Log(LogLevel.Warning, $"No metadata is generated for {value}.");
                applicationCache.SaveToCache(inputs, null, triggeredTime, outputFolder, null, _shouldSkipMarkup);
            }
            else
            {
                // TODO: need an intermediate folder? when to clean it up?
                // Save output to output folder
                var outputFiles = ResolveAndExportYamlMetadata(allMemebers, allReferences, outputFolder, _validInput.IndexFileName, _validInput.TocFileName, _validInput.ApiFolderName, _preserveRawInlineComments, _shouldSkipMarkup, _rawInput.ExternalReferences, _useCompatibilityFileName);
                applicationCache.SaveToCache(inputs, documentCache.Cache, triggeredTime, outputFolder, outputFiles, _shouldSkipMarkup);
            }
        }
        public bool HasChanged(BuildInfo buildInfo)
        {
            var check = new IncrementalCheck(buildInfo);

            return(Options.HasChanged(check, true) || check.AreFilesModified(Files));
        }
示例#6
0
        private async Task SaveAllMembersFromCacheAsync()
        {
            var forceRebuild = _rebuild;
            var outputFolder = _outputFolder;

            var projectCache = new ConcurrentDictionary <string, Project>();

            // Project<=>Documents
            var      documentCache          = new ProjectDocumentCache();
            var      projectDependencyGraph = new ConcurrentDictionary <string, List <string> >();
            DateTime triggeredTime          = DateTime.UtcNow;

            // Exclude not supported files from inputs
            var cacheKey = GetCacheKey(_files.SelectMany(s => s.Value));

            // Add filter config file into inputs and cache
            if (!string.IsNullOrEmpty(_filterConfigFile))
            {
                cacheKey = cacheKey.Concat(new string[] { _filterConfigFile });
                documentCache.AddDocument(_filterConfigFile, _filterConfigFile);
            }

            Logger.LogInfo("Loading projects...");
            if (_files.TryGetValue(FileType.Solution, out var sln))
            {
                var solutions = sln.Select(s => s.NormalizedPath);
                // No matter is incremental or not, we have to load solutions into memory
                foreach (var path in solutions)
                {
                    documentCache.AddDocument(path, path);
                    var solution = await GetSolutionAsync(path);

                    if (solution != null)
                    {
                        foreach (var project in solution.Projects)
                        {
                            var projectFile = new FileInformation(project.FilePath);

                            // If the project is csproj/vbproj, add to project dictionary, otherwise, ignore
                            if (projectFile.IsSupportedProject())
                            {
                                projectCache.GetOrAdd(projectFile.NormalizedPath, s => project);
                            }
                            else
                            {
                                Logger.LogWarning($"Project {projectFile.RawPath} inside solution {path} is ignored, supported projects are csproj and vbproj.");
                            }
                        }
                    }
                }
            }

            if (_files.TryGetValue(FileType.Project, out var p))
            {
                foreach (var pp in p)
                {
                    GetProject(projectCache, pp.NormalizedPath);
                }
            }

            if (_files.TryGetValue(FileType.ProjectJsonProject, out var pjp))
            {
                await pjp.Select(s => s.NormalizedPath).ForEachInParallelAsync(path =>
                {
                    projectCache.GetOrAdd(path, s => GetProjectJsonProject(s));
                    return(Task.CompletedTask);
                }, 60);
            }

            foreach (var item in projectCache)
            {
                var path    = item.Key;
                var project = item.Value;
                documentCache.AddDocument(path, path);
                if (project.HasDocuments)
                {
                    documentCache.AddDocuments(path, project.Documents.Select(s => s.FilePath));
                }
                else
                {
                    Logger.Log(LogLevel.Warning, $"Project '{project.FilePath}' does not contain any documents.");
                }
                documentCache.AddDocuments(path, project.MetadataReferences
                                           .Where(s => s is PortableExecutableReference)
                                           .Select(s => ((PortableExecutableReference)s).FilePath));
                FillProjectDependencyGraph(projectCache, projectDependencyGraph, project);
                // duplicate project references will fail Project.GetCompilationAsync
                var groups = project.ProjectReferences.GroupBy(r => r);
                if (groups.Any(g => g.Count() > 1))
                {
                    projectCache[path] = project.WithProjectReferences(groups.Select(g => g.Key));
                }
            }

            var csFiles       = new List <string>();
            var vbFiles       = new List <string>();
            var assemblyFiles = new List <string>();

            if (_files.TryGetValue(FileType.CSSourceCode, out var cs))
            {
                csFiles.AddRange(cs.Select(s => s.NormalizedPath));
                documentCache.AddDocuments(csFiles);
            }

            if (_files.TryGetValue(FileType.VBSourceCode, out var vb))
            {
                vbFiles.AddRange(vb.Select(s => s.NormalizedPath));
                documentCache.AddDocuments(vbFiles);
            }

            if (_files.TryGetValue(FileType.Assembly, out var asm))
            {
                assemblyFiles.AddRange(asm.Select(s => s.NormalizedPath));
                documentCache.AddDocuments(assemblyFiles);
            }

            // Incremental check for inputs as a whole:
            var applicationCache = ApplicationLevelCache.Get(cacheKey);

            if (!forceRebuild)
            {
                var buildInfo = applicationCache.GetValidConfig(cacheKey);
                if (buildInfo != null && buildInfo.ShouldSkipMarkup == _shouldSkipMarkup)
                {
                    IncrementalCheck check = new IncrementalCheck(buildInfo);
                    // 1. Check if sln files/ project files and its contained documents/ source files are modified
                    var projectModified = check.AreFilesModified(documentCache.Documents) || check.MSBuildPropertiesUpdated(_msbuildProperties);

                    if (!projectModified)
                    {
                        // 2. Check if documents/ assembly references are changed in a project
                        // e.g. <Compile Include="*.cs* /> and file added/deleted
                        foreach (var project in projectCache.Values)
                        {
                            var key = StringExtension.ToNormalizedFullPath(project.FilePath);
                            IEnumerable <string> currentContainedFiles = documentCache.GetDocuments(project.FilePath);
                            var previousDocumentCache = new ProjectDocumentCache(buildInfo.ContainedFiles);

                            IEnumerable <string> previousContainedFiles = previousDocumentCache.GetDocuments(project.FilePath);
                            if (previousContainedFiles != null && currentContainedFiles != null)
                            {
                                projectModified = !previousContainedFiles.SequenceEqual(currentContainedFiles);
                            }
                            else
                            {
                                // When one of them is not null, project is modified
                                if (!object.Equals(previousContainedFiles, currentContainedFiles))
                                {
                                    projectModified = true;
                                }
                            }
                            if (projectModified)
                            {
                                break;
                            }
                        }
                    }

                    if (!projectModified)
                    {
                        // Nothing modified, use the result in cache
                        try
                        {
                            CopyFromCachedResult(buildInfo, cacheKey, outputFolder);
                            return;
                        }
                        catch (Exception e)
                        {
                            Logger.Log(LogLevel.Warning, $"Unable to copy results from cache: {e.Message}. Rebuild starts.");
                        }
                    }
                }
            }

            // Build all the projects to get the output and save to cache
            List <MetadataItem> projectMetadataList = new List <MetadataItem>();
            ConcurrentDictionary <string, bool>        projectRebuildInfo = new ConcurrentDictionary <string, bool>();
            ConcurrentDictionary <string, Compilation> compilationCache   = await GetProjectCompilationAsync(projectCache);

            var extensionMethods = GetAllExtensionMethodsFromCompilation(compilationCache.Values);

            foreach (var key in GetTopologicalSortedItems(projectDependencyGraph))
            {
                var dependencyRebuilt     = projectDependencyGraph[key].Any(r => projectRebuildInfo[r]);
                var projectMetadataResult = await GetProjectMetadataFromCacheAsync(projectCache[key], compilationCache[key], outputFolder, documentCache, forceRebuild, _shouldSkipMarkup, _preserveRawInlineComments, _filterConfigFile, extensionMethods, dependencyRebuilt);

                var projectMetadata = projectMetadataResult.Item1;
                if (projectMetadata != null)
                {
                    projectMetadataList.Add(projectMetadata);
                }
                projectRebuildInfo[key] = projectMetadataResult.Item2;
            }

            if (csFiles.Count > 0)
            {
                var csContent     = string.Join(Environment.NewLine, csFiles.Select(s => File.ReadAllText(s)));
                var csCompilation = CompilationUtility.CreateCompilationFromCsharpCode(csContent);
                if (csCompilation != null)
                {
                    var csMetadata = await GetFileMetadataFromCacheAsync(csFiles, csCompilation, outputFolder, forceRebuild, _shouldSkipMarkup, _preserveRawInlineComments, _filterConfigFile, extensionMethods);

                    if (csMetadata != null)
                    {
                        projectMetadataList.Add(csMetadata.Item1);
                    }
                }
            }

            if (vbFiles.Count > 0)
            {
                var vbContent     = string.Join(Environment.NewLine, vbFiles.Select(s => File.ReadAllText(s)));
                var vbCompilation = CompilationUtility.CreateCompilationFromVBCode(vbContent);
                if (vbCompilation != null)
                {
                    var vbMetadata = await GetFileMetadataFromCacheAsync(vbFiles, vbCompilation, outputFolder, forceRebuild, _preserveRawInlineComments, _shouldSkipMarkup, _filterConfigFile, extensionMethods);

                    if (vbMetadata != null)
                    {
                        projectMetadataList.Add(vbMetadata.Item1);
                    }
                }
            }

            if (assemblyFiles.Count > 0)
            {
                var assemblyCompilation = CompilationUtility.CreateCompilationFromAssembly(assemblyFiles);
                if (assemblyCompilation != null)
                {
                    var commentFiles = (from file in assemblyFiles
                                        select Path.ChangeExtension(file, XmlCommentFileExtension) into xmlFile
                                        where File.Exists(xmlFile)
                                        select xmlFile).ToList();

                    var referencedAssemblyList = CompilationUtility.GetAssemblyFromAssemblyComplation(assemblyCompilation).ToList();
                    var assemblyExtension      = GetAllExtensionMethodsFromAssembly(assemblyCompilation, referencedAssemblyList.Select(s => s.Item2));

                    foreach (var assembly in referencedAssemblyList)
                    {
                        var mta = await GetAssemblyMetadataFromCacheAsync(new string[] { assembly.Item1.Display }, assemblyCompilation, assembly.Item2, outputFolder, forceRebuild, _filterConfigFile, assemblyExtension);

                        if (mta != null)
                        {
                            MergeCommentsHelper.MergeComments(mta.Item1, commentFiles);
                            projectMetadataList.Add(mta.Item1);
                        }
                    }
                }
            }

            var allMemebers   = MergeYamlProjectMetadata(projectMetadataList);
            var allReferences = MergeYamlProjectReferences(projectMetadataList);

            if (allMemebers == null || allMemebers.Count == 0)
            {
                var value = StringExtension.ToDelimitedString(projectMetadataList.Select(s => s.Name));
                Logger.Log(LogLevel.Warning, $"No metadata is generated for {value}.");
                applicationCache.SaveToCache(cacheKey, null, triggeredTime, outputFolder, null, _shouldSkipMarkup, _msbuildProperties);
            }
            else
            {
                // TODO: need an intermediate folder? when to clean it up?
                // Save output to output folder
                var outputFiles = ResolveAndExportYamlMetadata(allMemebers, allReferences, outputFolder, _preserveRawInlineComments, _shouldSkipMarkup, _useCompatibilityFileName).ToList();
                applicationCache.SaveToCache(cacheKey, documentCache.Cache, triggeredTime, outputFolder, outputFiles, _shouldSkipMarkup, _msbuildProperties);
            }
        }