| public static ToNormalizedFullPath ( this path ) : string | ||
| path | this | |
| return | string | |
public ExtractMetadataWorker(ExtractMetadataInputModel input, bool rebuild, bool useCompatibilityFileName)
{
_rawInput = input;
_validInput = ValidateInput(input);
_rebuild = rebuild;
_shouldSkipMarkup = input.ShouldSkipMarkup;
_preserveRawInlineComments = input.PreserveRawInlineComments;
_filterConfigFile = TypeForwardedToStringExtension.ToNormalizedFullPath(input.FilterConfigFile);
_useCompatibilityFileName = useCompatibilityFileName;
}
public void SaveToCache(IEnumerable <string> inputProjects, IDictionary <string, List <string> > containedFiles, DateTime triggeredTime, string outputFolder, IList <string> fileRelativePaths, bool shouldSkipMarkup)
{
var key = TypeForwardedToStringExtension.GetNormalizedFullPathKey(inputProjects);
DateTime completeTime = DateTime.UtcNow;
BuildInfo info = new BuildInfo
{
InputFilesKey = key,
ContainedFiles = containedFiles,
TriggeredUtcTime = triggeredTime,
CompleteUtcTime = completeTime,
OutputFolder = TypeForwardedToStringExtension.ToNormalizedFullPath(outputFolder),
RelatvieOutputFiles = TypeForwardedToStringExtension.GetNormalizedPathList(fileRelativePaths),
BuildAssembly = AssemblyName,
ShouldSkipMarkup = shouldSkipMarkup
};
this.SaveConfig(key, info);
}
private static IEnumerable <string> GetTransitiveProjectReferences(ConcurrentDictionary <string, Project> projectCache, Project project)
{
var solution = project.Solution;
foreach (var pr in project.ProjectReferences)
{
var refProject = solution.GetProject(pr.ProjectId);
var path = TypeForwardedToStringExtension.ToNormalizedFullPath(refProject.FilePath);
if (projectCache.ContainsKey(path))
{
yield return(path);
}
else
{
foreach (var rpr in GetTransitiveProjectReferences(projectCache, refProject))
{
yield return(rpr);
}
}
}
}
private static Task <Tuple <MetadataItem, bool> > GetProjectMetadataFromCacheAsync(Project project, Compilation compilation, string outputFolder, ProjectDocumentCache documentCache, bool forceRebuild, bool shouldSkipMarkup, bool preserveRawInlineComments, string filterConfigFile, IReadOnlyDictionary <Compilation, IEnumerable <IMethodSymbol> > extensionMethods, bool isReferencedProjectRebuilt)
{
var projectFilePath = project.FilePath;
var k = documentCache.GetDocuments(projectFilePath);
return(GetMetadataFromProjectLevelCacheAsync(
project,
new[] { projectFilePath, filterConfigFile },
s => Task.FromResult(forceRebuild || s.AreFilesModified(k.Concat(new string[] { filterConfigFile })) || isReferencedProjectRebuilt),
s => Task.FromResult(compilation),
s =>
{
return new Dictionary <string, List <string> > {
{ TypeForwardedToStringExtension.ToNormalizedFullPath(s.FilePath), k.ToList() }
};
},
outputFolder,
preserveRawInlineComments,
shouldSkipMarkup,
filterConfigFile,
extensionMethods));
}
private static void FillProjectDependencyGraph(ConcurrentDictionary <string, Project> projectCache, ConcurrentDictionary <string, List <string> > projectDependencyGraph, Project project)
{
projectDependencyGraph.GetOrAdd(TypeForwardedToStringExtension.ToNormalizedFullPath(project.FilePath), GetTransitiveProjectReferences(projectCache, project).Distinct().ToList());
}
private async Task SaveAllMembersFromCacheAsync(IEnumerable <string> inputs, string outputFolder, bool forceRebuild)
{
var projectCache = new ConcurrentDictionary <string, Project>();
// Project<=>Documents
var documentCache = new ProjectDocumentCache();
var projectDependencyGraph = new ConcurrentDictionary <string, List <string> >();
DateTime triggeredTime = DateTime.UtcNow;
var solutions = inputs.Where(s => IsSupportedSolution(s));
var projects = inputs.Where(s => IsSupportedProject(s));
var sourceFiles = inputs.Where(s => IsSupportedSourceFile(s));
// Exclude not supported files from inputs
inputs = solutions.Concat(projects).Concat(sourceFiles);
// Add filter config file into inputs and cache
if (!string.IsNullOrEmpty(_filterConfigFile))
{
inputs = inputs.Concat(new string[] { _filterConfigFile });
documentCache.AddDocument(_filterConfigFile, _filterConfigFile);
}
// No matter is incremental or not, we have to load solutions into memory
await solutions.ForEachInParallelAsync(async path =>
{
documentCache.AddDocument(path, path);
var solution = await GetSolutionAsync(path);
if (solution != null)
{
foreach (var project in solution.Projects)
{
var filePath = project.FilePath;
// If the project is csproj/vbproj, add to project dictionary, otherwise, ignore
if (IsSupportedProject(filePath))
{
projectCache.GetOrAdd(TypeForwardedToStringExtension.ToNormalizedFullPath(project.FilePath), s => project);
}
else
{
var value = string.Join(",", SupportedExtensions);
Logger.Log(LogLevel.Warning, $"Project {filePath} inside solution {path} is not supported, supported file extension are: {value}. The project will be ignored.");
}
}
}
}, 60);
// Load additional projects out if it is not contained in expanded solution
projects = projects.Except(projectCache.Keys).Distinct();
await projects.ForEachInParallelAsync(async path =>
{
var project = await GetProjectAsync(path);
if (project != null)
{
projectCache.GetOrAdd(path, s => project);
}
}, 60);
foreach (var item in projectCache)
{
var path = item.Key;
var project = item.Value;
documentCache.AddDocument(path, path);
documentCache.AddDocuments(path, project.Documents.Select(s => s.FilePath));
documentCache.AddDocuments(path, project.MetadataReferences
.Where(s => s is PortableExecutableReference)
.Select(s => ((PortableExecutableReference)s).FilePath));
FillProjectDependencyGraph(projectCache, projectDependencyGraph, project);
}
documentCache.AddDocuments(sourceFiles);
// Incremental check for inputs as a whole:
var applicationCache = ApplicationLevelCache.Get(inputs);
if (!forceRebuild)
{
BuildInfo buildInfo = applicationCache.GetValidConfig(inputs);
if (buildInfo != null && buildInfo.ShouldSkipMarkup == _shouldSkipMarkup)
{
IncrementalCheck check = new IncrementalCheck(buildInfo);
// 1. Check if sln files/ project files and its contained documents/ source files are modified
var projectModified = check.AreFilesModified(documentCache.Documents);
if (!projectModified)
{
// 2. Check if documents/ assembly references are changed in a project
// e.g. <Compile Include="*.cs* /> and file added/deleted
foreach (var project in projectCache.Values)
{
var key = TypeForwardedToStringExtension.ToNormalizedFullPath(project.FilePath);
IEnumerable <string> currentContainedFiles = documentCache.GetDocuments(project.FilePath);
var previousDocumentCache = new ProjectDocumentCache(buildInfo.ContainedFiles);
IEnumerable <string> previousContainedFiles = previousDocumentCache.GetDocuments(project.FilePath);
if (previousContainedFiles != null && currentContainedFiles != null)
{
projectModified = !previousContainedFiles.SequenceEqual(currentContainedFiles);
}
else
{
// When one of them is not null, project is modified
if (!object.Equals(previousContainedFiles, currentContainedFiles))
{
projectModified = true;
}
}
if (projectModified)
{
break;
}
}
}
if (!projectModified)
{
// Nothing modified, use the result in cache
try
{
CopyFromCachedResult(buildInfo, inputs, outputFolder);
return;
}
catch (Exception e)
{
Logger.Log(LogLevel.Warning, $"Unable to copy results from cache: {e.Message}. Rebuild starts.");
}
}
}
}
// Build all the projects to get the output and save to cache
List <MetadataItem> projectMetadataList = new List <MetadataItem>();
ConcurrentDictionary <string, bool> projectRebuildInfo = new ConcurrentDictionary <string, bool>();
ConcurrentDictionary <string, Compilation> compilationCache = await GetProjectCompilationAsync(projectCache);
var extensionMethods = GetAllExtensionMethods(compilationCache.Values);
foreach (var key in GetTopologicalSortedItems(projectDependencyGraph))
{
var dependencyRebuilt = projectDependencyGraph[key].Any(r => projectRebuildInfo[r]);
var projectMetadataResult = await GetProjectMetadataFromCacheAsync(projectCache[key], compilationCache[key], outputFolder, documentCache, forceRebuild, _shouldSkipMarkup, _preserveRawInlineComments, _filterConfigFile, extensionMethods, dependencyRebuilt);
var projectMetadata = projectMetadataResult.Item1;
if (projectMetadata != null)
{
projectMetadataList.Add(projectMetadata);
}
projectRebuildInfo[key] = projectMetadataResult.Item2;
}
var csFiles = sourceFiles.Where(s => IsSupportedCSSourceFile(s));
if (csFiles.Any())
{
var csContent = string.Join(Environment.NewLine, csFiles.Select(s => File.ReadAllText(s)));
var csCompilation = CompilationUtility.CreateCompilationFromCsharpCode(csContent);
if (csCompilation != null)
{
var csMetadata = await GetFileMetadataFromCacheAsync(csFiles, csCompilation, outputFolder, forceRebuild, _shouldSkipMarkup, _preserveRawInlineComments, _filterConfigFile, extensionMethods);
if (csMetadata != null)
{
projectMetadataList.Add(csMetadata.Item1);
}
}
}
var vbFiles = sourceFiles.Where(s => IsSupportedVBSourceFile(s));
if (vbFiles.Any())
{
var vbContent = string.Join(Environment.NewLine, vbFiles.Select(s => File.ReadAllText(s)));
var vbCompilation = CompilationUtility.CreateCompilationFromVBCode(vbContent);
if (vbCompilation != null)
{
var vbMetadata = await GetFileMetadataFromCacheAsync(vbFiles, vbCompilation, outputFolder, forceRebuild, _preserveRawInlineComments, _shouldSkipMarkup, _filterConfigFile, extensionMethods);
if (vbMetadata != null)
{
projectMetadataList.Add(vbMetadata.Item1);
}
}
}
var allMemebers = MergeYamlProjectMetadata(projectMetadataList);
var allReferences = MergeYamlProjectReferences(projectMetadataList);
if (allMemebers == null || allMemebers.Count == 0)
{
var value = TypeForwardedToStringExtension.ToDelimitedString(projectMetadataList.Select(s => s.Name));
Logger.Log(LogLevel.Warning, $"No metadata is generated for {value}.");
applicationCache.SaveToCache(inputs, null, triggeredTime, outputFolder, null, _shouldSkipMarkup);
}
else
{
// TODO: need an intermediate folder? when to clean it up?
// Save output to output folder
var outputFiles = ResolveAndExportYamlMetadata(allMemebers, allReferences, outputFolder, _validInput.IndexFileName, _validInput.TocFileName, _validInput.ApiFolderName, _preserveRawInlineComments, _shouldSkipMarkup, _rawInput.ExternalReferences, _useCompatibilityFileName);
applicationCache.SaveToCache(inputs, documentCache.Cache, triggeredTime, outputFolder, outputFiles, _shouldSkipMarkup);
}
}
private static ExtractMetadataInputModel ValidateInput(ExtractMetadataInputModel input)
{
if (input == null)
{
return(null);
}
if (input.Items == null || input.Items.Count == 0)
{
Logger.Log(LogLevel.Warning, "No source project or file to process, exiting...");
return(null);
}
var items = new Dictionary <string, List <string> >();
// 1. Input file should exists
foreach (var pair in input.Items)
{
if (string.IsNullOrWhiteSpace(pair.Key))
{
var value = string.Join(", ", pair.Value);
Logger.Log(LogLevel.Warning, $"Empty folder name is found: '{pair.Key}': '{value}'. It is not supported, skipping.");
continue;
}
// HashSet to guarantee the input file path is unique
HashSet <string> validFilePath = new HashSet <string>();
foreach (var inputFilePath in pair.Value)
{
if (!string.IsNullOrEmpty(inputFilePath))
{
if (File.Exists(inputFilePath))
{
if (IsSupported(inputFilePath))
{
var path = TypeForwardedToStringExtension.ToNormalizedFullPath(inputFilePath);
validFilePath.Add(path);
}
else
{
var value = string.Join(",", SupportedExtensions);
Logger.Log(LogLevel.Warning, $"File {inputFilePath} is not supported, supported file extension are: {value}. The file will be ignored.");
}
}
else
{
Logger.Log(LogLevel.Warning, $"File {inputFilePath} does not exist, will be ignored.");
}
}
}
if (validFilePath.Count > 0)
{
items.Add(pair.Key, validFilePath.ToList());
}
}
if (items.Count > 0)
{
var clone = input.Clone();
clone.Items = items;
return(clone);
}
else
{
return(null);
}
}
