private IEnumerable <string> GetFiles(string directory, string searchPattern, int searchLevel)
{
if (searchLevel < 1)
{
return(Enumerable.Empty <string>());
}
var files = Directory.GetFiles(directory, searchPattern, SearchOption.TopDirectoryOnly);
var dirs = Directory.GetDirectories(directory);
if (searchLevel == 1)
{
foreach (var dir in dirs)
{
var remainingFiles = Directory.GetFiles(dir, searchPattern, SearchOption.AllDirectories);
if (remainingFiles.Length > 0)
{
throw new ResourceFileExceedsMaxDepthException(_maxDepth, TypeForwardedToPathUtility.MakeRelativePath(_directory, remainingFiles[0]), Name);
}
}
return(files);
}
List <string> allFiles = new List <string>(files);
foreach (var dir in dirs)
{
allFiles.AddRange(GetFiles(dir, searchPattern, searchLevel - 1));
}
return(allFiles);
}
private static ChangeList ParseCore(string tsvFile, string baseDir)
{
var result = new ChangeList();
bool hasError = false;
foreach (var line in File.ReadLines(tsvFile))
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
var columns = line.Split('\t');
if (columns.Length >= 2)
{
if (string.Equals(columns[0], "<from>", StringComparison.OrdinalIgnoreCase))
{
result.From = columns[1];
continue;
}
if (string.Equals(columns[0], "<to>", StringComparison.OrdinalIgnoreCase))
{
result.To = columns[1];
continue;
}
string path;
if (TypeForwardedToPathUtility.IsRelativePath(columns[0]))
{
path = columns[0];
}
else
{
path = TypeForwardedToPathUtility.MakeRelativePath(baseDir, columns[0]);
}
if (path != null)
{
ChangeKind kind;
if (Enum.TryParse(columns[1], true, out kind))
{
if (kind != ChangeKind.Deleted)
{
if (!File.Exists(Path.Combine(baseDir, path)))
{
Logger.LogError($"File:{path} not existed.");
hasError = true;
continue;
}
}
result.Add(path, kind);
continue;
}
}
}
Logger.LogWarning($"Ignore unknown line: {line}");
}
if (hasError)
{
throw new DocfxException($"Some error ocurred while parsing changelist file: {tsvFile}.");
}
return(result);
}
private static ImmutableDictionary <string, object> ApplyFileMetadata(
string file,
ImmutableDictionary <string, object> metadata,
FileMetadata fileMetadata)
{
if (fileMetadata == null || fileMetadata.Count == 0)
{
return(metadata);
}
var result = new Dictionary <string, object>(metadata);
var baseDir = string.IsNullOrEmpty(fileMetadata.BaseDir) ? Directory.GetCurrentDirectory() : fileMetadata.BaseDir;
var TypeForwardedToRelativePath = TypeForwardedToPathUtility.MakeRelativePath(baseDir, file);
foreach (var item in fileMetadata)
{
// As the latter one overrides the former one, match the pattern from latter to former
for (int i = item.Value.Length - 1; i >= 0; i--)
{
if (item.Value[i].Glob.Match(TypeForwardedToRelativePath))
{
// override global metadata if metadata is defined in file metadata
result[item.Value[i].Key] = item.Value[i].Value;
Logger.LogVerbose($"{TypeForwardedToRelativePath} matches file metadata with glob pattern {item.Value[i].Glob.Raw} for property {item.Value[i].Key}");
break;
}
}
}
return(result.ToImmutableDictionary());
}
public override FileModel Load(FileAndType file, ImmutableDictionary <string, object> metadata)
{
if (file.Type != DocumentType.Article)
{
throw new NotSupportedException();
}
var content = MarkdownReader.ReadMarkdownAsConceptual(file.BaseDir, file.File);
foreach (var item in metadata)
{
if (!content.ContainsKey(item.Key))
{
content[item.Key] = item.Value;
}
}
var displayLocalPath = TypeForwardedToPathUtility.MakeRelativePath(EnvironmentContext.BaseDirectory, file.FullPath);
return(new FileModel(
file,
content,
serializer: Environment.Is64BitProcess?null: new BinaryFormatter())
{
LocalPathFromRepoRoot = (content["source"] as SourceDetail)?.Remote?.RelativePath,
LocalPathFromRoot = displayLocalPath
});
}
private void WriteLine(ILogItem item)
{
if (item == null)
{
throw new ArgumentNullException(nameof(item));
}
if (item.File == null)
{
return;
}
string fileFromWorkingDir = TypeForwardedToStringExtension.BackSlashToForwardSlash(item.File);
if (!TypeForwardedToPathUtility.IsRelativePath(item.File))
{
fileFromWorkingDir = TypeForwardedToPathUtility.MakeRelativePath(EnvironmentContext.BaseDirectory, item.File);
}
List <LogItem> logsPerFile;
if (!_logs.TryGetValue(fileFromWorkingDir, out logsPerFile))
{
logsPerFile = _logs[fileFromWorkingDir] = new List <LogItem>();
}
logsPerFile.Add(new LogItem
{
File = TypeForwardedToStringExtension.BackSlashToForwardSlash(item.File),
Line = item.Line,
LogLevel = item.LogLevel,
Message = item.Message,
Phase = item.Phase,
});
}
public virtual StringBuffer Render(IMarkdownRenderer renderer, DfmFencesToken token, IMarkdownContext context)
{
if (!TypeForwardedToPathUtility.IsRelativePath(token.Path))
{
string errorMessage = $"Code absolute path: {token.Path} is not supported in file {context.GetFilePathStack().Peek()}";
Logger.LogError(errorMessage);
return(DfmFencesBlockHelper.GetRenderedFencesBlockString(token, renderer.Options, errorMessage));
}
try
{
// Always report original dependency
context.ReportDependency(token.Path);
var filePathWithStatus = DfmFallbackHelper.GetFilePathWithFallback(token.Path, context);
var extractResult = _dfmCodeExtractor.ExtractFencesCode(token, filePathWithStatus.Item1);
var result = DfmFencesBlockHelper.GetRenderedFencesBlockString(token, renderer.Options, extractResult.ErrorMessage, extractResult.FencesCodeLines);
return(result);
}
catch (DirectoryNotFoundException)
{
return(DfmFencesBlockHelper.GenerateReferenceNotFoundErrorMessage(renderer, token));
}
catch (FileNotFoundException)
{
return(DfmFencesBlockHelper.GenerateReferenceNotFoundErrorMessage(renderer, token));
}
}
/// <summary>
/// Append default extension to href by condition
/// </summary>
/// <param name="href">original href string</param>
/// <param name="defaultExtension">default extension to append</param>
/// <returns>Href with default extension appended</returns>
private AppendDefaultExtensionResult AppendDefaultExtension(string href, string defaultExtension, IMarkdownContext context, string rawMarkdown, string line)
{
// If the context doesn't have necessary info, return the original href
if (!context.Variables.ContainsKey("path"))
{
return(new AppendDefaultExtensionResult(false, href, null));
}
var currentFilePath = (string)context.Variables["path"];
try
{
if (!TypeForwardedToPathUtility.IsRelativePath(href))
{
return(new AppendDefaultExtensionResult(false, href, null));
}
}
catch (ArgumentException)
{
Logger.LogWarning($"Invalid reference {href} in file: {currentFilePath}. Raw: {rawMarkdown}", null, currentFilePath, line);
return(new AppendDefaultExtensionResult(false, href, null));
}
var index = href.IndexOf('#');
if (index == -1)
{
href = href.TrimEnd('/');
var extension = Path.GetExtension(href);
// Regard all the relative path with no extension as markdown file that missing .md
if (string.IsNullOrEmpty(extension))
{
return(new AppendDefaultExtensionResult(true, $"{href}{defaultExtension}", true));
}
else
{
bool isMarkdownFile = extension.Equals(MarkdownExtension, StringComparison.OrdinalIgnoreCase);
return(new AppendDefaultExtensionResult(true, href, isMarkdownFile));
}
}
else if (index == 0)
{
return(new AppendDefaultExtensionResult(true, href, false));
}
else
{
var hrefWithoutAnchor = href.Remove(index).TrimEnd('/');
var anchor = href.Substring(index);
var extension = Path.GetExtension(hrefWithoutAnchor);
if (string.IsNullOrEmpty(extension))
{
return(new AppendDefaultExtensionResult(true, $"{hrefWithoutAnchor}{defaultExtension}{anchor}", true));
}
else
{
bool isMarkdownFile = extension.Equals(MarkdownExtension, StringComparison.OrdinalIgnoreCase);
return(new AppendDefaultExtensionResult(true, $"{hrefWithoutAnchor}{anchor}", isMarkdownFile));
}
}
}
private ImmutableDictionary <string, ImmutableList <LinkSourceInfo> > GetFileLinkSource(FileAndType ft, HtmlDocument doc)
{
var fileLinkSources = new Dictionary <string, List <LinkSourceInfo> >();
foreach (var pair in (from n in doc.DocumentNode.Descendants()
where !string.Equals(n.Name, "xref", StringComparison.OrdinalIgnoreCase)
from attr in n.Attributes
where string.Equals(attr.Name, "src", StringComparison.OrdinalIgnoreCase) ||
string.Equals(attr.Name, "href", StringComparison.OrdinalIgnoreCase)
where !string.IsNullOrWhiteSpace(attr.Value)
select new { Node = n, Attr = attr }).ToList())
{
string linkFile;
string anchor = null;
var link = pair.Attr;
if (TypeForwardedToPathUtility.IsRelativePath(link.Value))
{
var index = link.Value.IndexOfAny(UriFragmentOrQueryString);
if (index == -1)
{
linkFile = link.Value;
}
else if (index == 0)
{
continue;
}
else
{
linkFile = link.Value.Remove(index);
anchor = link.Value.Substring(index);
}
var path = (TypeForwardedToRelativePath)ft.File + (TypeForwardedToRelativePath)linkFile;
string file = path.GetPathFromWorkingFolder().UrlDecode();
if (SourceFiles.ContainsKey(file))
{
link.Value = file;
if (!string.IsNullOrEmpty(anchor) &&
string.Equals(link.Name, "href", StringComparison.OrdinalIgnoreCase))
{
pair.Node.SetAttributeValue("anchor", anchor);
}
}
List <LinkSourceInfo> sources;
if (!fileLinkSources.TryGetValue(file, out sources))
{
sources = new List <LinkSourceInfo>();
fileLinkSources[file] = sources;
}
sources.Add(new LinkSourceInfo
{
Target = file,
Anchor = anchor,
SourceFile = pair.Node.GetAttributeValue("sourceFile", null),
LineNumber = pair.Node.GetAttributeValue("sourceStartLineNumber", 0),
});
}
}
return(fileLinkSources.ToImmutableDictionary(x => x.Key, x => x.Value.ToImmutableList()));
}
public string ResolveSourceRelativePath(string originPath, string currentFileOutputPath)
{
if (string.IsNullOrEmpty(originPath) || !TypeForwardedToPathUtility.IsRelativePath(originPath))
{
return(originPath);
}
var origin = (TypeForwardedToRelativePath)originPath;
if (origin == null)
{
return(originPath);
}
var destPath = _context.GetFilePath(origin.GetPathFromWorkingFolder().ToString());
if (destPath != null)
{
return(((TypeForwardedToRelativePath)destPath - ((TypeForwardedToRelativePath)currentFileOutputPath).GetPathFromWorkingFolder()).ToString());
}
else
{
Logger.LogWarning($"Can't find output file for {originPath}");
return(originPath);
}
}
public static HrefType GetHrefType(string href)
{
if (!TypeForwardedToPathUtility.IsRelativePath(href))
{
return(HrefType.AbsolutePath);
}
var fileName = Path.GetFileName(href);
if (string.IsNullOrEmpty(fileName))
{
return(HrefType.RelativeFolder);
}
var tocFileType = GetTocFileType(href);
if (tocFileType == TocFileType.Markdown)
{
return(HrefType.MarkdownTocFile);
}
if (tocFileType == TocFileType.Yaml)
{
return(HrefType.YamlTocFile);
}
return(HrefType.RelativeFile);
}
private string LoadCore(IMarkdownRenderer adapter, string currentPath, string raw, SourceInfo sourceInfo, IMarkdownContext context, DfmEngine engine)
{
try
{
if (!TypeForwardedToPathUtility.IsRelativePath(currentPath))
{
return(GenerateErrorNodeWithCommentWrapper("INCLUDE", $"Absolute path \"{currentPath}\" is not supported.", raw, sourceInfo));
}
// Always report original include file dependency
var originalRelativePath = currentPath;
context.ReportDependency(currentPath);
var parents = context.GetFilePathStack();
string parent = string.Empty;
if (parents == null)
{
parents = ImmutableStack <string> .Empty;
}
// Update currentPath to be referencing to sourcePath
else if (!parents.IsEmpty)
{
parent = parents.Peek();
currentPath = ((TypeForwardedToRelativePath)currentPath).BasedOn((TypeForwardedToRelativePath)parent);
}
if (parents.Contains(currentPath, TypeForwardedToFilePathComparer.OSPlatformSensitiveComparer))
{
return(GenerateErrorNodeWithCommentWrapper("INCLUDE", $"Unable to resolve {raw}: Circular dependency found in \"{parent}\"", raw, sourceInfo));
}
// Add current file path to chain when entering recursion
parents = parents.Push(currentPath);
string result;
HashSet <string> dependency;
if (!_dependencyCache.TryGetValue(currentPath, out dependency) ||
!_cache.TryGet(currentPath, out result))
{
var filePathWithStatus = DfmFallbackHelper.GetFilePathWithFallback(originalRelativePath, context);
var src = File.ReadAllText(filePathWithStatus.Item1);
dependency = new HashSet <string>();
src = engine.InternalMarkup(src, context.SetFilePathStack(parents).SetDependency(dependency).SetIsInclude());
result = UpdateToHrefFromWorkingFolder(src, currentPath);
result = GenerateNodeWithCommentWrapper("INCLUDE", $"Include content from \"{currentPath}\"", result);
_cache.Add(currentPath, result);
_dependencyCache[currentPath] = dependency;
}
context.ReportDependency(
from d in dependency
select(string)((TypeForwardedToRelativePath)currentPath + (TypeForwardedToRelativePath)d - (TypeForwardedToRelativePath)parent));
return(result);
}
catch (Exception e)
{
return(GenerateErrorNodeWithCommentWrapper("INCLUDE", $"Unable to resolve {raw}:{e.Message}", raw, sourceInfo));
}
}
private static void UpdateToHrefFromWorkingFolder(HtmlNode html, string filePath)
{
foreach (var pair in GetHrefNodes(html))
{
var link = pair.Attr;
if (TypeForwardedToPathUtility.IsRelativePath(link.Value) && !TypeForwardedToRelativePath.IsPathFromWorkingFolder(link.Value) && !link.Value.StartsWith("#"))
{
link.Value = ((TypeForwardedToRelativePath)filePath + (TypeForwardedToRelativePath)link.Value).GetPathFromWorkingFolder();
}
}
}
/// <summary>
/// Append default extension to href by condition
/// </summary>
/// <param name="href">original href string</param>
/// <param name="defaultExtension">default extension to append</param>
/// <param name="isHrefRelativeNonMdFile">true if it is a relative path and not a markdown file. Otherwise false</param>
/// <returns>Href with default extension appended</returns>
private string AppendDefaultExtension(string href, string defaultExtension, out bool isHrefRelativeNonMdFile)
{
isHrefRelativeNonMdFile = false;
if (!TypeForwardedToPathUtility.IsRelativePath(href))
{
return(href);
}
var index = href.IndexOf('#');
if (index == -1)
{
href = href.TrimEnd('/');
var extension = Path.GetExtension(href);
// Regard all the relative path with no extension as markdown file that missing .md
if (string.IsNullOrEmpty(extension))
{
return($"{href}{defaultExtension}");
}
else
{
if (!extension.Equals(MarkdownExtension, StringComparison.OrdinalIgnoreCase))
{
isHrefRelativeNonMdFile = true;
}
return(href);
}
}
else if (index == 0)
{
return(href);
}
else
{
var hrefWithoutAnchor = href.Remove(index).TrimEnd('/');
var anchor = href.Substring(index);
var extension = Path.GetExtension(hrefWithoutAnchor);
if (string.IsNullOrEmpty(extension))
{
return($"{hrefWithoutAnchor}{defaultExtension}{anchor}");
}
else
{
if (!extension.Equals(MarkdownExtension, StringComparison.OrdinalIgnoreCase))
{
isHrefRelativeNonMdFile = true;
}
return($"{hrefWithoutAnchor}{anchor}");
}
}
}
private static void CopyFromCachedResult(BuildInfo buildInfo, IEnumerable <string> inputs, string outputFolder)
{
var outputFolderSource = buildInfo.OutputFolder;
var relativeFiles = buildInfo.RelatvieOutputFiles;
if (relativeFiles == null)
{
Logger.Log(LogLevel.Warning, $"No metadata is generated for '{TypeForwardedToStringExtension.ToDelimitedString(inputs)}'.");
return;
}
Logger.Log(LogLevel.Info, $"'{TypeForwardedToStringExtension.ToDelimitedString(inputs)}' keep up-to-date since '{buildInfo.TriggeredUtcTime.ToString()}', cached result from '{buildInfo.OutputFolder}' is used.");
TypeForwardedToPathUtility.CopyFilesToFolder(relativeFiles.Select(s => Path.Combine(outputFolderSource, s)), outputFolderSource, outputFolder, true, s => Logger.Log(LogLevel.Info, s), null);
}
// TODO: use this method instead of directly accessing UidMap
public void RegisterInternalXrefSpec(XRefSpec xrefSpec)
{
if (xrefSpec == null)
{
throw new ArgumentNullException(nameof(xrefSpec));
}
if (string.IsNullOrEmpty(xrefSpec.Href))
{
throw new ArgumentException("Href for xref spec must contain value");
}
if (!TypeForwardedToPathUtility.IsRelativePath(xrefSpec.Href))
{
throw new ArgumentException("Only relative href path is supported");
}
XRefSpecMap[xrefSpec.Uid] = xrefSpec;
}
public void Add(string filePath, ChangeKind kind)
{
if (filePath == null)
{
throw new ArgumentNullException(nameof(filePath));
}
if (filePath.Length == 0)
{
throw new ArgumentException("File path cannot be empty", nameof(filePath));
}
if (!TypeForwardedToPathUtility.IsRelativePath(filePath))
{
throw new ArgumentException("Expect relative path.", nameof(filePath));
}
AddCore(filePath, kind);
}
public IMarkdownToken TryMatch(IMarkdownParser engine, IMarkdownParsingContext context)
{
var match = AzureIncludeRegex.Match(context.CurrentMarkdown);
if (match.Length == 0)
{
return(null);
}
var sourceInfo = context.Consume(match.Length);
// [!azure.include[title](path "optionalTitle")]
// 1. Get include file path
var path = match.Groups[2].Value;
// 2. Get title
var value = match.Groups[1].Value;
var title = match.Groups[4].Value;
if (!TypeForwardedToPathUtility.IsRelativePath(path))
{
Logger.LogWarning($"Azure inline include path {path} is not a relative path, can't expand it");
return(new MarkdownTextToken(this, engine.Context, match.Value, sourceInfo));
}
// 3. Apply inline rules to the included content
object currentFilePath;
if (!engine.Context.Variables.TryGetValue("path", out currentFilePath))
{
Logger.LogWarning($"Can't get path for the file that ref azure inline include file, return MarkdownTextToken. Raw: {match.Value}");
return(new MarkdownTextToken(this, engine.Context, match.Value, sourceInfo));
}
var includeFilePath = TypeForwardedToPathUtility.NormalizePath(Path.Combine(Path.GetDirectoryName(currentFilePath.ToString()), path));
if (!File.Exists(includeFilePath))
{
Logger.LogWarning($"Can't get include file path {includeFilePath} in the file {currentFilePath}, return MarkdownTextToken. Raw: {match.Value}");
return(new MarkdownTextToken(this, engine.Context, match.Value, sourceInfo));
}
return(new TwoPhaseBlockToken(this, engine.Context, sourceInfo, (p, t) =>
{
var inlineTokens = p.Tokenize(SourceInfo.Create(MarkdownEngine.Normalize(File.ReadAllText(includeFilePath)), includeFilePath));
return new AzureIncludeInlineToken(t.Rule, t.Context, path, value, title, inlineTokens, match.Groups[0].Value, t.SourceInfo);
}));
}
public FileResourceCollection(string directory, int maxSearchLevel = MaxSearchLevel)
{
if (string.IsNullOrEmpty(directory))
{
_directory = Directory.GetCurrentDirectory();
}
else
{
_directory = directory;
}
Name = _directory;
_maxDepth = maxSearchLevel;
var includedFiles = GetFiles(_directory, "*", maxSearchLevel);
Names = includedFiles.Select(s => TypeForwardedToPathUtility.MakeRelativePath(_directory, s)).Where(s => s != null);
IsEmpty = !Names.Any();
}
private string CheckNonMdRelativeFileHref(string nonMdHref, IMarkdownContext context, string rawMarkdown, string line)
{
// If the context doesn't have necessary info or nonMdHref is not a relative path, return the original href
if (!context.Variables.ContainsKey("path") || !TypeForwardedToPathUtility.IsRelativePath(nonMdHref))
{
return(nonMdHref);
}
var currentFilePath = (string)context.Variables["path"];
var currentFolderPath = Path.GetDirectoryName(currentFilePath);
var nonMdExpectedPath = Path.Combine(currentFolderPath, nonMdHref);
if (!File.Exists(nonMdExpectedPath))
{
Logger.LogWarning($"Can't find resource reference: {nonMdHref}. Raw: {rawMarkdown}.", null, currentFilePath, line);
}
return(nonMdHref);
}
public override FileModel Load(FileAndType file, ImmutableDictionary <string, object> metadata)
{
switch (file.Type)
{
case DocumentType.Article:
var filePath = Path.Combine(file.BaseDir, file.File);
var swaggerContent = File.ReadAllText(filePath);
var swagger = SwaggerJsonParser.Parse(swaggerContent);
swagger.Metadata[DocumentTypeKey] = RestApiDocumentType;
swagger.Raw = swaggerContent;
CheckOperationId(swagger, file.File);
var repoInfo = GitUtility.GetGitDetail(filePath);
if (repoInfo != null)
{
swagger.Metadata["source"] = new SourceDetail()
{
Remote = repoInfo
};
}
swagger.Metadata = MergeMetadata(swagger.Metadata, metadata);
var vm = SwaggerModelConverter.FromSwaggerModel(swagger);
var displayLocalPath = TypeForwardedToPathUtility.MakeRelativePath(EnvironmentContext.BaseDirectory, file.FullPath);
return(new FileModel(file, vm, serializer: Environment.Is64BitProcess?null: new BinaryFormatter())
{
Uids = new[] { new UidDefinition(vm.Uid, displayLocalPath) }
.Concat(from item in vm.Children select new UidDefinition(item.Uid, displayLocalPath))
.Concat(from tag in vm.Tags select new UidDefinition(tag.Uid, displayLocalPath)).ToImmutableArray(),
LocalPathFromRepoRoot = repoInfo?.RelativePath ?? TypeForwardedToStringExtension.ToDisplayPath(filePath),
LocalPathFromRoot = displayLocalPath
});
case DocumentType.Overwrite:
// TODO: Refactor current behavior that overwrite file is read multiple times by multiple processors
return(OverwriteDocumentReader.Read(file));
default:
throw new NotSupportedException();
}
}
public override FileModel Load(FileAndType file, ImmutableDictionary <string, object> metadata)
{
switch (file.Type)
{
case DocumentType.Article:
var page = YamlUtility.Deserialize <PageViewModel>(Path.Combine(file.BaseDir, file.File));
if (page.Items == null || page.Items.Count == 0)
{
return(null);
}
if (page.Metadata == null)
{
page.Metadata = metadata.ToDictionary(p => p.Key, p => p.Value);
}
else
{
foreach (var item in metadata)
{
if (!page.Metadata.ContainsKey(item.Key))
{
page.Metadata[item.Key] = item.Value;
}
}
}
var displayLocalPath = TypeForwardedToPathUtility.MakeRelativePath(EnvironmentContext.BaseDirectory, file.FullPath);
return(new FileModel(file, page, serializer: Environment.Is64BitProcess?null: new BinaryFormatter())
{
Uids = (from item in page.Items select new UidDefinition(item.Uid, displayLocalPath)).ToImmutableArray(),
LocalPathFromRepoRoot = displayLocalPath,
LocalPathFromRoot = displayLocalPath
});
case DocumentType.Overwrite:
// TODO: Refactor current behavior that overwrite file is read multiple times by multiple processors
return(OverwriteDocumentReader.Read(file));
default:
throw new NotSupportedException();
}
}
public override FileModel Load(FileAndType file, ImmutableDictionary <string, object> metadata)
{
var filePath = file.FullPath;
var tocViewModel = Utility.LoadSingleToc(filePath);
var toc = new TocItemViewModel
{
Items = tocViewModel
};
var repoDetail = GitUtility.GetGitDetail(filePath);
var displayLocalPath = TypeForwardedToPathUtility.MakeRelativePath(EnvironmentContext.BaseDirectory, file.FullPath);
// todo : metadata.
return(new FileModel(file, toc)
{
Uids = new[] { new UidDefinition(file.File, displayLocalPath) }.ToImmutableArray(),
LocalPathFromRepoRoot = repoDetail?.RelativePath ?? filePath,
LocalPathFromRoot = displayLocalPath
});
}
public static SourceDetail GetSourceDetail(ISymbol symbol)
{
// For namespace, definition is meaningless
if (symbol == null || symbol.Kind == SymbolKind.Namespace)
{
return(null);
}
var syntaxRef = symbol.DeclaringSyntaxReferences.LastOrDefault();
if (symbol.IsExtern || syntaxRef == null)
{
return(new SourceDetail
{
IsExternalPath = true,
Path = symbol.ContainingAssembly?.Name,
});
}
var syntaxNode = syntaxRef.GetSyntax();
Debug.Assert(syntaxNode != null);
if (syntaxNode != null)
{
var source = new SourceDetail
{
StartLine = syntaxNode.SyntaxTree.GetLineSpan(syntaxNode.Span).StartLinePosition.Line,
Path = syntaxNode.SyntaxTree.FilePath,
Name = symbol.Name
};
source.Remote = GitUtility.GetGitDetail(source.Path);
if (source.Remote != null)
{
source.Path = TypeForwardedToPathUtility.FormatPath(source.Path, UriKind.Relative, source.Remote.LocalWorkingDirectory);
}
return(source);
}
return(null);
}
public override FileModel Load(FileAndType file, ImmutableDictionary <string, object> metadata)
{
string uid = null;
Dictionary <string, object> content = null;
var metafile = Path.Combine(file.BaseDir, file.File.TrimEnd('.') + ".meta");
if (File.Exists(metafile))
{
content = YamlUtility.Deserialize <Dictionary <string, object> >(metafile);
if (content != null)
{
foreach (var item in metadata)
{
if (!content.ContainsKey(item.Key))
{
content[item.Key] = item.Value;
}
if (item.Key == Constants.PropertyName.Uid)
{
uid = item.Value as string;
}
}
}
}
if (content == null)
{
content = metadata.ToDictionary(p => p.Key, p => p.Value);
}
var filePath = Path.Combine(file.BaseDir, file.File);
var repoDetail = GitUtility.GetGitDetail(filePath);
var displayLocalPath = TypeForwardedToPathUtility.MakeRelativePath(EnvironmentContext.BaseDirectory, file.FullPath);
return(new FileModel(file, content)
{
Uids = string.IsNullOrEmpty(uid) ? ImmutableArray <UidDefinition> .Empty : ImmutableArray <UidDefinition> .Empty.Add(new UidDefinition(uid, displayLocalPath)),
LocalPathFromRepoRoot = repoDetail?.RelativePath ?? TypeForwardedToStringExtension.ToDisplayPath(Path.Combine(file.BaseDir, file.File)),
LocalPathFromRoot = displayLocalPath
});
}
public override void UpdateHref(FileModel model, IDocumentBuildContext context)
{
var toc = (TocItemViewModel)model.Content;
var key = model.Key;
// Add current folder to the toc mapping, e.g. `a/` maps to `a/toc`
var directory = ((TypeForwardedToRelativePath)key).GetPathFromWorkingFolder().GetDirectoryPath();
context.RegisterToc(key, directory);
UpdateTocItemHref(toc, model, context);
var tocInfo = new TocInfo(key);
if (toc.Homepage != null)
{
if (TypeForwardedToPathUtility.IsRelativePath(toc.Homepage))
{
var pathToRoot = ((TypeForwardedToRelativePath)model.File + (TypeForwardedToRelativePath)toc.Homepage).GetPathFromWorkingFolder();
tocInfo.Homepage = pathToRoot;
}
}
context.RegisterTocInfo(tocInfo);
}
private string GenerateAzureLinkHref(IMarkdownContext context, string href, string rawMarkdown, string line)
{
if (string.IsNullOrEmpty(href))
{
return(string.Empty);
}
StringBuffer content = StringBuffer.Empty;
// If the context doesn't have necessary info, return the original href
if (!context.Variables.ContainsKey("path") || !context.Variables.ContainsKey("azureMarkdownFileInfoMapping"))
{
return(href);
}
// if the href is not relative path, return it. Add try catch to keep this method safe.
try
{
if (!TypeForwardedToPathUtility.IsRelativePath(href))
{
return(href);
}
}
catch (ArgumentException)
{
Logger.LogWarning($"Invalid reference {href} in file: {(string)context.Variables["path"]}. Raw: {rawMarkdown}", null, (string)context.Variables["path"], line);
return(href);
}
// deal with bookmark. Get file name and anchor
string hrefFileName = string.Empty;
string anchor = string.Empty;
var index = href.IndexOf('#');
if (index == -1)
{
hrefFileName = Path.GetFileName(href);
}
else if (index == 0)
{
return(href);
}
else
{
hrefFileName = Path.GetFileName(href.Remove(index));
anchor = href.Substring(index);
}
// deal with different kinds of relative paths
var currentFilePath = (string)context.Variables["path"];
var azureMarkdownFileInfoMapping = (IReadOnlyDictionary <string, AzureFileInfo>)context.Variables["azureMarkdownFileInfoMapping"];
if (azureMarkdownFileInfoMapping == null || !azureMarkdownFileInfoMapping.ContainsKey(hrefFileName))
{
Logger.LogWarning($"Can't find markdown reference: {href}. Raw: {rawMarkdown}.", null, currentFilePath, line);
return(href);
}
string azureHref = null;
var hrefFileInfo = azureMarkdownFileInfoMapping[hrefFileName];
azureHref = string.Format("{0}{1}", TypeForwardedToPathUtility.MakeRelativePath(Path.GetDirectoryName(currentFilePath), hrefFileInfo.FilePath), anchor);
return(azureHref);
}
private string GenerateAzureLinkHref(IMarkdownContext context, string href, string rawMarkdown)
{
StringBuffer content = StringBuffer.Empty;
// If the context doesn't have necessary info, return the original href
if (!context.Variables.ContainsKey("path") || !context.Variables.ContainsKey("azureMarkdownFileInfoMapping"))
{
return(href);
}
// if the href is not relative path, return it
if (!TypeForwardedToPathUtility.IsRelativePath(href))
{
return(href);
}
// deal with bookmark. Get file name and anchor
string hrefFileName = string.Empty;
string anchor = string.Empty;
var index = href.IndexOf('#');
if (index == -1)
{
hrefFileName = Path.GetFileName(href);
}
else if (index == 0)
{
return(href);
}
else
{
hrefFileName = Path.GetFileName(href.Remove(index));
anchor = href.Substring(index);
}
// deal with different kinds of relative paths
var currentFilePath = (string)context.Variables["path"];
var azureMarkdownFileInfoMapping = (IReadOnlyDictionary <string, AzureFileInfo>)context.Variables["azureMarkdownFileInfoMapping"];
if (azureMarkdownFileInfoMapping == null || !azureMarkdownFileInfoMapping.ContainsKey(hrefFileName))
{
Logger.LogWarning($"Can't fild reference file: {href} in azure file system for file {currentFilePath}. Raw: {rawMarkdown}");
return(href);
}
string azureHref = null;
var hrefFileInfo = azureMarkdownFileInfoMapping[hrefFileName];
// Not in docsets and transform to azure external link
if (hrefFileInfo.NeedTransformToAzureExternalLink)
{
azureHref = $"{hrefFileInfo.UriPrefix}/{Path.GetFileNameWithoutExtension(hrefFileName)}{anchor}";
}
else
{
var hrefPath = hrefFileInfo.FilePath;
// It is correct for Azure strucuture. Azure articles are all under same folder
var isHrefInsameDocset = TypeForwardedToPathUtility.IsPathUnderSpecificFolder(hrefPath, Path.GetDirectoryName(currentFilePath));
// In same docset with current file, use relative path. Otherwise, use docset link prefix
if (isHrefInsameDocset)
{
azureHref = string.Format("{0}{1}", TypeForwardedToPathUtility.MakeRelativePath(Path.GetDirectoryName(currentFilePath), hrefFileInfo.FilePath), anchor);
}
else
{
// If the file is in different docset, then append the absolute path prefix. .html should be remove as docs also don't need it now.
azureHref = $"{hrefFileInfo.UriPrefix}/{Path.GetFileNameWithoutExtension(hrefFileName)}{anchor}";
}
}
return(azureHref);
}
private string FixNonMdRelativeFileHref(string nonMdHref, IMarkdownContext context, string rawMarkdown)
{
// If the context doesn't have necessary info or nonMdHref is not a relative path, return the original href
if (!context.Variables.ContainsKey("path") || !TypeForwardedToPathUtility.IsRelativePath(nonMdHref))
{
return(nonMdHref);
}
var currentFilePath = (string)context.Variables["path"];
var currentFolderPath = Path.GetDirectoryName(currentFilePath);
try
{
// if the relative path (not from azure resource file info mapping) is under docset. Just return it.
var nonMdHrefFullPath = Path.GetFullPath(Path.Combine(currentFolderPath, nonMdHref));
if (TypeForwardedToPathUtility.IsPathUnderSpecificFolder(nonMdHrefFullPath, currentFolderPath))
{
return(nonMdHref);
}
else
{
Logger.LogVerbose($"Relative path:{nonMdHref} is not under {currentFolderPath} of file {currentFilePath}. Use ex_resource to replace the link.");
}
// if azure resource file info doesn't exist, log warning and return
if (!context.Variables.ContainsKey("azureResourceFileInfoMapping"))
{
Logger.LogWarning($"Can't find azure resource file info mapping. Couldn't fix href: {nonMdHref} in file {currentFilePath}. raw: {rawMarkdown}");
return(nonMdHref);
}
var nonMdHrefFileName = Path.GetFileName(nonMdHref);
var azureResourceFileInfoMapping = (Dictionary <string, AzureFileInfo>)context.Variables["azureResourceFileInfoMapping"];
AzureFileInfo azureResourceFileInfo;
if (!azureResourceFileInfoMapping.TryGetValue(nonMdHrefFileName, out azureResourceFileInfo))
{
Logger.LogWarning($"Can't find info for file name {nonMdHrefFileName} in azure resource file info mapping. Couldn't fix href: {nonMdHref} in file {currentFilePath}. raw: {rawMarkdown}");
return(nonMdHref);
}
// If the nonMdHref is under same docset with current file. No need to fix that.
if (TypeForwardedToPathUtility.IsPathUnderSpecificFolder(azureResourceFileInfo.FilePath, currentFolderPath))
{
return(nonMdHref);
}
// If the nonMdHref is under different docset with current file but not exists. Then log warning and won't fix.
if (!File.Exists(azureResourceFileInfo.FilePath))
{
Logger.LogWarning($"{nonMdHref} refer by {currentFilePath} doesn't exists. Won't do link fix. raw: {rawMarkdown}");
return(nonMdHref);
}
// If the nonMdHref is under different docset with current file and also exists, then fix the link.
// 1. copy the external file to ex_resource folder. 2. Return new href path to the file under external folder
var exResourceDir = Directory.CreateDirectory(Path.Combine(currentFolderPath, ExternalResourceFolderName));
var resDestPath = Path.Combine(exResourceDir.FullName, Path.GetFileName(azureResourceFileInfo.FilePath));
File.Copy(azureResourceFileInfo.FilePath, resDestPath, true);
return(TypeForwardedToPathUtility.MakeRelativePath(currentFolderPath, resDestPath));
}
catch (NotSupportedException nse)
{
Logger.LogWarning($"Warning: FixNonMdRelativeFileHref can't be apply on reference: {nonMdHref}. Exception: {nse.Message}");
return(nonMdHref);
}
}
public override void Handle(HtmlDocument document, ManifestItem manifestItem, string inputFile, string outputFile)
{
_fileMapping[outputFile] = inputFile;
// RFC 3986: relative-ref = relative-part [ "?" query ] [ "#" fragment ]
_linksWithBookmark[outputFile] =
(from node in GetNodesWithAttribute(document, "href")
let link = node.GetAttributeValue("href", null)
let bookmarkIndex = link.IndexOf("#")
where bookmarkIndex != -1
let bookmark = link.Substring(bookmarkIndex + 1)
let index = link.IndexOfAny(new[] { '?', '#' })
let decodedLink = HttpUtility.UrlDecode(link.Remove(index))
where !WhiteList.Contains(bookmark) && TypeForwardedToPathUtility.IsRelativePath(decodedLink)
select new LinkItem
{
Title = node.InnerText,
Href = TransformPath(outputFile, decodedLink),
Bookmark = bookmark,
SourceFragment = WebUtility.HtmlDecode(node.GetAttributeValue("data-raw-source", null)),
SourceFile = WebUtility.HtmlDecode(node.GetAttributeValue("sourceFile", null)),
SourceLineNumber = node.GetAttributeValue("sourceStartLineNumber", 0),
TargetLineNumber = node.Line
}).ToList();
var anchors = GetNodeAttribute(document, "id").Concat(GetNodeAttribute(document, "name"));
_registeredBookmarks[outputFile] = new HashSet <string>(anchors);
}
public Manifest Process(Manifest manifest, string outputFolder)
{
if (outputFolder == null)
{
throw new ArgumentNullException("Base directory can not be null");
}
var indexData = new Dictionary <string, SearchIndexItem>();
var indexDataFilePath = Path.Combine(outputFolder, IndexFileName);
var htmlFiles = (from item in manifest.Files ?? Enumerable.Empty <ManifestItem>()
from output in item.OutputFiles
where output.Key.Equals(".html", StringComparison.OrdinalIgnoreCase)
select output.Value.RelativePath).ToList();
if (htmlFiles.Count == 0)
{
return(manifest);
}
Logger.LogInfo($"Extracting index data from {htmlFiles.Count} html files");
foreach (var relativePath in htmlFiles)
{
var filePath = Path.Combine(outputFolder, relativePath);
var html = new HtmlDocument();
Logger.LogVerbose($"Extracting index data from {filePath}");
if (File.Exists(filePath))
{
try
{
html.Load(filePath, Encoding.UTF8);
}
catch (Exception ex)
{
Logger.LogWarning($"Warning: Can't load content from {filePath}: {ex.Message}");
continue;
}
var indexItem = ExtractItem(html, relativePath);
if (indexItem != null)
{
indexData[relativePath] = indexItem;
}
}
}
JsonUtility.Serialize(indexDataFilePath, indexData, Formatting.Indented);
// add index.json to mainfest as resource file
var manifestItem = new ManifestItem
{
DocumentType = "Resource",
Metadata = new Dictionary <string, object>(),
OutputFiles = new Dictionary <string, OutputFileInfo>()
};
manifestItem.OutputFiles.Add("resource", new OutputFileInfo
{
RelativePath = TypeForwardedToPathUtility.MakeRelativePath(outputFolder, indexDataFilePath),
});
manifest.Files?.Add(manifestItem);
return(manifest);
}
