public override Manifest PostHandle(Manifest manifest)
{
foreach (var pair in _linksWithBookmark)
{
string currentFile = pair.Key;
foreach (var linkItem in pair.Value)
{
string title = linkItem.Title;
string linkedToFile = linkItem.Href == string.Empty ? currentFile : linkItem.Href;
string bookmark = linkItem.Bookmark;
HashSet<string> bookmarks;
if (_registeredBookmarks.TryGetValue(linkedToFile, out bookmarks) && !bookmarks.Contains(bookmark))
{
string currentFileSrc = linkItem.SourceFile ?? _fileMapping[currentFile];
string linkedToFileSrc = _fileMapping[linkedToFile];
string link = linkItem.Href == string.Empty ? $"#{bookmark}" : $"{linkedToFileSrc}#{bookmark}";
string content = linkItem.SourceFragment;
if (string.IsNullOrEmpty(content))
{
// Invalid bookmarks introduced from templates is a corner case, ignored.
content = $"<a href=\"{link}\">{title}</a>";
}
Logger.LogWarning($"Illegal link: `{content}` -- missing bookmark. The file {linkedToFileSrc} doesn't contain a bookmark named {bookmark}.",
file: currentFileSrc,
line: linkItem.SourceLineNumber != 0 ? linkItem.SourceLineNumber.ToString() : null);
}
}
}
return manifest;
}
public override Manifest PreHandle(Manifest manifest)
{
_registeredBookmarks = new Dictionary<string, HashSet<string>>(TypeForwardedToFilePathComparer.OSPlatformSensitiveStringComparer);
_linksWithBookmark = new Dictionary<string, List<LinkItem>>(TypeForwardedToFilePathComparer.OSPlatformSensitiveStringComparer);
_fileMapping = new Dictionary<string, string>(TypeForwardedToFilePathComparer.OSPlatformSensitiveStringComparer);
return manifest;
}
public Manifest Process(Manifest manifest, string outputFolder)
{
if (outputFolder == null)
{
throw new ArgumentNullException("Base directory can not be null");
}
var indexData = new Dictionary<string, SearchIndexItem>();
var indexDataFilePath = Path.Combine(outputFolder, IndexFileName);
var htmlFiles = (from item in manifest.Files ?? Enumerable.Empty<ManifestItem>()
from output in item.OutputFiles
where output.Key.Equals(".html", StringComparison.OrdinalIgnoreCase)
select output.Value.RelativePath).ToList();
if (htmlFiles.Count == 0)
{
return manifest;
}
Logger.LogInfo($"Extracting index data from {htmlFiles.Count} html files");
foreach (var relativePath in htmlFiles)
{
var filePath = Path.Combine(outputFolder, relativePath);
var html = new HtmlDocument();
Logger.LogVerbose($"Extracting index data from {filePath}");
if (File.Exists(filePath))
{
try
{
html.Load(filePath, Encoding.UTF8);
}
catch (Exception ex)
{
Logger.LogWarning($"Warning: Can't load content from {filePath}: {ex.Message}");
continue;
}
var indexItem = ExtractItem(html, relativePath);
if (indexItem != null)
{
indexData[relativePath] = indexItem;
}
}
}
JsonUtility.Serialize(indexDataFilePath, indexData, Formatting.Indented);
// add index.json to mainfest as resource file
var manifestItem = new ManifestItem
{
DocumentType = "Resource",
Metadata = new Dictionary<string, object>(),
OutputFiles = new Dictionary<string, OutputFileInfo>()
};
manifestItem.OutputFiles.Add("resource", new OutputFileInfo
{
RelativePath = TypeForwardedToPathUtility.MakeRelativePath(outputFolder, indexDataFilePath),
});
manifest.Files?.Add(manifestItem);
return manifest;
}
public Manifest Process(Manifest manifest, string outputFolder)
{
if (manifest == null)
{
throw new ArgumentNullException(nameof(manifest));
}
if (outputFolder == null)
{
throw new ArgumentNullException("Base directory can not be null");
}
foreach (var handler in Handlers)
{
manifest = handler.PreHandleWithScopeWrapper(manifest);
}
foreach (var tuple in from item in manifest.Files ?? Enumerable.Empty<ManifestItem>()
from output in item.OutputFiles
where output.Key.Equals(".html", StringComparison.OrdinalIgnoreCase)
select new
{
Item = item,
InputFile = item.SourceRelativePath,
OutputFile = output.Value.RelativePath,
})
{
var filePath = Path.Combine(outputFolder, tuple.OutputFile);
if (!File.Exists(filePath))
{
continue;
}
var document = new HtmlDocument();
try
{
document.Load(filePath, Encoding.UTF8);
}
catch (Exception ex)
{
Logger.LogWarning($"Warning: Can't load content from {filePath}: {ex.Message}");
continue;
}
foreach (var handler in Handlers)
{
handler.HandleWithScopeWrapper(document, tuple.Item, tuple.InputFile, tuple.OutputFile);
}
document.Save(filePath, Encoding.UTF8);
}
foreach (var handler in Handlers)
{
manifest = handler.PostHandleWithScopeWrapper(manifest);
}
return manifest;
}
public void TestBasicFeature()
{
var outputFolder = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "validate_bookmark");
Directory.CreateDirectory(outputFolder);
Manifest manifest = new Manifest
{
SourceBasePath = outputFolder,
Files = new List<ManifestItem>
{
new ManifestItem { SourceRelativePath = "a.md", OutputFiles = new Dictionary<string, OutputFileInfo> { { ".html", new OutputFileInfo { RelativePath = "a.html" } } } },
new ManifestItem { SourceRelativePath = "b.md", OutputFiles = new Dictionary<string, OutputFileInfo> { { ".html", new OutputFileInfo { RelativePath = "b.html" } } } },
new ManifestItem { SourceRelativePath = "c.md", OutputFiles = new Dictionary<string, OutputFileInfo> { { ".html", new OutputFileInfo { RelativePath = "c.html" } } } },
}
};
File.WriteAllText(Path.Combine(outputFolder, "a.html"), @"<a href='http://bing.com#top'>Microsoft Bing</a> <p id='b1'>section</p><a href='#b1'/>");
File.WriteAllText(Path.Combine(outputFolder, "b.html"), @"<a href='a.html#b1' sourceFile='b.md' sourceStartLineNumber='1'>bookmark existed</a><a href='a.html#b2' data-raw-source='[link with source info](a.md#b2)' sourceFile='b.md' sourceStartLineNumber='1'>link with source info</a> <a href='a.html#b3' data-raw-source='[link in token file](a.md#b3)' sourceFile='token.md' sourceStartLineNumber='1'>link in token file</a><a href='a.html#b4'>link without source info</a>");
File.WriteAllText(Path.Combine(outputFolder, "c.html"), @"<a href='illegal_path_%3Cillegal character%3E.html#b1'>Test illegal link path</a>");
Logger.RegisterListener(_listener);
using (new LoggerPhaseScope("validate_bookmark"))
{
new HtmlPostProcessor
{
Handlers = { new ValidateBookmark() }
}.Process(manifest, outputFolder);
}
Logger.UnregisterListener(_listener);
var logs = _listener.Items;
Console.WriteLine(string.Concat(logs.Select(l => Tuple.Create(l.Message, l.File))));
Assert.Equal(3, logs.Count);
var expected = new[]
{
Tuple.Create(@"Illegal link: `[link with source info](a.md#b2)` -- missing bookmark. The file a.md doesn't contain a bookmark named b2.", "b.md"),
Tuple.Create(@"Illegal link: `[link in token file](a.md#b3)` -- missing bookmark. The file a.md doesn't contain a bookmark named b3.", "token.md"),
Tuple.Create(@"Illegal link: `<a href=""a.md#b4"">link without source info</a>` -- missing bookmark. The file a.md doesn't contain a bookmark named b4.", "b.md"),
};
var actual = logs.Select(l => Tuple.Create(l.Message, l.File)).ToList();
Assert.True(!expected.Except(actual).Any() && expected.Length == actual.Count);
}
public void TestBasicFeature()
{
var outputFolder = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "RemoveDebugInfo");
Directory.CreateDirectory(outputFolder);
Manifest manifest = new Manifest
{
SourceBasePath = outputFolder,
Files = new List<ManifestItem>
{
new ManifestItem { SourceRelativePath = "a.md", OutputFiles = new Dictionary<string, OutputFileInfo> { { ".html", new OutputFileInfo { RelativePath = "a.html" } } } },
}
};
File.WriteAllText(Path.Combine(outputFolder, "a.html"), @"<p id='b1' sourceFile='a.md' sourceStartLineNumber='1' sourceEndLineNumber='2'>section<a sourcefile=""a.md"" href='http://bing.com#top'>Microsoft Bing</a></p>");
new HtmlPostProcessor
{
Handlers = { new RemoveDebugInfo() }
}.Process(manifest, outputFolder);
var actual = File.ReadAllText(Path.Combine(outputFolder, "a.html"));
Assert.Equal("<p id='b1'>section<a href='http://bing.com#top'>Microsoft Bing</a></p>", actual);
Directory.Delete(outputFolder, true);
}
private static void SaveManifest(Manifest manifest, string outputDirectory)
{
var manifestJsonPath = Path.Combine(outputDirectory ?? string.Empty, Constants.ManifestFileName);
JsonUtility.Serialize(manifestJsonPath, manifest);
Logger.LogInfo($"Manifest file saved to {manifestJsonPath}.");
}
private void PostProcess(Manifest manifest, string outputDir)
{
// post process
foreach (var postProcessor in _postProcessors)
{
using (new LoggerPhaseScope($"Process in post processor {postProcessor.ContractName}", false))
using (new PerformanceScope($"Process in post processor {postProcessor.ContractName}", LogLevel.Verbose))
{
manifest = postProcessor.Processor.Process(manifest, outputDir);
if (manifest == null)
{
throw new DocfxException($"Plugin {postProcessor.ContractName} should not return null manifest");
}
// To make sure post processor won't generate duplicate output files
RemoveDuplicateOutputFiles(manifest.Files);
}
}
}
public void TestIndexDotJsonWithNonEnglishCharacters()
{
var rawHtml = @"
<!DOCTYPE html>
<html>
<head>
<meta charset=""utf-8"">
<title>This is title in head metadata</title>
</head>
<body>
<h1> This is Title </h1>
<p class='data-searchable'> Hello World,
Microsoft
</p>
<article>
<h1>
This is article title
</h1>
docfx can do anything...
and it supports non-english characters like these: ãâáà êé í õôó Типы шрифтов 人物 文字
</article>
</body>
</html>
";
// prepares temp folder and file for testing purposes
// ExtractSearchIndex should probably be refactored so we can test it without depending on the filesystem
var tempTestFolder = "temp_test_folder";
if (Directory.Exists(tempTestFolder)) Directory.Delete(tempTestFolder, true);
Directory.CreateDirectory(tempTestFolder);
File.WriteAllText(Path.Combine(tempTestFolder, "index.html"), rawHtml, new UTF8Encoding(false));
// prepares fake manifest object
var outputFileInfo = new OutputFileInfo();
outputFileInfo.RelativePath = "index.html";
var manifestItem = new ManifestItem() { OutputFiles = new Dictionary<string, OutputFileInfo>() };
manifestItem.OutputFiles.Add(".html", outputFileInfo);
var manifest = new Manifest() { Files = new List<ManifestItem>() };
manifest.Files.Add(manifestItem);
// process the fake manifest, using tempTestFolder as the output folder
_extractor.Process(manifest, tempTestFolder);
var expectedIndexJSON = @"{
""index.html"": {
""href"": ""index.html"",
""title"": ""This is title in head metadata"",
""keywords"": ""Hello World, Microsoft This is article title docfx can do anything... and it supports non-english characters like these: ãâáà êé í õôó Типы шрифтов 人物 文字""
}
}";
var actualIndexJSON = File.ReadAllText(Path.Combine(tempTestFolder, "index.json"), Encoding.UTF8);
Assert.Equal(expectedIndexJSON, actualIndexJSON);
}
public override Manifest PostHandle(Manifest manifest)
{
return manifest;
}