private void PatchYamlFile(string yamlFile, List <UidAndComment> list)
{
var path = Path.Combine(_yamlDirectory, yamlFile);
if (!File.Exists(path))
{
Console.WriteLine($"Cannot find file {yamlFile}.");
return;
}
var pageVM = YamlUtility.Deserialize <PageViewModel>(path);
foreach (var uidAndComment in list)
{
var item = pageVM.Items.Find(x => x.Uid == uidAndComment.Uid);
if (item == null)
{
Console.WriteLine($"Cannot find {uidAndComment.Uid} in file {yamlFile}.");
continue;
}
PatchViewModel(item, uidAndComment.Comment);
}
Console.WriteLine($"Patching yaml: {yamlFile}");
YamlUtility.Serialize(path, pageVM, YamlMime.ManagedReference);
}
private static T ResolveContent <T>(T item, Func <T, T> itemBuilder) where T : IOverwriteDocumentViewModel
{
if (itemBuilder != null)
{
item = itemBuilder(item);
}
using (var sw = new StringWriter())
{
YamlUtility.Serialize(sw, item);
using (var sr = new StringReader(sw.ToString()))
{
var serializer = new YamlDeserializer(ignoreUnmatched: true);
var placeholderValueDeserializer = new PlaceholderValueDeserializer(serializer.ValueDeserializer, item.Conceptual);
item = serializer.Deserialize <T>(sr, placeholderValueDeserializer);
if (placeholderValueDeserializer.ContainPlaceholder)
{
item.Conceptual = null;
}
}
}
return(item);
}
/// <summary>
/// Export xref map file.
/// </summary>
private static string ExportXRefMap(DocumentBuildParameters parameters, DocumentBuildContext context)
{
Logger.LogVerbose("Exporting xref map...");
var xrefMap = new XRefMap
{
Tags = context.GroupInfo?.XRefTags ?? context.XRefTags,
};
xrefMap.References =
(from xref in context.XRefSpecMap.Values.AsParallel().WithDegreeOfParallelism(parameters.MaxParallelism)
select new XRefSpec(xref)
{
Href = context.UpdateHref(xref.Href, RelativePath.WorkingFolder)
}).ToList();
xrefMap.Sort();
string xrefMapFileNameWithVersion = GetXrefMapFileNameWithGroup(parameters);
YamlUtility.Serialize(
Path.GetFullPath(Environment.ExpandEnvironmentVariables(Path.Combine(parameters.OutputBaseDir, xrefMapFileNameWithVersion))),
xrefMap,
YamlMime.XRefMap);
Logger.LogInfo("XRef map exported.");
return(xrefMapFileNameWithVersion);
}
private static void MergePageViewModel(List <FileAndType> files, MetadataMergeParameters parameters, string outputBase)
{
var p = new ManagedReferenceDocumentProcessor();
var host = new HostService(
parameters.Files.DefaultBaseDir,
from f in files
where f.Type == DocumentType.Article && !"toc.yml".Equals(Path.GetFileName(f.File), StringComparison.OrdinalIgnoreCase)
select Load(p, parameters.Metadata, parameters.FileMetadata, f));
// todo : temp reuse plugin
var pv = new ApplyPlatformVersion();
pv.Prebuild(host.Models, host);
var core = new MergeManagedReferenceDocument();
host.Reload(core.Prebuild(host.Models, host));
foreach (var m in host.Models)
{
m.File = m.PathRewriter(m.File);
}
foreach (var m in host.Models)
{
YamlUtility.Serialize(Path.Combine(outputBase, m.File), m.Content);
}
}
private void RebuildReference()
{
var references = GetReferences();
foreach (var model in GetViewModels())
{
bool dirty = false;
// Add references for exceptions
var types = from i in model.Item2.Items
where i.Exceptions != null
from e in i.Exceptions
select e.Type;
HashSet <string> set = new HashSet <string>(model.Item2.References.Select(r => r.Uid));
foreach (var type in types)
{
if (set.Add(type))
{
if (!references.TryGetValue(type, out ReferenceViewModel reference))
{
reference = references[type] = new ReferenceViewModel {
Uid = type
};
}
model.Item2.References.Add(reference);
dirty = true;
}
}
if (dirty)
{
Console.WriteLine($"Rebuilding references: {model.Item1}");
YamlUtility.Serialize(model.Item1, model.Item2, YamlMime.ManagedReference);
}
}
}
public void TestClassWithInvalidExtensibleMember()
{
Assert.Throws <YamlException>(() => YamlUtility.Serialize(new StringWriter(), new ClassWithInvalidExtensibleMember {
}));
Assert.Throws <YamlException>(() => YamlUtility.Deserialize <ClassWithInvalidExtensibleMember>(new StringReader("A: a")));
}
public void TestInternalClass()
{
Assert.Throws <YamlException>(() => YamlUtility.Serialize(new StringWriter(), new InternalClass {
}));
Assert.Throws <YamlException>(() => YamlUtility.Deserialize <InternalClass>(new StringReader("A: a")));
}
public static void Run(
string xmlDirectory,
string outputDirectory,
string fallbackXmlDirectory = null,
string fallbackOutputDirectory = null,
Action <LogItem> logWriter = null,
string logContentBaseDirectory = null,
string sourceMapFilePath = null,
string publicGitRepoUrl = null,
string publicGitBranch = null,
ECMA2YamlRepoConfig config = null)
{
if (xmlDirectory == null)
{
throw new ArgumentNullException(xmlDirectory);
}
if (!Directory.Exists(xmlDirectory))
{
throw new DirectoryNotFoundException($"{nameof(xmlDirectory)} {xmlDirectory} does not exist.");
}
if (outputDirectory == null)
{
throw new ArgumentNullException(outputDirectory);
}
if (!string.IsNullOrEmpty(fallbackXmlDirectory))
{
if (!Directory.Exists(fallbackXmlDirectory))
{
throw new DirectoryNotFoundException($"{nameof(fallbackXmlDirectory)} {fallbackXmlDirectory} does not exist.");
}
if (string.IsNullOrEmpty(fallbackOutputDirectory))
{
throw new ArgumentNullException(fallbackOutputDirectory,
$"{nameof(fallbackOutputDirectory)} cannot be empty if {nameof(fallbackXmlDirectory)} is present.");
}
}
if (!string.IsNullOrEmpty(logContentBaseDirectory))
{
OPSLogger.PathTrimPrefix = logContentBaseDirectory.NormalizePath().AppendDirectorySeparator();
}
if (!string.IsNullOrEmpty(fallbackXmlDirectory))
{
OPSLogger.FallbackPathTrimPrefix = fallbackXmlDirectory.NormalizePath().AppendDirectorySeparator();
}
if (logWriter != null)
{
OPSLogger.WriteLogCallback = logWriter;
}
var fileAccessor = new FileAccessor(xmlDirectory, fallbackXmlDirectory);
ECMALoader loader = new ECMALoader(fileAccessor);
Console.WriteLine("Loading ECMAXML files...");
var store = loader.LoadFolder("");
if (store == null)
{
return;
}
Console.WriteLine("Building loaded files...");
Console.WriteLine($"ECMA2YamlRepoConfig:{JsonConvert.SerializeObject(config)}");
store.UWPMode = config?.UWP ?? false;
store.Build();
if (!string.IsNullOrEmpty(publicGitRepoUrl) && !string.IsNullOrEmpty(publicGitBranch))
{
store.TranlateContentSourceMeta(publicGitRepoUrl, publicGitBranch);
}
else
{
Console.WriteLine("Not enough information, unable to generate git url related metadata. -publicRepo {0}, -publicBranch {1}", publicGitRepoUrl, publicGitBranch);
}
var xmlYamlFileMapping = SDPYamlGenerator.Generate(store, outputDirectory, flatten: config?.Flatten ?? true, withVersioning: true);
if (loader.FallbackFiles != null && loader.FallbackFiles.Any() && !string.IsNullOrEmpty(fallbackOutputDirectory))
{
if (!Directory.Exists(fallbackOutputDirectory))
{
Directory.CreateDirectory(fallbackOutputDirectory);
}
foreach (var fallbackFile in loader.FallbackFiles)
{
if (xmlYamlFileMapping.TryGetValue(fallbackFile, out var originalYamls))
{
foreach (var originalYaml in originalYamls)
{
var newYaml = originalYaml.Replace(outputDirectory, fallbackOutputDirectory);
File.Copy(originalYaml, newYaml, overwrite: true);
File.Delete(originalYaml);
}
xmlYamlFileMapping.Remove(fallbackFile);
}
}
}
if (!string.IsNullOrEmpty(sourceMapFilePath))
{
WriteYamlXMLFileMap(sourceMapFilePath, xmlYamlFileMapping);
}
var toc = SDPTOCGenerator.Generate(store);
var tocOutputDirectory = string.IsNullOrEmpty(config?.BatchId) ? outputDirectory : Path.Combine(outputDirectory, config.BatchId);
if (!Directory.Exists(tocOutputDirectory))
{
Directory.CreateDirectory(tocOutputDirectory);
}
YamlUtility.Serialize(Path.Combine(tocOutputDirectory, "toc.yml"), toc, "YamlMime:TableOfContent");
}
public static void SaveToc(string tocPath, TocViewModel model)
{
YamlUtility.Serialize(tocPath, model);
}
private static async Task <Tuple <MetadataItem, bool> > GetMetadataFromProjectLevelCacheAsync <T>(
T input,
IEnumerable <string> inputKey,
Func <IncrementalCheck, Task <bool> > rebuildChecker,
Func <T, Task <Compilation> > compilationProvider,
Func <T, IDictionary <string, List <string> > > containedFilesProvider,
string outputFolder,
bool preserveRawInlineComments,
bool shouldSkipMarkup,
string filterConfigFile,
IReadOnlyDictionary <Compilation, IEnumerable <IMethodSymbol> > extensionMethods)
{
DateTime triggeredTime = DateTime.UtcNow;
var projectLevelCache = ProjectLevelCache.Get(inputKey);
var projectConfig = projectLevelCache.GetValidConfig(inputKey);
var rebuildProject = true;
if (projectConfig != null)
{
var projectCheck = new IncrementalCheck(projectConfig);
rebuildProject = await rebuildChecker(projectCheck);
}
MetadataItem projectMetadata;
if (!rebuildProject)
{
// Load from cache
var cacheFile = Path.Combine(projectConfig.OutputFolder, projectConfig.RelatvieOutputFiles.First());
Logger.Log(LogLevel.Info, $"'{projectConfig.InputFilesKey}' keep up-to-date since '{projectConfig.TriggeredUtcTime.ToString()}', cached intermediate result '{cacheFile}' is used.");
if (TryParseYamlMetadataFile(cacheFile, out projectMetadata))
{
return(Tuple.Create(projectMetadata, rebuildProject));
}
else
{
Logger.Log(LogLevel.Info, $"'{projectConfig.InputFilesKey}' is invalid, rebuild needed.");
}
}
var compilation = await compilationProvider(input);
projectMetadata = GenerateYamlMetadata(compilation, preserveRawInlineComments, filterConfigFile, extensionMethods);
var file = Path.GetRandomFileName();
var cacheOutputFolder = projectLevelCache.OutputFolder;
var path = Path.Combine(cacheOutputFolder, file);
YamlUtility.Serialize(path, projectMetadata);
Logger.Log(LogLevel.Verbose, $"Successfully generated metadata {cacheOutputFolder} for {projectMetadata.Name}");
IDictionary <string, List <string> > containedFiles = null;
if (containedFilesProvider != null)
{
containedFiles = containedFilesProvider(input);
}
// Save to cache
projectLevelCache.SaveToCache(inputKey, containedFiles, triggeredTime, cacheOutputFolder, new List <string>()
{
file
}, shouldSkipMarkup);
return(Tuple.Create(projectMetadata, rebuildProject));
}
static void LoadAndConvert(CommandLineOptions opt)
{
var rootPath = Path.GetFullPath(opt.RepoRootPath ?? opt.SourceFolder);
var xmlFolder = Path.GetFullPath(opt.SourceFolder).Replace(rootPath, "").Trim(Path.DirectorySeparatorChar);
var fileAccessor = new FileAccessor(rootPath, opt.FallbackRepoRoot);
ECMALoader loader = new ECMALoader(fileAccessor);
WriteLine("Loading ECMAXML files...");
var store = loader.LoadFolder(xmlFolder);
if (store == null)
{
return;
}
store.StrictMode = opt.StrictMode;
store.UWPMode = opt.UWPMode;
store.DemoMode = opt.DemoMode;
WriteLine("Building loaded files...");
store.Build();
if (OPSLogger.ErrorLogged)
{
return;
}
if (!string.IsNullOrEmpty(opt.RepoRootPath) &&
!string.IsNullOrEmpty(opt.PublicRepoUrl) &&
!string.IsNullOrEmpty(opt.PublicRepoBranch) &&
!string.IsNullOrEmpty(opt.RepoUrl) &&
!string.IsNullOrEmpty(opt.RepoBranch))
{
store.TranslateSourceLocation(opt.RepoRootPath, opt.RepoUrl, opt.RepoBranch, opt.PublicRepoUrl, opt.PublicRepoBranch);
}
else
{
WriteLine("Not enough information, unable to generate git url related metadata. -repoRoot {0}, -publicRepo {1}, -publicBranch {2}",
opt.RepoRootPath, opt.PublicRepoUrl, opt.PublicRepoBranch);
}
WriteLine("Loaded {0} namespaces.", store.Namespaces.Count);
WriteLine("Loaded {0} types.", store.TypesByFullName.Count);
WriteLine("Loaded {0} members.", store.MembersByUid.Count);
WriteLine("Loaded {0} extension methods.", store.ExtensionMethodsByMemberDocId?.Values?.Count ?? 0);
WriteLine("Loaded {0} attribute filters.", store.FilterStore?.AttributeFilters?.Count ?? 0);
if (!string.IsNullOrEmpty(opt.UndocumentedApiReport))
{
UndocumentedApi.ReportGenerator.GenerateReport(store, opt.UndocumentedApiReport.NormalizePath(), opt.RepoBranch);
}
IDictionary <string, List <string> > xmlYamlFileMapping = null;
if (opt.SDPMode)
{
xmlYamlFileMapping = SDPYamlGenerator.Generate(store, opt.OutputFolder, opt.Flatten, opt.Versioning);
YamlUtility.Serialize(Path.Combine(opt.OutputFolder, "toc.yml"), SDPTOCGenerator.Generate(store), "YamlMime:TableOfContent");
}
//Translate change list
if (opt.ChangeListFiles.Count > 0)
{
foreach (var changeList in opt.ChangeListFiles)
{
if (File.Exists(changeList))
{
var count = ChangeListUpdater.TranslateChangeList(changeList, xmlYamlFileMapping);
WriteLine("Translated {0} file entries in {1}.", count, changeList);
}
}
}
WriteYamlXMLFileMap(opt.YamlXMLMappingFile, opt.RepoRootPath, xmlYamlFileMapping);
//Save fallback file list as skip publish
if (!string.IsNullOrEmpty(opt.SkipPublishFilePath) && loader.FallbackFiles?.Count > 0)
{
List <string> list = new List <string>();
if (File.Exists(opt.SkipPublishFilePath))
{
var jsonContent = File.ReadAllText(opt.SkipPublishFilePath);
list = JsonConvert.DeserializeObject <List <string> >(jsonContent);
WriteLine("Read {0} entries in {1}.", list.Count, opt.SkipPublishFilePath);
}
list.AddRange(loader.FallbackFiles
.Where(path => xmlYamlFileMapping.ContainsKey(path))
.SelectMany(path => xmlYamlFileMapping[path].Select(p => p.Replace(opt.RepoRootPath, "").TrimStart('\\')))
);
var jsonText = JsonConvert.SerializeObject(list, Formatting.Indented);
File.WriteAllText(opt.SkipPublishFilePath, jsonText);
WriteLine("Write {0} entries to {1}.", list.Count, opt.SkipPublishFilePath);
}
WriteLine("Done writing Yaml files.");
}
public void TestClassWithInterfaceMember()
{
var sw = new StringWriter();
YamlUtility.Serialize(sw, new ClassWithInterfaceMember
{
List = new List <string> {
"a"
},
ReadOnlyList = new[] { "b" },
Collection = new Collection <string> {
"c"
},
ReadOnlyCollection = ImmutableList.Create("d"),
Enumerable = Enumerable.Range(1, 1),
Dictionary = new Dictionary <string, string> {
["k1"] = "v1"
},
ReadOnlyDictionary = new SortedDictionary <string, string> {
["k2"] = "v2"
},
Set = new SortedSet <string> {
"s"
},
});
Assert.Equal(@"List:
- a
ReadOnlyList:
- b
Collection:
- c
ReadOnlyCollection:
- d
Enumerable:
- 1
Dictionary:
k1: v1
ReadOnlyDictionary:
k2: v2
Set:
- s
".Replace("\r\n", "\n"), sw.ToString().Replace("\r\n", "\n"));
var obj = YamlUtility.Deserialize <ClassWithInterfaceMember>(new StringReader(sw.ToString()));
Assert.NotNull(obj);
Assert.Single(obj.List);
Assert.Equal("a", obj.List[0]);
Assert.Single(obj.ReadOnlyList);
Assert.Equal("b", obj.ReadOnlyList[0]);
Assert.Single(obj.Collection);
Assert.Equal("c", obj.Collection.First());
Assert.Single(obj.ReadOnlyCollection);
Assert.Equal("d", obj.ReadOnlyCollection.First());
Assert.Single(obj.Enumerable);
Assert.Equal(1, obj.Enumerable.First());
Assert.Single(obj.Dictionary);
Assert.Equal(new KeyValuePair <string, string>("k1", "v1"), obj.Dictionary.First());
Assert.Single(obj.ReadOnlyDictionary);
Assert.Equal(new KeyValuePair <string, string>("k2", "v2"), obj.ReadOnlyDictionary.First());
Assert.Single(obj.Set);
Assert.Equal("s", obj.Set.First());
}
private static void SplitToc(string originalTocPath)
{
if (!Path.IsPathRooted(originalTocPath))
{
originalTocPath = Path.GetFullPath(originalTocPath);
}
if (!File.Exists(originalTocPath))
{
throw new FileNotFoundException($"The path of toc file: {originalTocPath} can't be found");
}
var originalTocDir = Path.GetDirectoryName(originalTocPath);
if (originalTocDir == null)
{
throw new Exception($"The directory of {originalTocPath} can't be null");
}
TocViewModel tocModel;
using (var stream = File.OpenRead(originalTocPath))
using (var reader = new StreamReader(stream))
{
try
{
tocModel = YamlUtility.Deserialize <TocViewModel>(reader);
}
catch (YamlException ex)
{
Console.WriteLine($"Error occurs while parsing toc {originalTocPath}, please check if the format is correct. {ex.Message}");
throw;
}
}
if (tocModel == null)
{
Console.WriteLine($"No toc model parsed for {originalTocPath}.");
return;
}
var mergedTocModel = new TocViewModel();
foreach (var ns in tocModel)
{
var splittedTocPath = Path.Combine(originalTocDir, "_splitted", ns.Uid, "toc.yml");
var splittedTocRelativePath = PathUtility.MakeRelativePath(originalTocDir, splittedTocPath);
var reversedRelativePath = PathUtility.MakeRelativePath(Path.GetDirectoryName(splittedTocPath), originalTocDir);
ProcessHref(ns, reversedRelativePath.Replace('\\', '/'));
var splittedTocDir = Path.GetDirectoryName(splittedTocPath);
if (splittedTocDir == null)
{
throw new Exception($"The directory of {splittedTocPath} can't be null");
}
if (!Directory.Exists(splittedTocDir))
{
Directory.CreateDirectory(splittedTocDir);
}
var splittedTocItem = new TocItemViewModel
{
Uid = ns.Uid,
Name = ns.Name,
Items = ns.Items
};
if (ns.Metadata != null)
{
splittedTocItem.Metadata = ns.Metadata;
}
var splittedTocModel = new TocViewModel(new List <TocItemViewModel> {
splittedTocItem
});
YamlUtility.Serialize(splittedTocPath, splittedTocModel);
Console.WriteLine($"Create new splitted toc ({splittedTocPath})");
var mergedTocItem = new TocItemViewModel
{
Uid = ns.Uid,
Name = ns.Name,
Href = Path.GetDirectoryName(splittedTocRelativePath).Replace('\\', '/') + "/"
};
mergedTocModel.Add(mergedTocItem);
}
YamlUtility.Serialize(originalTocPath, mergedTocModel);
Console.WriteLine($"Rewrite original toc file ({originalTocPath})");
}
private static void ProcessFilePair(
string ymlInputFile,
string ymlOutputFile,
string mdFile,
Dictionary <string, DocumentSchema> schemas,
bool initMode = false)
{
var yamlStream = new YamlStream();
using (var sr = new StreamReader(ymlInputFile))
{
yamlStream.Load(sr);
}
if (yamlStream.Documents.Count != 1)
{
throw new NotSupportedException("Does not support mutiple YAML documents");
}
var mime = YamlMime.ReadMime(ymlInputFile);
if (string.IsNullOrEmpty(mime))
{
Console.WriteLine("Cannot find MIME in {0}", ymlInputFile);
return;
}
var schemaName = mime.Substring(YamlMime.YamlMimePrefix.Length);
if (!schemas.ContainsKey(schemaName))
{
Console.WriteLine("Schema {0} not found", schemaName);
return;
}
var schema = schemas[schemaName];
var mdFragments = initMode ? new Dictionary <string, MarkdownFragment>() : FragmentModelHelper.LoadMarkdownFragment(mdFile);
_iterator.Traverse(yamlStream.Documents[0].RootNode, mdFragments, schema);
foreach (var fragment in mdFragments.Values)
{
fragment.Properties = fragment.Properties?.Where(pair => pair.Value.Touched)?.ToDictionary(pair => pair.Key, pair => pair.Value);
}
var validFragments = mdFragments.Values.Where(v => v.Properties?.Count > 0 && v.Touched).ToList();
if (validFragments.Count > 0)
{
StringBuilder sb = new StringBuilder();
foreach (var fragment in validFragments)
{
sb.AppendLine(fragment.ToString());
}
var mdFolder = Path.GetDirectoryName(mdFile);
if (!Directory.Exists(mdFolder))
{
Directory.CreateDirectory(mdFolder);
}
File.WriteAllText(mdFile, sb.ToString());
var ymlFolder = Path.GetDirectoryName(ymlOutputFile);
if (!Directory.Exists(ymlFolder))
{
Directory.CreateDirectory(ymlFolder);
}
YamlUtility.Serialize(ymlOutputFile, yamlStream.Documents[0].RootNode, mime);
}
}
