/// <summary>
/// Apply documentation from an external provider. This is optional.
/// </summary>
/// <param name="cache">The cache of doc items.</param>
/// <param name="group">The module to document.</param>
private Task <CppModule> ApplyDocumentation(DocItemCache cache, CppModule group)
{
IDocProvider docProvider = null;
// Try to load doc provider from an external assembly
if (DocProviderAssemblyPath != null)
{
try
{
var assembly = Assembly.LoadFrom(DocProviderAssemblyPath);
foreach (var type in assembly.GetTypes())
{
if (typeof(IDocProvider).GetTypeInfo().IsAssignableFrom(type))
{
docProvider = (IDocProvider)Activator.CreateInstance(type);
break;
}
}
}
catch (Exception)
{
Logger.Warning(null, "Warning, Unable to locate/load DocProvider Assembly.");
Logger.Warning(null, "Warning, DocProvider was not found from assembly [{0}]", DocProviderAssemblyPath);
}
}
if (docProvider == null)
{
return(Task.FromResult(group));
}
Logger.Progress(20, "Applying C++ documentation");
return(docProvider.ApplyDocumentation(cache, group));
}
private static async Task DocumentInterface(this IDocProvider docProvider, DocItemCache cache, CppInterface cppInterface)
{
await Task.WhenAll(
cppInterface.Methods
.Select(func => docProvider.DocumentCallable(cache, func, name: cppInterface.Name + "::" + func.Name))
.Concat(new[] { docProvider.DocumentElement(cache, cppInterface, documentInnerElements: false) }));
}
private static async Task <DocItem> DocumentElement(
this IDocProvider docProvider,
DocItemCache cache,
CppElement element,
bool documentInnerElements = true,
string name = null)
{
var docName = name ?? element.Name;
DocItem cacheEntry = cache.Find(docName);
var docItem = cacheEntry ?? await docProvider.FindDocumentationAsync(docName);
element.Id = docItem.ShortId;
element.Description = docItem.Summary;
element.Remarks = docItem.Remarks;
if (cacheEntry == null)
{
docItem.Name = docName;
cache.Add(docItem);
}
if (element.IsEmpty)
{
return(docItem);
}
if (documentInnerElements)
{
DocumentInnerElements(element.Items, docItem);
}
return(docItem);
}
public override bool Execute()
{
BindingRedirectResolution.Enable();
var docProvider = new MsdnProvider(message => Log.LogMessage(message));
var module = CppModel.CppModule.Read(CppModule.ItemSpec);
var cache = new DocItemCache();
var cachePath = ShadowCopy ? Path.GetTempFileName() : DocumentationCache.ItemSpec;
if (File.Exists(DocumentationCache.ItemSpec))
{
if (ShadowCopy)
{
File.Copy(DocumentationCache.ItemSpec, cachePath, true);
}
cache = DocItemCache.Read(cachePath);
}
var documented = docProvider.ApplyDocumentation(cache, module).Result;
cache.Write(cachePath);
if (ShadowCopy)
{
File.Copy(cachePath, DocumentationCache.ItemSpec, true);
File.Delete(cachePath);
}
documented.Write(CppModule.ItemSpec);
return(true);
}
private void UpdateCallback(CacheEntryUpdateArguments arguments)
{
var expired = arguments.RemovedReason == CacheEntryRemovedReason.Expired;
if (!expired)
{
logger.TraceError(
$"Temp cache: Can't refresh cache since remove reason is {arguments.RemovedReason}. Key:{arguments.Key}");
return;
}
var oldItem = (DocItemCache)memCache?[arguments.Key];
if (oldItem == null)
{
// If memcache was disposed don't error out
if (memCache != null)
{
logger.TraceError(
$"Temp cache: Can't refresh cache since item is not in the cache. Key:{arguments.Key}");
}
return;
}
var hits = oldItem.Hits;
var lowerHitsThreshold = LatestCacheConfiguration.LowerHitsThreshold;
if (hits < lowerHitsThreshold)
{
logger.TraceInfo(
$"Temp cache: Will not refresh cache item with {hits} hits. LowerHitsThreshold: {lowerHitsThreshold}. Key: {arguments.Key}");
return;
}
logger.TraceInfo(
$"Temp cache: Refreshing cache. Hits: {hits} higher than {lowerHitsThreshold}. Key: {arguments.Key}");
var reader = storeReader;
try
{
var oldKey = oldItem.Item.Key;
DocItem freshDocItem = reader.ReadFromStore(oldKey).GetAwaiter().GetResult();
freshDocItem = freshDocItem ?? EmptyDocItem.Create(oldKey);
var docItemCache = new DocItemCache(freshDocItem);
var cacheItem = new CacheItem(oldKey.ToString(), docItemCache);
arguments.UpdatedCacheItem = cacheItem;
arguments.UpdatedCacheItemPolicy = GetPolicy();
logger.TraceInfo(
$"Temp cache: Cache refreshed! Key: {arguments.Key}");
}
catch (Exception e)
{
logger.TraceError(
$"Temp cache: Failed to refresh cache. Key:{arguments.Key}. Error: {e}");
}
}
public static async Task ApplyDocumentation(IDocProvider?docProvider, DocItemCache cache, CsAssembly module,
DocumentationContext context)
{
var documentationTasks = new List <Task>();
Task DocumentSelector(CsBase cppElement) =>
DocumentElement(docProvider, cache, cppElement, context, true, null);
foreach (var cppInclude in module.Namespaces)
{
documentationTasks.AddRange(cppInclude.Enums.Select(DocumentSelector));
documentationTasks.AddRange(cppInclude.Structs.Select(DocumentSelector));
documentationTasks.AddRange(
cppInclude.Interfaces
.Select(cppInterface => DocumentInterface(docProvider, cache, cppInterface, context))
);
documentationTasks.AddRange(
cppInclude.Classes
.Select(cppFunction => DocumentGroup(docProvider, cache, cppFunction, context))
);
}
await Task.WhenAll(documentationTasks);
}
private static async Task <IDocItem?> DocumentElement(IDocProvider?docProvider,
DocItemCache cache,
CsBase element,
DocumentationContext context,
bool documentInnerElements,
string?name)
{
var docName = name ?? element.CppElementName;
if (string.IsNullOrEmpty(docName))
{
return(null);
}
docName = docName.Trim();
if (string.IsNullOrEmpty(docName))
{
return(null);
}
var cacheEntry = cache.Find(docName);
var docItem = cacheEntry ?? await QueryDocumentationProvider();
if (docItem == null)
{
return(null);
}
element.DocId = docItem.ShortId;
element.Description = docItem.Summary;
element.Remarks = docItem.Remarks;
docItem.Names.Add(docName);
if (cacheEntry == null)
{
cache.Add(docItem);
}
if (element.Items.Count == 0)
{
return(docItem);
}
if (documentInnerElements)
{
DocumentInnerElements(element.Items, docItem);
}
return(docItem);
async Task <IDocItem?> QueryDocumentationProvider()
{
if (docProvider == null)
{
return(null);
}
Lazy <string> docProviderName = new(
() =>
{
try
{
var friendlyName = docProvider.UserFriendlyName;
return(string.IsNullOrWhiteSpace(friendlyName) ? FullName() : friendlyName);
}
catch
{
return(FullName());
}
string FullName()
{
var type = docProvider.GetType();
var name = type.FullName;
return(string.IsNullOrEmpty(name) ? type.Name : name !);
}
},
LazyThreadSafetyMode.None
);
List <Exception> exceptions = new(3);
var(backoff, backoffIndex, nextDelay) = GenerateBackoff(TimeSpan.Zero);
try
{
for (uint retry = 0; retry <= 5; retry++)
{
if (retry != 0)
{
TimeSpan delay;
if (nextDelay.HasValue)
{
delay = nextDelay.Value;
}
else
{
// TODO: fix the bug and remove this hack
if (backoffIndex >= 5)
{
context.Logger.Message(
$"SharpGen internal invalid state on delay: backoffIndex == {backoffIndex}"
);
if (Debugger.IsAttached)
{
Debugger.Break();
}
backoffIndex = 0;
}
delay = backoff[backoffIndex++];
}
if (delay > TimeSpan.Zero)
{
await Task.Delay(delay);
}
nextDelay = null;
}
try
{
var result = await docProvider.FindDocumentationAsync(docName, context);
switch (result)
{
case null:
throw new ArgumentNullException(
nameof(result),
$"Unexpected null {nameof(IFindDocumentationResult)}"
);
case FindDocumentationResultFailure resultFailure:
{
var retryDelay = resultFailure.RetryDelay;
if (retryDelay == TimeSpan.MaxValue)
{
return(null);
}
if (retryDelay <= TimeSpan.Zero)
{
nextDelay = TimeSpan.Zero;
}
// TODO: fix the bug and remove this hack
if (backoffIndex >= 5)
{
context.Logger.Message(
$"SharpGen internal invalid state on reschedule: backoffIndex = {backoffIndex}"
);
if (Debugger.IsAttached)
{
Debugger.Break();
}
(backoff, backoffIndex, nextDelay) = GenerateBackoff(retryDelay);
}
nextDelay = backoff[backoffIndex++];
if (nextDelay < retryDelay)
{
(backoff, backoffIndex, nextDelay) = GenerateBackoff(retryDelay);
}
break;
}
case FindDocumentationResultSuccess resultSuccess:
return(resultSuccess.Item); // TODO: check if the item is empty (therefore, useless)
default:
throw new ArgumentOutOfRangeException(
nameof(result),
$"Unexpected {nameof(IFindDocumentationResult)}: {result.GetType().FullName}"
);
}
}
catch (Exception e)
{
e.Data["SDK:" + nameof(docProvider)] = docProvider;
e.Data["SDK:" + nameof(docName)] = docName;
e.Data["SDK:" + nameof(context)] = context;
e.Data["SDK:" + nameof(retry)] = retry;
e.Data["SDK:" + nameof(backoffIndex)] = backoffIndex;
e.Data["SDK:" + nameof(exceptions) + ".Count"] = exceptions.Count;
exceptions.Add(e);
// We should retry less when it's due to unhandled exception.
// So in exception case we step twice in retry count on each iteration.
retry++;
}
}
context.Logger.Message($"{docProviderName.Value} extension failed to find documentation for \"{docName}\"");
return(null);
}
finally
{
if (exceptions.Count > 0)
{
var failure = new DocumentationQueryFailure(docName)
{
Exceptions = exceptions,
FailedProviderName = docProviderName.Value,
TreatProviderFailuresAsErrors = docProvider.TreatFailuresAsErrors
};
context.Failures.Add(failure);
}
}
}
}
private static async Task DocumentCallable(this IDocProvider docProvider, DocItemCache cache, CppCallable callable, string name = null)
{
var docItem = await docProvider.DocumentElement(cache, callable, name : name);
callable.ReturnValue.Description = docItem.Return;
}
public static async Task <CppModule> ApplyDocumentation(this IDocProvider docProvider, DocItemCache cache, CppModule module)
{
var documentationTasks = new List <Task>();
foreach (CppInclude cppInclude in module.Includes)
{
documentationTasks.AddRange(cppInclude.Enums.Select(cppEnum => docProvider.DocumentElement(cache, cppEnum)));
documentationTasks.AddRange(cppInclude.Structs.Select(cppStruct => docProvider.DocumentElement(cache, cppStruct)));
documentationTasks.AddRange(cppInclude.Interfaces.Select(cppInterface => docProvider.DocumentInterface(cache, cppInterface)));
documentationTasks.AddRange(cppInclude.Functions.Select(cppFunction => docProvider.DocumentCallable(cache, cppFunction)));
}
await Task.WhenAll(documentationTasks);
return(module);
}
public void Set(DocItemKey key, DocItemCache docItemCache, CacheItemPolicy policy)
{
var cacheItem = new CacheItem(key.ToString(), docItemCache);
memCache?.Set(cacheItem, policy);
}
private bool Execute(ConfigFile config)
{
config.GetFilesWithIncludesAndExtensionHeaders(
out var configsWithHeaders,
out var configsWithExtensionHeaders
);
var cppHeaderGenerator = new CppHeaderGenerator(SharpGenLogger, OutputPath);
var cppHeaderGenerationResult = cppHeaderGenerator.GenerateCppHeaders(config, configsWithHeaders, configsWithExtensionHeaders);
if (SharpGenLogger.HasErrors)
{
return(false);
}
var resolver = new IncludeDirectoryResolver(SharpGenLogger);
resolver.Configure(config);
var castXml = new CastXmlRunner(SharpGenLogger, resolver, CastXmlExecutable.ItemSpec, CastXmlArguments)
{
OutputPath = OutputPath
};
var macroManager = new MacroManager(castXml);
var cppExtensionGenerator = new CppExtensionHeaderGenerator();
var module = config.CreateSkeletonModule();
macroManager.Parse(Path.Combine(OutputPath, config.HeaderFileName), module);
cppExtensionGenerator.GenerateExtensionHeaders(
config, OutputPath, module, configsWithExtensionHeaders, cppHeaderGenerationResult.UpdatedConfigs
);
GenerateInputsCache(
macroManager.IncludedFiles
.Concat(config.ConfigFilesLoaded.Select(x => x.AbsoluteFilePath))
.Concat(configsWithExtensionHeaders.Select(x => Path.Combine(OutputPath, x.ExtensionFileName)))
.Select(s => Utilities.FixFilePath(s, Utilities.EmptyFilePathBehavior.Ignore))
.Where(x => x != null)
.Distinct()
);
if (SharpGenLogger.HasErrors)
{
return(false);
}
// Run the parser
var parser = new CppParser(SharpGenLogger, config)
{
OutputPath = OutputPath
};
if (SharpGenLogger.HasErrors)
{
return(false);
}
CppModule group;
using (var xmlReader = castXml.Process(parser.RootConfigHeaderFileName))
{
// Run the C++ parser
group = parser.Run(module, xmlReader);
}
if (SharpGenLogger.HasErrors)
{
return(false);
}
config.ExpandDynamicVariables(SharpGenLogger, group);
var docLinker = new DocumentationLinker();
var typeRegistry = new TypeRegistry(SharpGenLogger, docLinker);
var namingRules = new NamingRulesManager();
var globalNamespace = new GlobalNamespaceProvider();
foreach (var nameOverride in GlobalNamespaceOverrides)
{
var wellKnownName = nameOverride.ItemSpec;
var overridenName = nameOverride.GetMetadata("Override");
if (string.IsNullOrEmpty(overridenName))
{
continue;
}
if (Enum.TryParse(wellKnownName, out WellKnownName name))
{
globalNamespace.OverrideName(name, overridenName);
}
else
{
SharpGenLogger.Warning(
LoggingCodes.InvalidGlobalNamespaceOverride,
"Invalid override of \"{0}\": unknown class name, ignoring the override.",
wellKnownName
);
}
}
// Run the main mapping process
var transformer = new TransformManager(
globalNamespace,
namingRules,
SharpGenLogger,
typeRegistry,
docLinker,
new ConstantManager(namingRules, docLinker)
);
var(solution, defines) = transformer.Transform(group, config);
var consumerConfig = new ConfigFile
{
Id = ConsumerBindMappingConfigId,
IncludeProlog = { cppHeaderGenerationResult.Prologue },
Extension = new List <ExtensionBaseRule>(defines)
};
var(bindings, generatedDefines) = transformer.GenerateTypeBindingsForConsumers();
consumerConfig.Bindings.AddRange(bindings);
consumerConfig.Extension.AddRange(generatedDefines);
consumerConfig.Mappings.AddRange(
docLinker.GetAllDocLinks().Select(
link => new MappingRule
{
DocItem = link.cppName,
MappingNameFinal = link.cSharpName
}
)
);
GenerateConfigForConsumers(consumerConfig);
if (SharpGenLogger.HasErrors)
{
return(false);
}
var documentationCacheItemSpec = DocumentationCache.ItemSpec;
Utilities.RequireAbsolutePath(documentationCacheItemSpec, nameof(DocumentationCache));
var cache = File.Exists(documentationCacheItemSpec)
? DocItemCache.Read(documentationCacheItemSpec)
: new DocItemCache();
DocumentationLogger docLogger = new(SharpGenLogger) { MaxLevel = LogLevel.Warning };
var docContext = new Lazy <DocumentationContext>(() => new DocumentationContext(docLogger));
ExtensibilityDriver.Instance.DocumentModule(SharpGenLogger, cache, solution, docContext).Wait();
if (docContext.IsValueCreated)
{
Regex[] silencePatterns = null;
var docLogLevelDefault = DocumentationFailuresAsErrors ? LogLevel.Error : LogLevel.Warning;
foreach (var queryFailure in docContext.Value.Failures)
{
if (silencePatterns == null)
{
silencePatterns = new Regex[SilenceMissingDocumentationErrorIdentifierPatterns.Length];
for (var i = 0; i < silencePatterns.Length; i++)
{
silencePatterns[i] = new Regex(
SilenceMissingDocumentationErrorIdentifierPatterns[i].ItemSpec,
RegexOptions.CultureInvariant
);
}
}
if (silencePatterns.Length != 0)
{
bool SilencePredicate(Regex x) => x.Match(queryFailure.Query).Success;
if (silencePatterns.Any(SilencePredicate))
{
continue;
}
}
var providerName = queryFailure.FailedProviderName ?? "<null>";
var docLogLevel = queryFailure.TreatProviderFailuresAsErrors
? docLogLevelDefault
: docLogLevelDefault > LogLevel.Warning
? LogLevel.Warning
: docLogLevelDefault;
if (queryFailure.Exceptions == null || queryFailure.Exceptions.Count <= 1)
{
SharpGenLogger.LogRawMessage(
docLogLevel,
LoggingCodes.DocumentationProviderInternalError,
"Documentation provider [{0}] query for \"{1}\" failed.",
queryFailure.Exceptions?.FirstOrDefault(),
providerName,
queryFailure.Query
);
}
else
{
var exceptionsCount = queryFailure.Exceptions.Count;
for (var index = 0; index < exceptionsCount; index++)
{
var exception = queryFailure.Exceptions[index];
SharpGenLogger.LogRawMessage(
docLogLevel,
LoggingCodes.DocumentationProviderInternalError,
"Documentation provider [{0}] query for \"{1}\" failed ({2}/{3}).",
exception,
providerName,
queryFailure.Query,
index + 1,
exceptionsCount
);
}
}
}
}
cache.WriteIfDirty(documentationCacheItemSpec);
if (SharpGenLogger.HasErrors)
{
return(false);
}
var documentationFiles = new Dictionary <string, XmlDocument>();
foreach (var file in ExternalDocumentation)
{
using var stream = File.OpenRead(file.ItemSpec);
var xml = new XmlDocument();
xml.Load(stream);
documentationFiles.Add(file.ItemSpec, xml);
}
PlatformDetectionType platformMask = 0;
foreach (var platform in Platforms)
{
if (!Enum.TryParse <PlatformDetectionType>(platform.ItemSpec, out var parsedPlatform))
{
SharpGenLogger.Warning(
LoggingCodes.InvalidPlatformDetectionType,
"The platform type {0} is an unknown platform to SharpGenTools. Falling back to Any platform detection.",
platform
);
platformMask = PlatformDetectionType.Any;
}
else
{
platformMask |= parsedPlatform;
}
}
if (platformMask == 0)
{
platformMask = PlatformDetectionType.Any;
}
if (SharpGenLogger.HasErrors)
{
return(false);
}
var generator = new RoslynGenerator(
SharpGenLogger,
globalNamespace,
docLinker,
new ExternalDocCommentsReader(documentationFiles),
new GeneratorConfig
{
Platforms = platformMask
}
);
generator.Run(solution, GeneratedCodeFolder);
return(!SharpGenLogger.HasErrors);
}
