/// <summary>
/// Sets up the reference to token iterator and context
/// </summary>
/// <param name="tk"></param>
/// <param name="ctx"></param>
public static void Setup(TokenIterator tk, Context ctx, string scriptName)
{
_tokenIt = tk;
_ctx = ctx;
_scriptName = scriptName;
_withStack = new Stack<string>();
}
/* Function: CalculateColumns
* Fills in <columnIndexes> for the passed parameter. If the parameter doesn't exist it will return false.
*/
protected bool CalculateColumns(int parameterIndex)
{
TokenIterator startParam, endParam;
if (parsedPrototype.GetParameter(parameterIndex, out startParam, out endParam) == false)
{
return(false);
}
if (columnIndexes == null)
{
columnIndexes = new int[NumberOfColumns];
}
TokenIterator iterator = startParam;
iterator.NextPastWhitespace(endParam);
PrototypeParsingType type = iterator.PrototypeParsingType;
if (parsedPrototype.Style == ParsedPrototype.ParameterStyle.C)
{
// ModifierQualifier
int currentColumn = 0;
columnIndexes[currentColumn] = iterator.TokenIndex;
// Null covers whitespace and any random symbols we encountered that went unmarked.
while (iterator < endParam &&
(type == PrototypeParsingType.TypeModifier ||
type == PrototypeParsingType.TypeQualifier ||
type == PrototypeParsingType.Null))
{
iterator.Next();
type = iterator.PrototypeParsingType;
}
// Type
currentColumn++;
columnIndexes[currentColumn] = iterator.TokenIndex;
int typeNesting = 0;
while (iterator < endParam &&
(type == PrototypeParsingType.Type ||
type == PrototypeParsingType.TypeSuffix ||
type == PrototypeParsingType.OpeningTypeSuffix ||
type == PrototypeParsingType.ClosingTypeSuffix ||
type == PrototypeParsingType.Null ||
typeNesting > 0))
{
if (type == PrototypeParsingType.OpeningTypeSuffix)
{
typeNesting++;
}
else if (type == PrototypeParsingType.ClosingTypeSuffix)
{
typeNesting--;
}
iterator.Next();
type = iterator.PrototypeParsingType;
}
// NamePrefix
currentColumn++;
columnIndexes[currentColumn] = iterator.TokenIndex;
while (iterator < endParam &&
type == PrototypeParsingType.NamePrefix_PartOfType)
{
iterator.Next();
type = iterator.PrototypeParsingType;
}
// Name
currentColumn++;
columnIndexes[currentColumn] = iterator.TokenIndex;
// Include the parameter separator because there may not be a default value
while (iterator < endParam &&
(type == PrototypeParsingType.Name ||
type == PrototypeParsingType.NameSuffix_PartOfType ||
type == PrototypeParsingType.ParamSeparator ||
type == PrototypeParsingType.Null))
{
iterator.Next();
type = iterator.PrototypeParsingType;
}
// DefaultValueSeparator
currentColumn++;
columnIndexes[currentColumn] = iterator.TokenIndex;
while (iterator < endParam &&
type == PrototypeParsingType.DefaultValueSeparator)
{
iterator.Next();
type = iterator.PrototypeParsingType;
}
// DefaultValue
currentColumn++;
columnIndexes[currentColumn] = iterator.TokenIndex;
}
else if (parsedPrototype.Style == ParsedPrototype.ParameterStyle.Pascal)
{
// Name
int columnSymbolIndex = 0;
columnIndexes[columnSymbolIndex] = iterator.TokenIndex;
// Null covers whitespace and any random symbols we encountered that went unmarked.
// Include the parameter separator because there may not be a type
while (iterator < endParam &&
(type == PrototypeParsingType.Name ||
type == PrototypeParsingType.NamePrefix_PartOfType ||
type == PrototypeParsingType.NameSuffix_PartOfType ||
type == PrototypeParsingType.ParamSeparator ||
type == PrototypeParsingType.Null))
{
iterator.Next();
type = iterator.PrototypeParsingType;
}
// TypeNameSeparator
columnSymbolIndex++;
columnIndexes[columnSymbolIndex] = iterator.TokenIndex;
while (iterator < endParam &&
type == PrototypeParsingType.NameTypeSeparator)
{
iterator.Next();
type = iterator.PrototypeParsingType;
}
// Type
columnSymbolIndex++;
columnIndexes[columnSymbolIndex] = iterator.TokenIndex;
int typeNesting = 0;
// Include the parameter separator because there may not be a default value
while (iterator < endParam &&
(type == PrototypeParsingType.TypeModifier ||
type == PrototypeParsingType.TypeQualifier ||
type == PrototypeParsingType.Type ||
type == PrototypeParsingType.TypeSuffix ||
type == PrototypeParsingType.OpeningTypeSuffix ||
type == PrototypeParsingType.ClosingTypeSuffix ||
type == PrototypeParsingType.ParamSeparator ||
type == PrototypeParsingType.Null ||
typeNesting > 0))
{
if (type == PrototypeParsingType.OpeningTypeSuffix)
{
typeNesting++;
}
else if (type == PrototypeParsingType.ClosingTypeSuffix)
{
typeNesting--;
}
iterator.Next();
type = iterator.PrototypeParsingType;
}
// DefaultValueSeparator
columnSymbolIndex++;
columnIndexes[columnSymbolIndex] = iterator.TokenIndex;
while (iterator < endParam &&
type == PrototypeParsingType.DefaultValueSeparator)
{
iterator.Next();
type = iterator.PrototypeParsingType;
}
// DefaultValue
columnSymbolIndex++;
columnIndexes[columnSymbolIndex] = iterator.TokenIndex;
}
// End of parameter
endOfColumnsIndex = endParam.TokenIndex;
return(true);
}
/// <summary>
/// Initialize the combinator.
/// </summary>
/// <param name="parser">The core parser</param>
/// <param name="tokenIt">The token iterator</param>
public virtual void Init(Parser parser, TokenIterator tokenIt)
{
_parser = parser;
_tokenIt = tokenIt;
}
/* Function: BuildSyntaxHighlightedText
* Formats the text between the two iterators with syntax highlighting. If output is null it will be appended to
* <htmlOutput>.
*/
public void BuildSyntaxHighlightedText(TokenIterator iterator, TokenIterator end, StringBuilder output = null)
{
if (output == null)
{
output = htmlOutput;
}
while (iterator < end)
{
if (iterator.FundamentalType == FundamentalType.LineBreak)
{
output.Append("<br />");
iterator.Next();
}
else
{
TokenIterator startStretch = iterator;
TokenIterator endStretch = iterator;
endStretch.Next();
SyntaxHighlightingType stretchType = startStretch.SyntaxHighlightingType;
for (;;)
{
if (endStretch == end || endStretch.FundamentalType == FundamentalType.LineBreak)
{
break;
}
else if (endStretch.SyntaxHighlightingType == stretchType)
{
endStretch.Next();
}
// We can include unhighlighted whitespace if there's content of the same type beyond it. This prevents
// unnecessary span tags.
else if (stretchType != SyntaxHighlightingType.Null &&
endStretch.SyntaxHighlightingType == SyntaxHighlightingType.Null &&
endStretch.FundamentalType == FundamentalType.Whitespace)
{
TokenIterator lookahead = endStretch;
do
{
lookahead.Next();
}while (lookahead.SyntaxHighlightingType == SyntaxHighlightingType.Null &&
lookahead.FundamentalType == FundamentalType.Whitespace &&
lookahead < end);
if (lookahead < end && lookahead.SyntaxHighlightingType == stretchType)
{
endStretch = lookahead;
endStretch.Next();
}
else
{
break;
}
}
else
{
break;
}
}
switch (stretchType)
{
case SyntaxHighlightingType.Comment:
output.Append("<span class=\"SHComment\">");
break;
case SyntaxHighlightingType.Keyword:
output.Append("<span class=\"SHKeyword\">");
break;
case SyntaxHighlightingType.Number:
output.Append("<span class=\"SHNumber\">");
break;
case SyntaxHighlightingType.String:
output.Append("<span class=\"SHString\">");
break;
case SyntaxHighlightingType.PreprocessingDirective:
output.Append("<span class=\"SHPreprocessingDirective\">");
break;
case SyntaxHighlightingType.Metadata:
output.Append("<span class=\"SHMetadata\">");
break;
case SyntaxHighlightingType.Null:
break;
default:
// Add this just in case there's an unaccounted for type in the future. This prevents the spans from
// being unbalanced until we handle it.
output.Append("<span>");
break;
}
output.EntityEncodeAndAppend(iterator.Tokenizer.TextBetween(startStretch, endStretch));
if (stretchType != SyntaxHighlightingType.Null)
{
output.Append("</span>");
}
iterator = endStretch;
}
}
}
/* Function: GetBounds
* Returns the bounds of the complete section, minus whitespace.
*/
public void GetBounds(out TokenIterator start, out TokenIterator end)
{
start = this.start;
end = this.end;
}
/* Function: BuildFullType
*
* Returns the full type if one is marked by <PrototypeParsingType.Type> tokens, combining all its modifiers and qualifiers into
* one continuous string.
*
* If the type and all its modifiers and qualifiers are continuous in the original <Tokenizer> it will return <TokenIterators> based
* on it. However, if the type and all its modifiers and qualifiers are NOT continuous it will create a separate <Tokenizer> to hold
* a continuous version of it. The returned bounds will be <TokenIterators> based on that rather than on the original <Tokenizer>.
* The new <Tokenizer> will still contain the same <PrototypeParsingTypes> and <SyntaxHighlightingTypes> of the original.
*/
virtual public bool BuildFullType(out TokenIterator fullTypeStart, out TokenIterator fullTypeEnd)
{
// Find the first type token
TokenIterator iterator = start;
bool foundType = false;
while (iterator < end &&
iterator.PrototypeParsingType != PrototypeParsingType.Type &&
iterator.PrototypeParsingType != PrototypeParsingType.TypeModifier &&
iterator.PrototypeParsingType != PrototypeParsingType.TypeQualifier &&
iterator.PrototypeParsingType != PrototypeParsingType.OpeningTypeModifier &&
iterator.PrototypeParsingType != PrototypeParsingType.ParamModifier &&
iterator.PrototypeParsingType != PrototypeParsingType.OpeningParamModifier)
{
iterator.Next();
}
fullTypeStart = iterator;
// Find the rest of the type tokens that follow continuously
while (iterator < end)
{
if (iterator.PrototypeParsingType == PrototypeParsingType.Type)
{
foundType = true;
iterator.Next();
}
else if (iterator.PrototypeParsingType == PrototypeParsingType.TypeModifier ||
iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier ||
iterator.PrototypeParsingType == PrototypeParsingType.ParamModifier)
{
iterator.Next();
}
else if (TryToSkipModifierBlock(ref iterator))
{
}
else if (iterator.FundamentalType == FundamentalType.Whitespace)
{
// We'll allow whitespace if it's followed by another type token
TokenIterator lookahead = iterator;
lookahead.Next();
if (lookahead.PrototypeParsingType == PrototypeParsingType.Type ||
lookahead.PrototypeParsingType == PrototypeParsingType.TypeModifier ||
lookahead.PrototypeParsingType == PrototypeParsingType.TypeQualifier ||
lookahead.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier ||
lookahead.PrototypeParsingType == PrototypeParsingType.ParamModifier ||
lookahead.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier)
{
iterator = lookahead;
}
else
{
break;
}
}
else
{
break;
}
}
fullTypeEnd = iterator;
// See if there are any more type tokens past the continuous part
bool continuous = true;
while (iterator < end)
{
if (iterator.PrototypeParsingType == PrototypeParsingType.Type)
{
foundType = true;
continuous = false;
}
else if (iterator.PrototypeParsingType == PrototypeParsingType.TypeModifier ||
iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier ||
iterator.PrototypeParsingType == PrototypeParsingType.OpeningTypeModifier ||
iterator.PrototypeParsingType == PrototypeParsingType.ParamModifier ||
iterator.PrototypeParsingType == PrototypeParsingType.OpeningParamModifier)
{
continuous = false;
}
// If we already found a type and know it's not continuous we can quit early
if (foundType && !continuous)
{
break;
}
iterator.Next();
}
// If we didn't find a type we're done.
if (!foundType)
{
fullTypeStart = end;
fullTypeEnd = end;
return(false);
}
// If it's continuous, see if the spacing matches what we would have built on our own. We want the result to
// always be consistent.
bool acceptableSpacing = true;
if (continuous)
{
acceptableSpacing = TypeBuilder.HasSimilarSpacing(fullTypeStart, fullTypeEnd);
}
// Return the continuous one if it's acceptable or build a new one if it's not.
if (continuous && acceptableSpacing)
{
// fullTypeStart is already set
// fullTypeEnd is already set
#if DEBUG
// Test that this returns the same thing BuildFullType() would have.
Tokenizer tempTokenizer = BuildFullType();
if (fullTypeStart.TextBetween(fullTypeEnd) != tempTokenizer.RawText)
{
throw new Exception("Continuous and built types don't match: \"" + fullTypeStart.TextBetween(fullTypeEnd) +
"\", \"" + tempTokenizer.RawText + "\"");
}
#endif
return(true);
}
else
{
Tokenizer fullTypeTokenizer = BuildFullType();
#if DEBUG
// Test that the call to BuildFullType() was necessary and not a quirk in our spacing detection logic.
// fullTypeStart and fullTypeEnd are still set to the original tokenizer.
if (continuous && !acceptableSpacing &&
fullTypeStart.TextBetween(fullTypeEnd) == fullTypeTokenizer.RawText)
{
throw new Exception("Built type matches continuous, building was unnecessary: \"" + fullTypeTokenizer.RawText + "\"");
}
#endif
fullTypeStart = fullTypeTokenizer.FirstToken;
fullTypeEnd = fullTypeTokenizer.LastToken;
return(true);
}
}
/* Function: GetSimpleText
* Converts a stretch of text to NDMarkup, ignoring the formatting. Unlike <GetText()> this will not surround the output
* in paragraph tags.
*/
protected void GetSimpleText(ref JavadocIterator iterator, StringBuilder output)
{
while (iterator.IsInBounds)
{
if (iterator.IsOn(JavadocElementType.Text))
{
output.EntityEncodeAndAppend(iterator.String);
iterator.Next();
}
else if (iterator.IsOn(JavadocElementType.EntityChar))
{
output.EntityEncodeAndAppend(iterator.EntityValue);
iterator.Next();
}
else if (iterator.IsOn(JavadocElementType.LineBreak))
{
output.Append('\n');
iterator.Next();
}
else if (iterator.IsOnJavadocTag("code") ||
iterator.IsOnJavadocTag("literal"))
{
output.EntityEncodeAndAppend(iterator.JavadocTagValue);
iterator.Next();
}
else if (iterator.IsOnJavadocTag("link") ||
iterator.IsOnJavadocTag("linkPlain"))
{
Tokenizer linkContent = new Tokenizer(iterator.JavadocTagValue);
TokenIterator linkIterator = linkContent.FirstToken;
string symbol = GetJavadocLinkSymbol(ref linkIterator);
linkIterator.NextPastWhitespace();
string description = GetSimpleText(linkIterator, linkContent.LastToken);
description = Normalize(description);
if (description == null || description == "")
{
output.EntityEncodeAndAppend(symbol);
}
else
{
output.EntityEncodeAndAppend(description);
}
iterator.Next();
}
else if (iterator.IsOnJavadocTag("value"))
{
string symbol = iterator.JavadocTagValue;
if (symbol == null || symbol == "")
{
output.EntityEncodeAndAppend(
Locale.Get("NaturalDocs.Engine", "Javadoc.Substitution.value")
);
}
else
{
output.EntityEncodeAndAppend(
Locale.Get("NaturalDocs.Engine", "Javadoc.Substitution.value(symbol)", symbol)
);
}
iterator.Next();
}
else
{
// Ignore indent. Spaces between words will be handled by line breaks.
// Ignore HTML comments.
// Ignore HTML tags.
iterator.Next();
}
}
}
/// <summary>
/// Initialize the combinator.
/// </summary>
/// <param name="parser">The core parser</param>
/// <param name="tokenIt">The token iterator</param>
public virtual void Init(Parser parser, TokenIterator tokenIt)
{
_parser = parser;
_tokenIt = tokenIt;
}
/* Function: GetParent
* Gets the bounds of the numbered parent, or returns false if it doesn't exist. Numbers start at zero.
*/
public bool GetParent(int index, out TokenIterator start, out TokenIterator end)
{
return(GetSectionBounds(SectionType.Parent, index, out start, out end));
}
/* Function: GetTemplateSuffix
* Gets the bounds of the template suffix attached to the class name, such as "<T>" in "List<T>", or returns false if there isn't one.
*/
public bool GetTemplateSuffix(out TokenIterator start, out TokenIterator end)
{
return(GetTokensInSection(SectionType.BeforeParents, 0, ClassPrototypeParsingType.TemplateSuffix,
out start, out end));
}
/* Function: GetModifiers
* Gets the bounds of any modifiers to the class, such as "static" or "public", or returns false if there aren't any.
*/
public bool GetModifiers(out TokenIterator start, out TokenIterator end)
{
return(GetTokensInSection(SectionType.BeforeParents, 0, ClassPrototypeParsingType.Modifier,
out start, out end));
}
override public string Process(string javascript, bool shrink = true)
{
Tokenizer source = new Tokenizer(javascript);
StringToStringTable substitutions = FindSubstitutions(source);
source = ApplySubstitutions(source, substitutions);
if (!shrink)
{
return(source.RawText);
}
// Search comments for sections to include in the output
StringBuilder output = new StringBuilder(javascript.Length);
string includeInOutput = FindIncludeInOutput(GetPossibleDocumentationComments(source));
if (includeInOutput != null)
{
output.AppendLine("/*");
output.Append(includeInOutput);
output.AppendLine("*/");
output.AppendLine();
}
// Shrink the source
TokenIterator iterator = source.FirstToken;
string spaceSeparatedSymbols = "+-";
while (iterator.IsInBounds)
{
char lastChar = (output.Length > 0 ? output[output.Length - 1] : '\0');
if (TryToSkipWhitespace(ref iterator) == true) // includes comments
{
char nextChar = iterator.Character;
if ((spaceSeparatedSymbols.IndexOf(lastChar) != -1 &&
spaceSeparatedSymbols.IndexOf(nextChar) != -1) ||
(Tokenizer.FundamentalTypeOf(lastChar) == FundamentalType.Text &&
Tokenizer.FundamentalTypeOf(nextChar) == FundamentalType.Text))
{
output.Append(' ');
}
}
else
{
TokenIterator prevIterator = iterator;
if (TryToSkipString(ref iterator) ||
TryToSkipRegex(ref iterator))
{
source.AppendTextBetweenTo(prevIterator, iterator, output);
}
else
{
iterator.AppendTokenTo(output);
iterator.Next();
}
}
}
return(output.ToString());
}
/* Function: GetPrototypeLinks
* Goes through the prototype of the passed <Topic> and adds any type links it finds to <LinkSet>.
*/
protected void GetPrototypeLinks(Topic topic, ref LinkSet linkSet)
{
if (topic.Prototype == null)
{
return;
}
Language language = EngineInstance.Languages.FromID(topic.LanguageID);
// We do this even for topics in the class hierarchy because the HTML output falls back to regular prototypes
// if there's no class prototype. Also, if there's parameter lists in the description the HTML generator will require
// type links to exist regardless of what type of prototype it creates. For example, this SystemVerilog interface:
//
// // Interface: myInterface
// //
// // Parameters:
// // PARAMNAME - description
//
// interface myInterface #(parameter PARAMNAME = 8) (input reset, clk);
//
// The HTML generation for the Parameters section will expect a type link to exist for PARAMNAME.
TokenIterator symbolStart = topic.ParsedPrototype.Tokenizer.FirstToken;
TokenIterator symbolEnd;
while (symbolStart.IsInBounds)
{
if (symbolStart.PrototypeParsingType == PrototypeParsingType.Type ||
symbolStart.PrototypeParsingType == PrototypeParsingType.TypeQualifier)
{
symbolEnd = symbolStart;
do
{
symbolEnd.Next();
}while (symbolEnd.PrototypeParsingType == PrototypeParsingType.Type ||
symbolEnd.PrototypeParsingType == PrototypeParsingType.TypeQualifier);
if (language.IsBuiltInType(symbolStart, symbolEnd) == false)
{
Link link = new Link();
// ignore LinkID
link.Type = LinkType.Type;
link.Symbol = SymbolString.FromPlainText_NoParameters(symbolStart.TextBetween(symbolEnd));
link.Context = topic.PrototypeContext;
// ignore contextID
link.FileID = topic.FileID;
link.ClassString = topic.ClassString;
// ignore classID
link.LanguageID = topic.LanguageID;
// ignore EndingSymbol
// ignore TargetTopicID
// ignore TargetScore
linkSet.Add(link);
}
symbolStart = symbolEnd;
}
else
{
symbolStart.Next();
}
}
}
override public string Process(string css, bool shrink = true)
{
Tokenizer source = new Tokenizer(css);
StringToStringTable substitutions = FindSubstitutions(source);
source = ApplySubstitutions(source, substitutions);
if (!shrink)
{
return(source.RawText);
}
// Search comments for sections to include in the output
StringBuilder output = new StringBuilder(css.Length);
string includeInOutput = FindIncludeInOutput(GetPossibleDocumentationComments(source));
if (includeInOutput != null)
{
output.AppendLine("/*");
output.Append(includeInOutput);
output.AppendLine("*/");
output.AppendLine();
}
// Shrink the source
TokenIterator iterator = source.FirstToken;
// We have to be more cautious than the JavaScript shrinker. You don't want something like "head .class" to become
// "head.class". Colon is a special case because we only want to remove spaces after it ("font-size: 12pt") and not
// before ("body :link").
string safeToCondenseAround = "{},;:+>[]= \0\n\r";
while (iterator.IsInBounds)
{
char lastChar = (output.Length > 0 ? output[output.Length - 1] : '\0');
if (TryToSkipWhitespace(ref iterator)) // includes comments
{
char nextChar = iterator.Character;
if (nextChar == ':' ||
(safeToCondenseAround.IndexOf(lastChar) == -1 &&
safeToCondenseAround.IndexOf(nextChar) == -1))
{
output.Append(' ');
}
}
else
{
TokenIterator prevIterator = iterator;
if (TryToSkipString(ref iterator))
{
source.AppendTextBetweenTo(prevIterator, iterator, output);
}
else
{
if (iterator.Character == '}' && lastChar == ';')
{
// Semicolons are unnecessary at the end of blocks. However, we have to do this here instead of in a
// global search and replace for ";}" because we don't want to alter that sequence if it appears in a string.
output[output.Length - 1] = '}';
}
else
{
iterator.AppendTokenTo(output);
}
iterator.Next();
}
}
}
return(output.ToString());
}
protected void SetupTokenIteratorReferences(TokenIterator tokenIt)
{
_context.PluginsMeta.TokenIt = tokenIt;
_context.PluginsMeta.Symbols = _context.Symbols;
}
/// <summary>
/// Parses an interpolated token into a set of tokens making up an interpolated expression.
/// </summary>
/// <param name="t"></param>
/// <returns></returns>
public Expr ParseInterpolatedExpression(Token t)
{
var iexp = new InterpolatedExpr();
iexp.Expressions = new List<Expr>();
// Convert each token in the interpolated string, need to convert it into
// it's own expression.
var tokens = t.Value as List<TokenData>;
foreach (var tokenData in tokens)
{
var token = tokenData.Token;
Expr exp = null;
// 1. true / false / "name" / 123 / null;
if (token.IsLiteralAny())
{
exp = token == Tokens.Null
? Exprs.Const(LObjects.Null, tokenData)
: Exprs.Const(TokenHelper.ConvertToLangLiteral(token), tokenData);
this.SetupContext(exp, tokenData);
_state.ExpressionCount++;
}
// 2. ${first + 'abc'} or ${ result / 2 + max }
else if (token.Kind == TokenKind.Multi)
{
var tokenIterator = new TokenIterator();
var tokens2 = token.Value as List<TokenData>;
tokenIterator.Init(tokens2, 1, 100);
tokenIterator.Advance();
var exisiting = _tokenIt;
// a. Temporarily set the token iterator for the parser to the one for the interpolation.
_tokenIt = tokenIterator;
SetupTokenIteratorReferences(this._tokenIt);
Exprs.Setup(_tokenIt, _context, _scriptPath);
// b. Now parse only the tokens supplied.
exp = ParseExpression(null);
// c. Reset the token iterator to the global one for the entire script.
_tokenIt = exisiting;
SetupTokenIteratorReferences(this._tokenIt);
Exprs.Setup(_tokenIt, _context, _scriptPath);
}
iexp.Expressions.Add(exp);
}
_tokenIt.Advance();
return iexp;
}
/* Function: GetParentModifiers
* Gets the bounds of the parent's modifiers, such as "public", or returns false if it couldn't find any.
*/
public bool GetParentModifiers(int index, out TokenIterator start, out TokenIterator end)
{
return(GetTokensInSection(SectionType.Parent, index, ClassPrototypeParsingType.Modifier,
out start, out end));
}
// Group: Parsing Functions
// __________________________________________________________________________
/* Function: TryToSkipClassDeclarationLine
*
* If the iterator is on a class's declaration line, moves it past it and returns true. It does not handle the class body.
*
* Supported Modes:
*
* - <ParseMode.IterateOnly>
* - <ParseMode.ParseClassPrototype>
* - Everything else is treated as <ParseMode.IterateOnly>.
*/
protected bool TryToSkipClassDeclarationLine(ref TokenIterator iterator, ParseMode mode = ParseMode.IterateOnly)
{
TokenIterator lookahead = iterator;
// Decorators
if (TryToSkipDecorators(ref lookahead, mode))
{
TryToSkipWhitespace(ref lookahead);
}
// Keyword
if (lookahead.MatchesToken("class") == false)
{
ResetTokensBetween(iterator, lookahead, mode);
return(false);
}
if (mode == ParseMode.ParseClassPrototype)
{
lookahead.ClassPrototypeParsingType = ClassPrototypeParsingType.Keyword;
}
lookahead.Next();
TryToSkipWhitespace(ref lookahead);
// Name
TokenIterator startOfIdentifier = lookahead;
if (TryToSkipIdentifier(ref lookahead) == false)
{
ResetTokensBetween(iterator, lookahead, mode);
return(false);
}
if (mode == ParseMode.ParseClassPrototype)
{
startOfIdentifier.SetClassPrototypeParsingTypeBetween(lookahead, ClassPrototypeParsingType.Name);
}
TryToSkipWhitespace(ref lookahead);
// Base classes
if (lookahead.Character == '(')
{
if (mode == ParseMode.ParseClassPrototype)
{
lookahead.ClassPrototypeParsingType = ClassPrototypeParsingType.StartOfParents;
}
lookahead.Next();
TryToSkipWhitespace(ref lookahead);
for (;;)
{
if (lookahead.Character == ')')
{
if (mode == ParseMode.ParseClassPrototype)
{
lookahead.ClassPrototypeParsingType = ClassPrototypeParsingType.EndOfParents;
}
break;
}
if (TryToSkipClassParent(ref lookahead, mode) == false)
{
ResetTokensBetween(iterator, lookahead, mode);
return(false);
}
TryToSkipWhitespace(ref lookahead);
if (lookahead.Character == ',')
{
if (mode == ParseMode.ParseClassPrototype)
{
lookahead.ClassPrototypeParsingType = ClassPrototypeParsingType.ParentSeparator;
}
lookahead.Next();
TryToSkipWhitespace(ref lookahead);
}
}
}
iterator = lookahead;
return(true);
}
/* Function: GetParentTemplateSuffix
* Gets the bounds of the parent's template suffix, or returns false if it couldn't find one.
*/
public bool GetParentTemplateSuffix(int index, out TokenIterator start, out TokenIterator end)
{
return(GetTokensInSection(SectionType.Parent, index, ClassPrototypeParsingType.TemplateSuffix,
out start, out end));
}
// Group: Parsing Functions
// __________________________________________________________________________
/* Function: TryToSkipClassDeclarationLine
*
* If the iterator is on a class's declaration line, moves it past it and returns true. It does not handle the class body.
*
* Supported Modes:
*
* - <ParseMode.IterateOnly>
* - <ParseMode.ParseClassPrototype>
* - Everything else is treated as <ParseMode.IterateOnly>.
*/
protected bool TryToSkipClassDeclarationLine(ref TokenIterator iterator, ParseMode mode = ParseMode.IterateOnly)
{
TokenIterator lookahead = iterator;
// Keyword
if (lookahead.MatchesToken("class") == false)
{
return(false);
}
if (mode == ParseMode.ParseClassPrototype)
{
lookahead.ClassPrototypeParsingType = ClassPrototypeParsingType.Keyword;
}
lookahead.Next();
TryToSkipWhitespace(ref lookahead);
// Name
TokenIterator startOfIdentifier = lookahead;
if (TryToSkipUnqualifiedIdentifier(ref lookahead) == false)
{
ResetTokensBetween(iterator, lookahead, mode);
return(false);
}
if (mode == ParseMode.ParseClassPrototype)
{
startOfIdentifier.SetClassPrototypeParsingTypeBetween(lookahead, ClassPrototypeParsingType.Name);
}
TryToSkipWhitespace(ref lookahead);
// Base class
if (lookahead.Character == '<')
{
if (mode == ParseMode.ParseClassPrototype)
{
lookahead.ClassPrototypeParsingType = ClassPrototypeParsingType.StartOfParents;
}
lookahead.Next();
TryToSkipWhitespace(ref lookahead);
TokenIterator startOfParent = lookahead;
if (TryToSkipUnqualifiedIdentifier(ref lookahead) == false)
{
ResetTokensBetween(iterator, lookahead, mode);
return(false);
}
if (mode == ParseMode.ParseClassPrototype)
{
startOfParent.SetClassPrototypeParsingTypeBetween(lookahead, ClassPrototypeParsingType.Name);
}
}
iterator = lookahead;
return(true);
}
/* Function: CalculateSections
*/
protected void CalculateSections()
{
sections = new List <Section>();
Section section = null;
TokenIterator iterator = tokenizer.FirstToken;
iterator.NextPastWhitespace();
// Pre-Prototype Lines
while (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType == ClassPrototypeParsingType.StartOfPrePrototypeLine)
{
section = new Section();
section.Type = SectionType.PrePrototypeLine;
section.StartIndex = iterator.TokenIndex;
do
{
iterator.Next();
}while (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType == ClassPrototypeParsingType.PrePrototypeLine);
section.EndIndex = iterator.TokenIndex;
sections.Add(section);
iterator.NextPastWhitespace();
}
// Before Parents
TokenIterator startOfSection = iterator;
section = new Section();
section.Type = SectionType.BeforeParents;
section.StartIndex = startOfSection.TokenIndex;
while (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfParents &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfPostPrototypeLine &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfBody)
{
iterator.Next();
}
TokenIterator lookbehind = iterator;
lookbehind.PreviousPastWhitespace(PreviousPastWhitespaceMode.EndingBounds, startOfSection);
section.EndIndex = lookbehind.TokenIndex;
sections.Add(section);
// Parents
if (iterator.ClassPrototypeParsingType == ClassPrototypeParsingType.StartOfParents)
{
do
{
iterator.Next();
}while (iterator.ClassPrototypeParsingType == ClassPrototypeParsingType.StartOfParents);
iterator.NextPastWhitespace();
while (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.EndOfParents &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfPostPrototypeLine &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfBody)
{
startOfSection = iterator;
section = new Section();
section.Type = SectionType.Parent;
section.StartIndex = startOfSection.TokenIndex;
while (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.ParentSeparator &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.EndOfParents &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfPostPrototypeLine &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfBody)
{
iterator.Next();
}
lookbehind = iterator;
lookbehind.PreviousPastWhitespace(PreviousPastWhitespaceMode.EndingBounds, startOfSection);
section.EndIndex = lookbehind.TokenIndex;
sections.Add(section);
if (iterator.ClassPrototypeParsingType == ClassPrototypeParsingType.ParentSeparator)
{
do
{
iterator.Next();
}while (iterator.ClassPrototypeParsingType == ClassPrototypeParsingType.ParentSeparator);
iterator.NextPastWhitespace();
}
}
}
// After Parents
if (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfPostPrototypeLine &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfBody)
{
startOfSection = iterator;
section = new Section();
section.Type = SectionType.AfterParents;
section.StartIndex = iterator.TokenIndex;
do
{
iterator.Next();
}while (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfPostPrototypeLine &&
iterator.ClassPrototypeParsingType != ClassPrototypeParsingType.StartOfBody);
lookbehind = iterator;
lookbehind.PreviousPastWhitespace(PreviousPastWhitespaceMode.EndingBounds, startOfSection);
section.EndIndex = lookbehind.TokenIndex;
sections.Add(section);
}
// Post-Prototype Lines
while (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType == ClassPrototypeParsingType.StartOfPostPrototypeLine)
{
section = new Section();
section.Type = SectionType.PostPrototypeLine;
section.StartIndex = iterator.TokenIndex;
do
{
iterator.Next();
}while (iterator.IsInBounds &&
iterator.ClassPrototypeParsingType == ClassPrototypeParsingType.PostPrototypeLine);
section.EndIndex = iterator.TokenIndex;
sections.Add(section);
iterator.NextPastWhitespace();
}
}
// Group: Functions
// __________________________________________________________________________
/* Constructor: Section
*/
public Section(TokenIterator start, TokenIterator end)
{
this.start = start;
this.end = end;
}
/* Function: GetSectionBounds
* Returns the bounds of the passed section and whether it exists. An index of zero represents the first section of that
* type, 1 represents the second, etc.
*/
protected bool GetSectionBounds(SectionType type, int index, out TokenIterator start, out TokenIterator end)
{
Section section = FindSection(type, index);
if (section == null)
{
start = tokenizer.LastToken;
end = start;
return(false);
}
else
{
start = tokenizer.FirstToken;
start.Next(section.StartIndex);
end = start;
end.Next(section.EndIndex - section.StartIndex);
return(true);
}
}
/* Function: ScoreParameter
* Returns a two bit value representing how well the parameters match, or -1 if they match so poorly that the link and
* the target shouldn't be considered a match at all.
*/
private long ScoreParameter(ParsedPrototype prototype, ParameterString linkParameters, int index, bool ignoreCase)
{
// -1 - The link has a parameter but the prototype does not.
// 00 - The prototype has a parameter but the link does not. This allows links on partial parameters.
// 00 - They both have parameters but do not match at all.
// 01 - The link doesn't have a parameter but the prototype has one with a default value set.
// 10 - The parameters match except for qualifiers or modifiers like "unsigned".
// 11 - The parameters match completely, by type or by name, or both don't exist.
SimpleTokenIterator linkParamStart, linkParamEnd;
TokenIterator prototypeParamStart, prototypeParamEnd;
bool hasLinkParam = linkParameters.GetParameter(index, out linkParamStart, out linkParamEnd);
bool hasPrototypeParam;
if (prototype == null)
{
hasPrototypeParam = false;
// To shut the compiler up.
prototypeParamStart = new TokenIterator();
prototypeParamEnd = new TokenIterator();
}
else
{
hasPrototypeParam = prototype.GetParameter(index, out prototypeParamStart, out prototypeParamEnd);
}
if (!hasLinkParam)
{
if (!hasPrototypeParam)
{
return(3);
}
else
{
// There is a prototype parameter but not a link parameter. This will be 0 or 1 depending on whether the
// prototype parameter has a default value.
while (prototypeParamStart < prototypeParamEnd)
{
if (prototypeParamStart.PrototypeParsingType == PrototypeParsingType.DefaultValue)
{
return(1);
}
prototypeParamStart.Next();
}
return(0);
}
}
else // hasLinkParam == true
{
if (hasPrototypeParam == false)
{
return(-1);
}
// Both the link and the prototype have parameters at index.
bool typeMatch = false;
bool typeMismatch = false;
bool typeModifierMismatch = false;
bool nameMatch = false;
bool nameMismatch = false;
int modifierBlockLevel = 0;
while (prototypeParamStart < prototypeParamEnd)
{
var type = prototypeParamStart.PrototypeParsingType;
// We want any mismatches that occur nested in type modifier blocks to be scored as a modifier mismatch.
if (type == PrototypeParsingType.OpeningTypeModifier ||
type == PrototypeParsingType.OpeningParamModifier)
{
modifierBlockLevel++;
}
else if (type == PrototypeParsingType.ClosingTypeModifier ||
type == PrototypeParsingType.ClosingParamModifier)
{
modifierBlockLevel--;
}
else if (modifierBlockLevel > 0)
{
type = PrototypeParsingType.TypeModifier;
}
switch (type)
{
case PrototypeParsingType.TypeModifier:
case PrototypeParsingType.TypeQualifier:
case PrototypeParsingType.OpeningTypeModifier:
case PrototypeParsingType.ClosingTypeModifier:
case PrototypeParsingType.ParamModifier:
case PrototypeParsingType.OpeningParamModifier:
case PrototypeParsingType.ClosingParamModifier:
if (linkParamStart < linkParamEnd && linkParamStart.MatchesToken(prototypeParamStart, ignoreCase))
{
linkParamStart.Next();
linkParamStart.NextPastWhitespace();
}
else
{
typeModifierMismatch = true;
}
break;
case PrototypeParsingType.Type:
if (linkParamStart < linkParamEnd && linkParamStart.MatchesToken(prototypeParamStart, ignoreCase))
{
typeMatch = true;
linkParamStart.Next();
linkParamStart.NextPastWhitespace();
}
else
{
typeMismatch = true;
}
break;
case PrototypeParsingType.Name:
if (linkParamStart < linkParamEnd && linkParamStart.MatchesToken(prototypeParamStart, ignoreCase))
{
nameMatch = true;
linkParamStart.Next();
linkParamStart.NextPastWhitespace();
}
else
{
nameMismatch = true;
}
break;
}
prototypeParamStart.Next();
prototypeParamStart.NextPastWhitespace();
}
if (linkParamStart < linkParamEnd)
{
return(0);
}
if (nameMatch && !nameMismatch)
{
return(3);
}
if (typeMatch && !typeMismatch)
{
if (!typeModifierMismatch)
{
return(3);
}
else
{
return(2);
}
}
return(0);
}
}
/* Function: GetPrePrototypeLine
* Returns the bounds of a numbered pre-prototype line. Numbers start at zero. It will return false if one does not
* exist at that number.
*/
public bool GetPrePrototypeLine(int lineNumber, out TokenIterator start, out TokenIterator end)
{
return(GetSectionBounds(SectionType.PrePrototypeLine, lineNumber, out start, out end));
}
/* Function: BuildTypeLinkedAndSyntaxHighlightedText
*
* Formats the text between the iterators with syntax highlighting and links for any tokens marked with
* <PrototypeParsingType.Type> and <PrototypeParsingType.TypeQualifier>.
*
* Parameters:
* start - The first token of the text to convert.
* end - The end of the text to convert, which is one token past the last one included.
* extendTypeSearch - If true, it will search beyond the bounds of the iterators to get the complete type.
* This allows you to format only a portion of the link with this function yet still have
* the link go to the complete destination.
* output - The StringBuilder to append the output to. If null, it will used <htmlOutput>.
*
* Variables Required:
* - <topic>, <links>, and <linkTargets> must be set.
* - <htmlOutput> must be set if the output parameter is not set.
*/
protected void BuildTypeLinkedAndSyntaxHighlightedText(TokenIterator start, TokenIterator end,
bool extendTypeSearch = false,
StringBuilder output = null)
{
#if DEBUG
if (topic == null)
{
throw new Exception("Tried to call BuildTypeLinkedAndSyntaxHighlightedText() without setting the topic variable.");
}
if (links == null)
{
throw new Exception("Tried to call BuildTypeLinkedAndSyntaxHighlightedText() without setting the links variable.");
}
if (linkTargets == null)
{
throw new Exception("Tried to call BuildTypeLinkedAndSyntaxHighlightedText() without setting the linkTargets variable.");
}
if (output == null && htmlOutput == null)
{
throw new Exception("Tried to call BuildTypeLinkedAndSyntaxHighlightedText() without setting the output parameter or the htmlOutput variable.");
}
#endif
if (output == null)
{
output = htmlOutput;
}
Language language = EngineInstance.Languages.FromID(topic.LanguageID);
// Find each Type/TypeQualifier stretch in the text
TokenIterator iterator = start;
while (iterator < end)
{
if (iterator.PrototypeParsingType == PrototypeParsingType.Type ||
iterator.PrototypeParsingType == PrototypeParsingType.TypeQualifier)
{
TokenIterator textStart = iterator;
TokenIterator textEnd = iterator;
do
{
textEnd.Next();
}while (textEnd < end &&
(textEnd.PrototypeParsingType == PrototypeParsingType.Type ||
textEnd.PrototypeParsingType == PrototypeParsingType.TypeQualifier));
TokenIterator symbolStart = textStart;
TokenIterator symbolEnd = textEnd;
// Extend past start and end if the flag is set
if (extendTypeSearch && symbolStart == start)
{
TokenIterator temp = symbolStart;
temp.Previous();
while (temp.IsInBounds &&
(temp.PrototypeParsingType == PrototypeParsingType.Type ||
temp.PrototypeParsingType == PrototypeParsingType.TypeQualifier))
{
symbolStart = temp;
temp.Previous();
}
}
if (extendTypeSearch && symbolEnd == end)
{
while (symbolEnd.IsInBounds &&
(symbolEnd.PrototypeParsingType == PrototypeParsingType.Type ||
symbolEnd.PrototypeParsingType == PrototypeParsingType.TypeQualifier))
{
symbolEnd.Next();
}
}
// Built in types don't get links
if (language.IsBuiltInType(symbolStart, symbolEnd))
{
BuildSyntaxHighlightedText(textStart, textEnd, output);
}
else
{
// Create a link object with the identifying properties needed to look it up in the list of links.
Link linkStub = new Link();
linkStub.Type = LinkType.Type;
linkStub.Symbol = SymbolString.FromPlainText_NoParameters(symbolStart.Tokenizer.TextBetween(symbolStart, symbolEnd));
linkStub.Context = topic.PrototypeContext;
linkStub.ContextID = topic.PrototypeContextID;
linkStub.FileID = topic.FileID;
linkStub.ClassString = topic.ClassString;
linkStub.ClassID = topic.ClassID;
linkStub.LanguageID = topic.LanguageID;
// Find the actual link so we know if it resolved to anything.
Link fullLink = null;
foreach (Link link in links)
{
if (link.SameIDPropertiesAs(linkStub))
{
fullLink = link;
break;
}
}
#if DEBUG
if (fullLink == null)
{
throw new Exception("All links in a topic must be in the list passed to HTMLComponent.");
}
#endif
// If it didn't resolve, we just output the original text.
if (!fullLink.IsResolved)
{
BuildSyntaxHighlightedText(textStart, textEnd, output);
}
else
{
// If it did resolve, find Topic it resolved to.
Topic targetTopic = null;
foreach (Topic linkTarget in linkTargets)
{
if (linkTarget.TopicID == fullLink.TargetTopicID)
{
targetTopic = linkTarget;
break;
}
}
#if DEBUG
if (targetTopic == null)
{
throw new Exception("All links targets for a topic must be in the list passed to HTMLComponent.");
}
#endif
BuildLinkTag(targetTopic, null, output);
BuildSyntaxHighlightedText(textStart, textEnd, output);
output.Append("</a>");
}
}
iterator = textEnd;
}
else // not on a type
{
TokenIterator startText = iterator;
do
{
iterator.Next();
}while (iterator < end &&
iterator.PrototypeParsingType != PrototypeParsingType.Type &&
iterator.PrototypeParsingType != PrototypeParsingType.TypeQualifier);
BuildSyntaxHighlightedText(startText, iterator, output);
}
}
}
/* Function: GetKeyword
* Gets the bounds of the class keyword, such as "class", "struct", or "interface", or returns false if it couldn't find it.
*/
public bool GetKeyword(out TokenIterator start, out TokenIterator end)
{
return(GetTokensInSection(SectionType.BeforeParents, 0, ClassPrototypeParsingType.Keyword,
out start, out end));
}
/* Function: ApplySubstitutions
* Finds all substitutions in the source that match those in the table and replaces them with their values. Will also comment
* out any substitution definitions found.
*/
protected Tokenizer ApplySubstitutions(Tokenizer source, StringToStringTable substitutions, bool applyNestedSubstitutions = true)
{
TokenIterator iterator = source.FirstToken;
// Find the first valid substitution identifier or definition. If there aren't any we don't want to \do unnecessary memory
// allocation and processing.
bool foundSubstitution = false;
string identifier = null;
string localeIdentifier, value, declaration;
while (iterator.IsInBounds)
{
if (TryToSkipSubstitutionIdentifier(ref iterator, out identifier) ||
TryToSkipLocaleSubstitutionIdentifier(ref iterator, out identifier, out localeIdentifier))
{
foundSubstitution = true;
break;
}
// else if (TryToSkipSubstitutionDefinition())
// {
// Unnecessary because definitions will start with identifiers so it will get picked up by that
// }
else
{
GenericSkip(ref iterator);
}
}
if (!foundSubstitution)
{
return(source);
}
// Now that we know we have one, we can back up the iterator and build new output
iterator.PreviousByCharacters(identifier.Length);
StringBuilder output = new StringBuilder(source.RawText.Length);
output.Append(source.RawText, 0, iterator.RawTextIndex);
while (iterator.IsInBounds)
{
TokenIterator previousIterator = iterator;
if (TryToSkipSubstitutionDefinition(ref iterator, out identifier, out value, out declaration))
{
if (this.blockCommentStringPairs != null)
{
output.Append(this.blockCommentStringPairs[0] + ' ' + declaration + ' ' + this.blockCommentStringPairs[1]);
}
}
else if (TryToSkipSubstitutionIdentifier(ref iterator, out identifier))
{
string substitution = substitutions[identifier];
if (substitution == null)
{
output.Append(identifier);
}
else
{
if (applyNestedSubstitutions)
{
substitution = ApplyNestedSubstitutions(substitution, substitutions);
}
output.Append(substitution);
}
}
else if (TryToSkipLocaleSubstitutionIdentifier(ref iterator, out identifier, out localeIdentifier))
{
string substitution = Engine.Locale.SafeGet("NaturalDocs.Engine", localeIdentifier, null);
if (substitution == null)
{
output.Append(identifier);
}
else
{
output.Append('"' + substitution.StringEscape() + '"');
}
}
else
{
GenericSkip(ref iterator);
source.AppendTextBetweenTo(previousIterator, iterator, output);
}
}
return(new Tokenizer(output.ToString()));
}
/* Function: GetJavadocLinkSymbol
* Returns the symbol part of a @see or {@link} tag and moves the iterator past it.
*/
protected string GetJavadocLinkSymbol(ref TokenIterator iterator)
{
StringBuilder symbol = new StringBuilder();
// In Javadoc, spaces are only allowed in parentheses. We're going to go further and allow them in any braces to support
// templates and other languages. However, for angle brackets they must be preceded by a comma. This allows
// "Template<A, B>" to be supported while not getting tripped on "operator<".
// Most symbols won't have braces so create this on demand.
SafeStack <char> braceStack = null;
while (iterator.IsInBounds)
{
if (iterator.Character == '(' ||
iterator.Character == '[' ||
iterator.Character == '{' ||
iterator.Character == '<')
{
if (braceStack == null)
{
braceStack = new SafeStack <char>();
}
braceStack.Push(iterator.Character);
symbol.Append(iterator.Character);
}
else if ((iterator.Character == ')' && braceStack != null && braceStack.Peek() == '(') ||
(iterator.Character == ']' && braceStack != null && braceStack.Peek() == '[') ||
(iterator.Character == '}' && braceStack != null && braceStack.Peek() == '{') ||
(iterator.Character == '>' && braceStack != null && braceStack.Peek() == '<'))
{
braceStack.Pop();
symbol.Append(iterator.Character);
}
else if (iterator.Character == '}' && (braceStack == null || braceStack.Contains('{') == false))
{
// If we're at an unopened closing brace we're probably at the end of a {@link}. We check if the stack contains an
// opening brace instead of checking whether it's empty to ignore any possible opening angle brackets that could
// screw us up like "operator<".
break;
}
else if (iterator.FundamentalType == FundamentalType.Text ||
iterator.FundamentalType == FundamentalType.Symbol)
{
iterator.AppendTokenTo(symbol);
}
else if (iterator.FundamentalType == FundamentalType.Whitespace)
{
if (braceStack == null || braceStack.Count == 0)
{
break;
}
else if (braceStack.Peek() == '<')
{
TokenIterator lookbehind = iterator;
lookbehind.Previous();
if (lookbehind.Character == ',')
{
iterator.AppendTokenTo(symbol);
}
else
{
break;
}
}
else
{
iterator.AppendTokenTo(symbol);
}
}
else // line break
{
break;
}
iterator.Next();
}
// Javadoc uses Class.Class#Member. First remove leading hashes to handle just #Member
while (symbol.Length > 0 && symbol[0] == '#')
{
symbol.Remove(0, 1);
}
// Convert any remaining hashes to dots. Ideally we would use the language's native member operator but it's not
// easy to get it here.
return(symbol.ToString().Replace('#', '.'));
}
/// <summary>
/// Sets up the reference to token iterator and context
/// </summary>
/// <param name="tk"></param>
/// <param name="ctx"></param>
public static void Setup(TokenIterator tk, Context ctx, string scriptName)
{
_tokenIt = tk;
_ctx = ctx;
_scriptName = scriptName;
}
/* Function: TryToGetFirstBlockLine
* If the iterator is on a line that starts with one of the <BlockTags>, extracts the components and returns true.
* Use <GetBlockTag()> to get the complete block since it may span multiple lines.
*/
protected bool TryToGetFirstBlockLine(LineIterator lineIterator, out string tag, out TokenIterator startOfContent)
{
tag = null;
startOfContent = default(TokenIterator);
TokenIterator tokenIterator = lineIterator.FirstToken(LineBoundsMode.CommentContent);
if (tokenIterator.Character != '@')
{
return(false);
}
tokenIterator.Next();
if (tokenIterator.FundamentalType != FundamentalType.Text)
{
return(false);
}
string possibleTag = tokenIterator.String;
if (BlockTags.Contains(possibleTag) == false)
{
return(false);
}
tokenIterator.Next();
tokenIterator.NextPastWhitespace();
tag = possibleTag;
startOfContent = tokenIterator;
return(true);
}
/// <summary>
/// Initialize the context.
/// </summary>
/// <param name="context"></param>
public Parser(Context context) : base(context)
{
_tokenIt = new TokenIterator();
}
/* Function: TryToGetBlock
* If the iterator is on a line that starts with one of the <BlockTags>, parses it, adds its content to the comment,
* moves the iterator past it, and returns true. If it is not at the start of a tag block it will return false and change
* nothing.
*/
protected bool TryToGetBlock(ref LineIterator lineIterator, LineIterator limit, JavadocComment comment)
{
if (lineIterator >= limit)
{
return(false);
}
// Get the complete content across multiple lines.
string tag;
TokenIterator startOfContent;
if (TryToGetFirstBlockLine(lineIterator, out tag, out startOfContent) == false)
{
return(false);
}
for (;;)
{
lineIterator.Next();
if (lineIterator >= limit || IsFirstBlockLine(lineIterator))
{
break;
}
}
TokenIterator endOfContent = lineIterator.FirstToken(LineBoundsMode.Everything);
// Any "@tag item description", possibly in a list
if (tag == "exception" ||
tag == "param" ||
tag == "throws")
{
TokenIterator iterator = startOfContent;
string symbol = null;
TryToGetBlockSymbol(ref iterator, endOfContent, out symbol);
iterator.NextPastWhitespace();
string description = GetText(iterator, endOfContent);
description = Normalize(description);
if (symbol == null || symbol == "" || description == null || description == "")
{
return(false);
}
var commentBlock = comment.GetListBlock(tag);
commentBlock.Add(symbol, description);
return(true);
}
// Any "@tag description", possibly in a list
else if (tag == "author" ||
tag == "deprecated" ||
tag == "since" ||
tag == "version")
{
string description = GetText(startOfContent, endOfContent);
description = Normalize(description);
if (description == null || description == "")
{
return(false);
}
if (tag == "deprecated")
{
if (comment.Deprecated == null)
{
comment.Deprecated = description;
}
else
{
comment.Deprecated += description;
}
}
else
{
var commentBlock = comment.GetListBlock(tag);
commentBlock.Add(null, description);
}
return(true);
}
// Any "@tag description" that can't be in a list
else if (tag == "return")
{
string description = GetText(startOfContent, endOfContent);
description = Normalize(description);
if (description == null || description == "")
{
return(false);
}
var commentBlock = comment.GetTextBlock(tag);
commentBlock.Text.Append(description);
return(true);
}
else if (tag == "see")
{
string description = null;
TokenIterator iterator = startOfContent;
// @see "Description"
// @see <a href="link">Description</a>
if (iterator.Character == '"' ||
iterator.Character == '<')
{
// There's not symbol so interpret the whole thing as the description. We'll let GetText() handle the HTML link.
description = GetText(iterator, endOfContent);
description = Normalize(description);
}
// @see Class.Class#Member
// @see Class.Class#Member Description
else
{
string symbol = GetJavadocLinkSymbol(ref iterator);
iterator.NextPastWhitespace();
description = GetSimpleText(iterator, endOfContent);
description = Normalize(description);
if (description == null || description == "")
{
description = "<p><link type=\"naturaldocs\" originaltext=\"" + symbol.EntityEncode() + "\"></p>";
}
else
{
description = "<p><link type=\"naturaldocs\" originaltext=\"" + description.EntityEncode() + " at " + symbol.EntityEncode() + "\"></p>";
}
}
if (description == null || description == "")
{
return(false);
}
var commentBlock = comment.GetListBlock(tag);
commentBlock.Add(null, description);
return(true);
}
// Ignored blocks
// - serial
// - serialField
// - serialData
else
{
return(true);
}
}
/* Function: TryToGetBlockSymbol
* If the iterator is on the symbol part of a block tag that has one, such as "@param symbol description", extracts the symbol,
* moves the iterator past it, and returns true.
*/
protected bool TryToGetBlockSymbol(ref TokenIterator iterator, TokenIterator limit, out string entryText)
{
TokenIterator lookahead = iterator;
// Javadoc recommends documenting template parameters as "@param <T> ...".
if (lookahead.Character == '<')
{
lookahead.Next();
for (;;)
{
if (lookahead >= limit)
{
entryText = null;
return(false);
}
else if (lookahead.Character == '>')
{
lookahead.Next();
break;
}
else
{
lookahead.Next();
}
}
}
else // not '<'
{
for (;;)
{
if (lookahead >= limit)
{
entryText = null;
return(false);
}
else if (lookahead.FundamentalType == FundamentalType.Text ||
lookahead.Character == '_' ||
lookahead.Character == '.')
{
lookahead.Next();
}
else if (lookahead.MatchesAcrossTokens("::") ||
lookahead.MatchesAcrossTokens("->"))
{
lookahead.NextByCharacters(2);
}
else
{
break;
}
}
}
if (lookahead >= limit || lookahead.FundamentalType != FundamentalType.Whitespace)
{
entryText = null;
return(false);
}
entryText = iterator.TextBetween(lookahead);
iterator = lookahead;
return(true);
}
/* Function: GetText
*
* Converts a stretch of formatted text to NDMarkup.
*
* Modes:
*
* Normal - The iterator continues until it goes out of bounds.
* ListItem - The iterator continues until it reaches a closing li tag. It also skips certain formatting that is not supported
* in list items in NDMarkup.
*/
protected void GetText(ref JavadocIterator iterator, StringBuilder output, GetTextMode mode = GetTextMode.Normal)
{
output.Append("<p>");
TagStack tagStack = new TagStack();
tagStack.OpenTag(null, "</p>");
while (iterator.IsInBounds)
{
if (iterator.IsOn(JavadocElementType.Text))
{
output.EntityEncodeAndAppend(iterator.String);
iterator.Next();
}
else if (iterator.IsOn(JavadocElementType.EntityChar))
{
output.EntityEncodeAndAppend(iterator.EntityValue);
iterator.Next();
}
else if (iterator.IsOn(JavadocElementType.LineBreak))
{
// Add a literal line break. We'll replace these with spaces or double spaces later. Right now we can't decide
// which it should be because you can't run a regex directly on a StringBuilder and it would be inefficient to convert
// it to a string on every line break.
output.Append('\n');
iterator.Next();
}
else if (iterator.IsOnHTMLTag("p"))
{
// Text can appear both inside and outside of <p> tags, whitespace can appear between <p> tags that can be
// mistaken for content, and people can use <p> tags as standalone rather than opening tags. Rather than put in
// logic to try to account for all of this we handle it in a very dirty but simple way. Every <p> tag--opening, closing,
// or standalone--causes a paragraph break. Normalize() will clean it up for us afterwards.
tagStack.CloseTag(1, output); // Reuse our surrounding tag
output.Append("</p><p>");
iterator.Next();
}
else if (iterator.IsOnHTMLTag("b") ||
iterator.IsOnHTMLTag("strong"))
{
if (iterator.HTMLTagForm == TagForm.Opening)
{
tagStack.OpenTag(iterator.TagType, "</b>");
output.Append("<b>");
}
else if (iterator.HTMLTagForm == TagForm.Closing)
{
tagStack.CloseTag(iterator.TagType, output);
}
iterator.Next();
}
else if (iterator.IsOnHTMLTag("i") ||
iterator.IsOnHTMLTag("em"))
{
if (iterator.HTMLTagForm == TagForm.Opening)
{
tagStack.OpenTag(iterator.TagType, "</i>");
output.Append("<i>");
}
else if (iterator.HTMLTagForm == TagForm.Closing)
{
tagStack.CloseTag(iterator.TagType, output);
}
iterator.Next();
}
else if (iterator.IsOnHTMLTag("u"))
{
if (iterator.HTMLTagForm == TagForm.Opening)
{
tagStack.OpenTag(iterator.TagType, "</u>");
output.Append("<u>");
}
else if (iterator.HTMLTagForm == TagForm.Closing)
{
tagStack.CloseTag(iterator.TagType, output);
}
iterator.Next();
}
else if (iterator.IsOnHTMLTag("pre", TagForm.Opening) && mode == GetTextMode.Normal) // Ignore pre's in list items
{
output.Append("</p>");
GetPre(ref iterator, output);
output.Append("<p>");
}
else if (iterator.IsOnHTMLTag("ul") ||
iterator.IsOnHTMLTag("ol"))
{
if (iterator.HTMLTagForm == TagForm.Opening)
{
output.Append("</p>");
GetList(ref iterator, output);
output.Append("<p>");
}
else if (iterator.HTMLTagForm == TagForm.Closing && mode == GetTextMode.ListItem)
{
break;
}
else
{
iterator.Next();
}
}
else if (iterator.IsOnHTMLTag("li", TagForm.Closing) && mode == GetTextMode.ListItem)
{
break;
}
else if (iterator.IsOnHTMLTag("a", TagForm.Opening))
{
string href = iterator.HTMLTagProperty("href");
if (href == null || href == "" || href == "#" || href.StartsWith("{@docRoot}") ||
href.StartsWith("javascript:", StringComparison.CurrentCultureIgnoreCase))
{
iterator.Next();
}
else
{
GetHTMLLink(ref iterator, output);
}
}
else if (iterator.IsOnJavadocTag("code") ||
iterator.IsOnJavadocTag("literal"))
{
// These get added without searching the contents for nested tags
output.EntityEncodeAndAppend(iterator.JavadocTagValue);
iterator.Next();
}
else if (iterator.IsOnJavadocTag("link") ||
iterator.IsOnJavadocTag("linkPlain"))
{
Tokenizer linkContent = new Tokenizer(iterator.JavadocTagValue);
TokenIterator linkIterator = linkContent.FirstToken;
string symbol = GetJavadocLinkSymbol(ref linkIterator);
linkIterator.NextPastWhitespace();
string description = GetSimpleText(linkIterator, linkContent.LastToken);
description = Normalize(description);
if (description == null || description == "")
{
output.Append("<link type=\"naturaldocs\" originaltext=\"");
output.EntityEncodeAndAppend(symbol);
output.Append("\">");
}
else
{
output.Append("<link type=\"naturaldocs\" originaltext=\"");
output.EntityEncodeAndAppend(description);
output.Append(" at ");
output.EntityEncodeAndAppend(symbol);
output.Append("\">");
}
iterator.Next();
}
else if (iterator.IsOnJavadocTag("value"))
{
string symbol = iterator.JavadocTagValue;
if (symbol == null || symbol == "")
{
output.EntityEncodeAndAppend(
Locale.Get("NaturalDocs.Engine", "Javadoc.Substitution.value")
);
}
else
{
string substitution = Locale.Get("NaturalDocs.Engine", "Javadoc.Substitution.value(symbol)", '\x1F');
int substitutionIndex = substitution.IndexOf('\x1F');
if (substitutionIndex == -1)
{
output.EntityEncodeAndAppend(substitution);
}
else
{
if (substitutionIndex > 0)
{
output.EntityEncodeAndAppend(substitution, 0, substitutionIndex);
}
output.Append("<link type=\"naturaldocs\" originaltext=\"");
output.EntityEncodeAndAppend(symbol);
output.Append("\">");
if (substitutionIndex < substitution.Length - 1)
{
output.EntityEncodeAndAppend(substitution, substitutionIndex + 1, substitution.Length - (substitutionIndex + 1));
}
}
}
iterator.Next();
}
else
{
// Ignore indent. Spaces between words will be handled by line breaks.
// Ignore HTML comments.
// Ignore unrecognized HTML tags.
iterator.Next();
}
}
tagStack.CloseAllTags(output);
}
/// <summary>
/// Parses parameters.
/// </summary>
/// <param name="args">The list of arguments to store.</param>
/// <param name="tokenIt">The token iterator</param>
/// <param name="parser">The parser</param>
/// <param name="meta">The function meta for checking parameters</param>
/// <param name="expectParenthesis">Whether or not to expect parenthis to designate the start of the parameters.</param>
/// <param name="enableNewLineAsEnd">Whether or not to treat a newline as end</param>
public static void ParseFuncParameters(List<Expr> args, TokenIterator tokenIt, Parser.Parser parser, bool expectParenthesis, bool enableNewLineAsEnd, FunctionMetaData meta)
{
int totalParameters = 0;
if (tokenIt.NextToken.Token == Tokens.LeftParenthesis)
expectParenthesis = true;
// START with check for "("
if (expectParenthesis) tokenIt.Expect(Tokens.LeftParenthesis);
bool passNewLine = !enableNewLineAsEnd;
var endTokens = BuildEndTokens(enableNewLineAsEnd, meta);
int totalNamedParams = 0;
var hasMetaArguments = meta != null && meta.ArgumentNames != null && meta.ArgumentNames.Count > 0;
while (true)
{
Expr exp = null;
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
break;
if (tokenIt.NextToken.Token == Tokens.Comma)
tokenIt.Advance();
var token = tokenIt.NextToken.Token;
var peek = tokenIt.Peek().Token;
var isVar = parser.Context.Symbols.Contains(token.Text);
var isParamNameMatch = hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text);
var isKeywordParamName = token.Kind == TokenKind.Keyword && isParamNameMatch;
// CASE 1: Named params for external c# object method calls
// CASE 2: Named params for internal script functions ( where we have access to its param metadata )
if ( (meta == null && token.Kind == TokenKind.Ident && peek == Tokens.Colon )
|| (token.Kind == TokenKind.Ident && isParamNameMatch && !isVar)
|| (token.Kind == TokenKind.Ident && !isParamNameMatch && !isVar && peek == Tokens.Colon)
|| (isKeywordParamName && !isVar ) )
{
var paramName = token.Text;
var namedParamToken = tokenIt.NextToken;
tokenIt.Advance();
// Advance and check if ":"
if (tokenIt.NextToken.Token == Tokens.Colon)
tokenIt.Advance();
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
exp = Exprs.NamedParam(paramName, exp, namedParamToken);
args.Add(exp);
totalNamedParams++;
}
// CASE 2: Name of variable being passed to function is same as one of the parameter names.
else if (isVar && hasMetaArguments && meta.ArgumentsLookup.ContainsKey(token.Text))
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
var next = tokenIt.Peek();
if (next.Token.Kind == TokenKind.Symbol)
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, false);
else
exp = parser.ParseIdExpression(null, null, false);
args.Add(exp);
}
// CASE 3: Normal param
else
{
// Can not have normal parameters after named parameters.
if (totalNamedParams > 0)
throw tokenIt.BuildSyntaxException("Un-named parameters must come before named parameters");
exp = parser.ParseExpression(endTokens, true, false, true, passNewLine, true);
args.Add(exp);
}
totalParameters++;
parser.Context.Limits.CheckParserFunctionParams(exp, totalParameters);
// Check for end of statment or invalid end of script.
if (parser.IsEndOfParameterList(Tokens.RightParenthesis, enableNewLineAsEnd))
break;
// Advance if not using fluent-parameters
if(meta == null)
tokenIt.Expect(Tokens.Comma);
}
// END with check for ")"
if (expectParenthesis) tokenIt.Expect(Tokens.RightParenthesis);
}
/* Function: BuildCurrentClass
*/
protected void BuildCurrentClass()
{
htmlOutput.Append("<div class=\"CPEntry T" + EngineInstance.CommentTypes.FromID(topic.CommentTypeID).SimpleIdentifier + " Current\">");
// Pre-prototype lines
int lineCount = parsedPrototype.NumberOfPrePrototypeLines;
TokenIterator start, end;
for (int i = 0; i < lineCount; i++)
{
parsedPrototype.GetPrePrototypeLine(i, out start, out end);
htmlOutput.Append("<div class=\"CPPrePrototypeLine\">");
BuildSyntaxHighlightedText(start, end);
htmlOutput.Append("</div>");
}
// Keyword and modifiers. We only show the keyword if it's not "class".
TokenIterator startKeyword, endKeyword;
topic.ParsedClassPrototype.GetKeyword(out startKeyword, out endKeyword);
string keyword = startKeyword.String;
TokenIterator startModifiers, endModifiers;
bool hasModifiers = topic.ParsedClassPrototype.GetModifiers(out startModifiers, out endModifiers);
if (hasModifiers || keyword != "class")
{
StringBuilder modifiersOutput = new StringBuilder();
TokenIterator partial;
bool hasPartial = startModifiers.Tokenizer.FindTokenBetween("partial", EngineInstance.Languages.FromID(topic.LanguageID).CaseSensitive,
startModifiers, endModifiers, out partial);
if (hasPartial)
{
TokenIterator lookahead = partial;
lookahead.Next();
if (lookahead < endModifiers &&
(lookahead.FundamentalType == FundamentalType.Text ||
lookahead.Character == '_'))
{
hasPartial = false;
}
TokenIterator lookbehind = partial;
lookbehind.Previous();
if (lookbehind >= startModifiers &&
(lookbehind.FundamentalType == FundamentalType.Text ||
lookbehind.Character == '_'))
{
hasPartial = false;
}
}
if (hasModifiers && hasPartial)
{
if (partial > startModifiers)
{
TokenIterator lookbehind = partial;
lookbehind.PreviousPastWhitespace(PreviousPastWhitespaceMode.EndingBounds);
BuildSyntaxHighlightedText(startModifiers, lookbehind, modifiersOutput);
}
partial.Next();
partial.NextPastWhitespace();
if (partial < endModifiers)
{
if (modifiersOutput.Length > 0)
{
modifiersOutput.Append(' ');
}
BuildSyntaxHighlightedText(partial, endModifiers, modifiersOutput);
}
}
else if (hasModifiers)
{
BuildSyntaxHighlightedText(startModifiers, endModifiers, modifiersOutput);
}
if (keyword != "class")
{
if (modifiersOutput.Length > 0)
{
modifiersOutput.Append(' ');
}
BuildSyntaxHighlightedText(startKeyword, endKeyword, modifiersOutput);
}
if (modifiersOutput.Length > 0)
{
htmlOutput.Append("<div class=\"CPModifiers\">");
htmlOutput.Append(modifiersOutput.ToString());
htmlOutput.Append("</div>");
}
}
// Name. We use the fully resolved name in the symbol instead of the prototype name, which may just be the last segment.
htmlOutput.Append("<div class=\"CPName\">");
BuildWrappedTitle(topic.Symbol.FormatWithSeparator(this.language.MemberOperator), topic.CommentTypeID, htmlOutput);
TokenIterator startTemplate, endTemplate;
if (topic.ParsedClassPrototype.GetTemplateSuffix(out startTemplate, out endTemplate))
{
// Include a zero-width space for wrapping
htmlOutput.Append("​<span class=\"TemplateSignature\">");
htmlOutput.EntityEncodeAndAppend(startTemplate.Tokenizer.TextBetween(startTemplate, endTemplate));
htmlOutput.Append("</span>");
}
htmlOutput.Append("</div>");
// Post-prototype lines
lineCount = parsedPrototype.NumberOfPostPrototypeLines;
for (int i = 0; i < lineCount; i++)
{
parsedPrototype.GetPostPrototypeLine(i, out start, out end);
htmlOutput.Append("<div class=\"CPPostPrototypeLine\">");
BuildSyntaxHighlightedText(start, end);
htmlOutput.Append("</div>");
}
htmlOutput.Append("</div>");
}
