/* Function: TryToGetLineComment
* If the iterator is on a line that starts with a line comment symbol, this function moves the iterator past the entire
* comment and returns true. If the comment is a candidate for documentation it will also return it as a
* <PossibleDocumentationComment>. If the line does not start with a line comment symbol it will return false and
* leave the iterator where it is.
*/
protected bool TryToGetLineComment(ref LineIterator lineIterator, string commentSymbol,
out PossibleDocumentationComment comment)
{
TokenIterator firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(commentSymbol) == false)
{
comment = null;
return(false);
}
comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
lineIterator.Next();
// Since we're definitely returning a comment we can mark the comment symbols as we go rather than waiting until
// the end.
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, commentSymbol.Length);
while (lineIterator.IsInBounds)
{
firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(commentSymbol) == false)
{
break;
}
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, commentSymbol.Length);
lineIterator.Next();
}
comment.End = lineIterator;
return(true);
}
/* Function: GetPossibleDocumentationComments
*
* Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
* comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
* comments it will return an empty list.
*
* All the comments in the returned list will have their comment symbols marked as <CommentParsingType.CommentSymbol>
* in the tokenizer. This allows further operations to be done on them in a language independent manner. If you want to also
* filter out text boxes and lines, use <Comments.LineFinder>.
*/
override public List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> possibleDocumentationComments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
while (lineIterator.IsInBounds)
{
PossibleDocumentationComment possibleDocumentationComment = null;
if (TryToGetBlockComment(ref lineIterator, out possibleDocumentationComment) ||
TryToGetLineComment(ref lineIterator, out possibleDocumentationComment))
{
if (possibleDocumentationComment != null)
{
// XML can actually use the Javadoc comment format in addition to its own.
if (possibleDocumentationComment.Javadoc == true)
{
possibleDocumentationComment.XML = true;
}
possibleDocumentationComments.Add(possibleDocumentationComment);
}
// lineIterator should already be moved
}
else
{
lineIterator.Next();
}
}
return(possibleDocumentationComments);
}
/* Function: GetPossibleDocumentationComments
*
* Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
* comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
* comments it will return an empty list.
*/
protected List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> possibleDocumentationComments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
while (lineIterator.IsInBounds)
{
bool foundComment = false;
PossibleDocumentationComment possibleDocumentationComment = null;
// Block comments
if (blockCommentStringPairs != null)
{
for (int i = 0; foundComment == false && i < blockCommentStringPairs.Length; i += 2)
{
foundComment = TryToGetBlockComment(ref lineIterator, blockCommentStringPairs[i], blockCommentStringPairs[i + 1],
out possibleDocumentationComment);
}
}
// Plain line comments
if (foundComment == false && lineCommentStrings != null)
{
for (int i = 0; foundComment == false && i < lineCommentStrings.Length; i++)
{
foundComment = TryToGetLineComment(ref lineIterator, lineCommentStrings[i], out possibleDocumentationComment);
}
}
// Nada.
if (foundComment == false)
{
lineIterator.Next();
}
else
{
if (possibleDocumentationComment != null)
{
possibleDocumentationComments.Add(possibleDocumentationComment);
}
// lineIterator would have been moved already if foundComment is true
}
}
return(possibleDocumentationComments);
}
/* Function: TryToGetPDLineComment
*
* If the line iterator is on a line comment, return it and all connected line comments as a
* <PossibleDocumentationComment> and mark the symbols as <CommentParsingType.CommentSymbol>. Returns
* null otherwise.
*
* This function takes a separate comment symbol for the first line and all remaining lines, allowing you to detect
* Javadoc line comments that start with ## and the remaining lines use #. Both symbols can be the same if this isn't
* required. If openingMustBeAlone is set, no symbol can appear immediately after the first line symbol for this
* function to succeed. This allows you to specifically detect something like ## without also matching #######.
*/
protected PossibleDocumentationComment TryToGetPDLineComment(LineIterator lineIterator,
string firstSymbol, string remainderSymbol,
bool openingMustBeAlone)
{
TokenIterator firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(firstSymbol) == false)
{
return(null);
}
if (openingMustBeAlone)
{
TokenIterator nextToken = firstToken;
nextToken.NextByCharacters(firstSymbol.Length);
if (nextToken.FundamentalType == FundamentalType.Symbol)
{
return(null);
}
}
PossibleDocumentationComment comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
lineIterator.Next();
// Since we're definitely returning a comment (barring the operation being cancelled) we can mark the comment
// symbols as we go rather than waiting until the end.
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, firstSymbol.Length);
while (lineIterator.IsInBounds)
{
firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(remainderSymbol) == false)
{
break;
}
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, remainderSymbol.Length);
lineIterator.Next();
}
comment.End = lineIterator;
return(comment);
}
/* Function: Parse
*
* Attempts to parse the passed comment into <Topics>. Returns whether it was successful, and if so, adds them
* to the list. These fields will be set:
*
* - CommentLineNumber
* - Body, if present
* - Summary, if available
*/
public bool Parse(PossibleDocumentationComment sourceComment, List <Topic> topics)
{
XMLIterator iterator = new XMLIterator(sourceComment.Start.FirstToken(Tokenization.LineBoundsMode.Everything),
sourceComment.End.FirstToken(Tokenization.LineBoundsMode.Everything));
while (iterator.Type == XMLElementType.Indent ||
iterator.Type == XMLElementType.LineBreak)
{
iterator.Next();
}
if (iterator.Type != XMLElementType.Tag)
{
return(false);
}
XMLComment xmlComment = new XMLComment();
while (iterator.IsInBounds)
{
if (TryToGetTopLevelTextBlock(ref iterator, xmlComment) ||
TryToGetTopLevelListItem(ref iterator, xmlComment))
{
}
else if (iterator.IsOnTag(TagForm.Opening))
{
SkipBlock(ref iterator);
}
else
{
iterator.Next();
}
}
Topic topic = GenerateTopic(xmlComment);
if (topic != null)
{
topic.CommentLineNumber = sourceComment.Start.LineNumber;
topics.Add(topic);
}
return(true);
}
/* Function: Parse
*
* Attempts to parse the passed comment into <Topics>. Returns whether it was successful, and if so, adds them
* to the list. These fields will be set:
*
* - CommentLineNumber
* - Body, if present
* - Summary, if available
*/
public bool Parse(PossibleDocumentationComment sourceComment, List <Topic> topics)
{
if (HasAnyTag(sourceComment.Start.FirstToken(LineBoundsMode.CommentContent),
sourceComment.End.FirstToken(LineBoundsMode.Everything)) == false)
{
return(false);
}
LineIterator firstBlockLine = sourceComment.Start;
while (firstBlockLine < sourceComment.End && IsFirstBlockLine(firstBlockLine) == false)
{
firstBlockLine.Next();
}
JavadocComment parsedComment = new JavadocComment();
if (sourceComment.Start < firstBlockLine)
{
parsedComment.Description = GetText(sourceComment.Start, firstBlockLine);
}
while (firstBlockLine < sourceComment.End)
{
// There may be unrecognized blocks so we have to manually advance if GetBlockTag() fails.
if (!TryToGetBlock(ref firstBlockLine, sourceComment.End, parsedComment))
{
firstBlockLine.Next();
}
}
Topic topic = GenerateTopic(parsedComment);
if (topic != null)
{
topic.CommentLineNumber = sourceComment.Start.LineNumber;
topics.Add(topic);
}
return(true);
}
// Group: Support Functions
// __________________________________________________________________________
/* Function: TryToGetBlockComment
*
* If the iterator is on a line that starts with the opening symbol of a block comment, this function moves the iterator
* past the entire comment and returns true. If the comment is a candidate for documentation it will also return it as
* a <PossibleDocumentationComment> and mark the symbols as <CommentParsingType.CommentSymbol>. If the
* line does not start with an opening comment symbol it will return false and leave the iterator where it is.
*
* Not all the block comments it finds will be candidates for documentation, since some will have text after the closing
* symbol, so it's possible for this function to return true and have comment be null. This is important because in Lua
* the block comment symbol is --[[ and the line comment symbol is --, so if we didn't move past the block comment
* it could be interpreted as a line comment as well.
*
* If openingMustBeAlone is set, that means no symbol can appear immediately after the opening symbol. If it does
* the function will return false and not move past the comment. This allows you to specifically detect something like
* /** without also matching /******.
*/
protected bool TryToGetBlockComment(ref LineIterator lineIterator,
string openingSymbol, string closingSymbol, bool openingMustBeAlone,
out PossibleDocumentationComment comment)
{
TokenIterator firstToken, endOfLine;
lineIterator.GetBounds(LineBoundsMode.ExcludeWhitespace, out firstToken, out endOfLine);
if (firstToken.MatchesAcrossTokens(openingSymbol) == false)
{
comment = null;
return(false);
}
// Advance past the opening symbol because it's possible for it to be the same as the closing one, such as with
// Python's ''' and """ strings.
firstToken.NextByCharacters(openingSymbol.Length);
if (openingMustBeAlone && firstToken.FundamentalType == FundamentalType.Symbol)
{
comment = null;
return(false);
}
comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
var tokenizer = lineIterator.Tokenizer;
var lookahead = lineIterator;
for (;;)
{
TokenIterator closingSymbolIterator;
if (tokenizer.FindTokensBetween(closingSymbol, false, firstToken, endOfLine, out closingSymbolIterator) == true)
{
// Move past the end of the comment regardless of whether it's acceptable for documentation or not
lookahead.Next();
// Make sure nothing appears after the closing symbol on the line
closingSymbolIterator.NextByCharacters(closingSymbol.Length);
closingSymbolIterator.NextPastWhitespace();
if (closingSymbolIterator.FundamentalType != FundamentalType.LineBreak &&
closingSymbolIterator.FundamentalType != FundamentalType.Null)
{
comment = null;
}
else
{
comment.End = lookahead;
}
break;
}
lookahead.Next();
// If we're not in bounds that means there was an unclosed comment at the end of the file. Skip it but don't treat
// it as a documentation candidate.
if (!lookahead.IsInBounds)
{
comment = null;
break;
}
lookahead.GetBounds(LineBoundsMode.ExcludeWhitespace, out firstToken, out endOfLine);
}
if (comment != null)
{
// Mark the symbols before returning
firstToken = comment.Start.FirstToken(LineBoundsMode.ExcludeWhitespace);
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, openingSymbol.Length);
LineIterator lastLine = lookahead;
lastLine.Previous();
lastLine.GetBounds(LineBoundsMode.ExcludeWhitespace, out firstToken, out endOfLine);
endOfLine.PreviousByCharacters(closingSymbol.Length);
endOfLine.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, closingSymbol.Length);
}
// If we made it this far that means we found a comment and can move the line iterator and return true. Whether
// that comment was suitable for documentation will be determined by the comment variable, but we are moving the
// iterator and returning true either way.
lineIterator = lookahead;
return(true);
}
// Function: GetPossibleDocumentationComments
//
// Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
// comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
// comments it will return an empty list.
//
// All the comments in the returned list will have their comment symbols marked as <CommentParsingType.CommentSymbol>
// in the tokenizer. This allows further operations to be done on them in a language independent manner. If you want to also
// filter out text boxes and lines, use <Comments.LineFinder>.
//
// Default Implementation:
//
// The default implementation uses the comment symbols found in <Language> or passed to the constructor. You can override
// this function if you need to do something more sophisticated, such as interpret the POD directives in Perl.
//
// Comments must be alone on a line to be a candidate for documentation, meaning that the comment symbol must be the
// first non-whitespace character on a line, and in the case of block comments, nothing but whitespace may trail the closing
// symbol. The latter rule is important because a comment may start correctly but not end so, as in this prototype with Splint
// annotation:
//
// > int get_array(integer_t id,
// > /*@out@*/ array_t array);
//
// Speaking of which, block comments surrounded by @ symbols are not included because they're Splint comments. Not
// including them in the possible documentation comments list means the Splint comment below won't end prototype detection.
//
// > void initialize ()
// > /*@globals undef globnum,
// > undef globname @*/
// > { ... }
//
// It also goes through the code line by line in a simple manner, not accounting for things like strings, so if a language contains
// a multiline string whose content looks like a language comment it will be interpreted as one. This isn't ideal but is accepted
// as a conscious tradeoff because there are actually many different string formats (literal quotes denoted with \", literal quotes
// denoted with "", Perl's q{} forms and here doc) so you can't account for them all in a generalized way. Also, putting this in
// an independent stage even when using full language support means comments don't disappear the way prototypes do if the
// parser gets tripped up on something like an unmatched brace.
//
virtual public List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> possibleDocumentationComments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
while (lineIterator.IsInBounds)
{
bool foundComment = false;
PossibleDocumentationComment possibleDocumentationComment = null;
// Javadoc block comments
// We test for these before regular block comments because they are usually extended versions of them, such
// as /** and /*.
if (javadocBlockCommentStringPairs != null)
{
for (int i = 0; foundComment == false && i < javadocBlockCommentStringPairs.Length; i += 2)
{
foundComment = TryToGetBlockComment(ref lineIterator,
javadocBlockCommentStringPairs[i], javadocBlockCommentStringPairs[i + 1], true,
out possibleDocumentationComment);
}
if (possibleDocumentationComment != null)
{
possibleDocumentationComment.Javadoc = true;
}
}
// Plain block comments
// We test block comments ahead of line comments because in Lua the line comments are a substring of them: --
// versus --[[ and ]]--.
if (foundComment == false && blockCommentStringPairs != null)
{
for (int i = 0; foundComment == false && i < blockCommentStringPairs.Length; i += 2)
{
foundComment = TryToGetBlockComment(ref lineIterator,
blockCommentStringPairs[i], blockCommentStringPairs[i + 1], false,
out possibleDocumentationComment);
}
// Skip Splint comments so that they can appear in prototypes.
if (possibleDocumentationComment != null &&
possibleDocumentationComment.Start.FirstToken(LineBoundsMode.CommentContent).Character == '@')
{
LineIterator lastLine = possibleDocumentationComment.End;
lastLine.Previous();
TokenIterator lastToken, ignore;
lastLine.GetBounds(LineBoundsMode.CommentContent, out ignore, out lastToken);
lastToken.Previous();
if (lastToken.Character == '@')
{
possibleDocumentationComment = null;
}
}
}
// XML line comments
if (foundComment == false && xmlLineCommentStrings != null)
{
for (int i = 0; foundComment == false && i < xmlLineCommentStrings.Length; i++)
{
foundComment = TryToGetLineComment(ref lineIterator,
xmlLineCommentStrings[i], xmlLineCommentStrings[i], true,
out possibleDocumentationComment);
}
if (possibleDocumentationComment != null)
{
possibleDocumentationComment.XML = true;
}
}
// Ambiguous XML/Javadoc line comments
// If an XML comment is found we check the same position for Javadoc because they may share an opening
// symbol, such as ///.
if (possibleDocumentationComment != null && possibleDocumentationComment.XML == true &&
javadocLineCommentStringPairs != null)
{
LineIterator javadocLineIterator = possibleDocumentationComment.Start;
PossibleDocumentationComment possibleJavadocDocumentationComment = null;
bool foundJavadocComment = false;
for (int i = 0; foundJavadocComment == false && i < javadocLineCommentStringPairs.Length; i += 2)
{
foundJavadocComment = TryToGetLineComment(ref javadocLineIterator,
javadocLineCommentStringPairs[i], javadocLineCommentStringPairs[i + 1], true,
out possibleJavadocDocumentationComment);
}
if (possibleJavadocDocumentationComment != null)
{
// If the Javadoc comment is longer we use that instead of the XML since it may have detected the first
// line as XML and ignored the rest for not having the same symbol. For example:
//
// ## Comment
// #
// #
//
// This will be detected as a one line XML comment and a three line Javadoc comment.
if (possibleJavadocDocumentationComment.End.LineNumber >
possibleDocumentationComment.End.LineNumber)
{
possibleDocumentationComment = possibleJavadocDocumentationComment;
possibleDocumentationComment.Javadoc = true;
lineIterator = javadocLineIterator;
}
// If they're the same length...
else if (possibleJavadocDocumentationComment.End.LineNumber ==
possibleDocumentationComment.End.LineNumber)
{
// If the comments are both one line long then it's genuinely ambiguous. For example:
//
// ## Comment
//
// Is that a one line XML comment or a one line Javadoc comment? We can't tell, so mark it as
// potentially either.
if (possibleDocumentationComment.Start.LineNumber ==
possibleDocumentationComment.End.LineNumber - 1)
{
possibleDocumentationComment.Javadoc = true;
// XML should already be set to true
}
// If the comments are equal length but more than one line then it's just interpreting the XML as
// a Javadoc start with a vertical line for the remainder, so leave it as XML. For example:
//
// ## Comment
// ##
// ##
//
// That's clearly a three line XML comment and not a Javadoc comment with a vertical line.
}
// If the XML comment is longer just leave it and ignore the Javadoc one.
}
}
// Javadoc line comments
if (foundComment == false && javadocLineCommentStringPairs != null)
{
for (int i = 0; foundComment == false && i < javadocLineCommentStringPairs.Length; i += 2)
{
foundComment = TryToGetLineComment(ref lineIterator,
javadocLineCommentStringPairs[i], javadocLineCommentStringPairs[i + 1], true,
out possibleDocumentationComment);
}
if (possibleDocumentationComment != null)
{
possibleDocumentationComment.Javadoc = true;
}
}
// Plain line comments
if (foundComment == false && lineCommentStrings != null)
{
for (int i = 0; foundComment == false && i < lineCommentStrings.Length; i++)
{
foundComment = TryToGetLineComment(ref lineIterator,
lineCommentStrings[i], lineCommentStrings[i], false,
out possibleDocumentationComment);
}
}
// Nada.
if (foundComment == false)
{
lineIterator.Next();
}
else
{
if (possibleDocumentationComment != null)
{
// XML can actually use the Javadoc comment format in addition to its own.
if (possibleDocumentationComment.Javadoc == true)
{
possibleDocumentationComment.XML = true;
}
possibleDocumentationComments.Add(possibleDocumentationComment);
}
// lineIterator would have been moved already if foundComment is true
}
}
return(possibleDocumentationComments);
}
// Function: GetPossibleDocumentationComments
//
// Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
// comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
// comments it will return an empty list.
//
// All the comments in the returned list will have their comment symbols marked as <CommentParsingType.CommentSymbol>
// in the tokenizer. This allows further operations to be done on them in a language independent manner. If you want to also
// filter out text boxes and lines, use <Comments.LineFinder>.
//
override public List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> comments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
PODLineType podLineType;
while (lineIterator.IsInBounds)
{
TokenIterator tokenIterator = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
// Hash comments
if (tokenIterator.Character == '#')
{
PossibleDocumentationComment comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
// First line
if (tokenIterator.MatchesAcrossTokens("###"))
{
comment.Javadoc = false;
comment.XML = true;
}
else if (tokenIterator.MatchesAcrossTokens("##"))
{
comment.Javadoc = true;
comment.XML = false;
}
else // just "#"
{
comment.Javadoc = false;
comment.XML = false;
}
lineIterator.Next();
// Subsequent lines
while (lineIterator.IsInBounds)
{
tokenIterator = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (tokenIterator.Character != '#')
{
break;
}
if (tokenIterator.MatchesAcrossTokens("###"))
{
comment.Javadoc = false;
// XML is still possible
}
else if (tokenIterator.MatchesAcrossTokens("##"))
{
comment.Javadoc = false;
comment.XML = false;
}
else // just "#"
{
// Javadoc is still possible
comment.XML = false;
}
lineIterator.Next();
}
comment.End = lineIterator;
comments.Add(comment);
// Go back and mark the tokens
int firstLineCount, subsequentLineCount;
if (comment.XML)
{
firstLineCount = 3;
subsequentLineCount = 3;
}
else if (comment.Javadoc)
{
firstLineCount = 2;
subsequentLineCount = 1;
}
else // plain
{
firstLineCount = 1;
subsequentLineCount = 1;
}
LineIterator temp = comment.Start;
tokenIterator = temp.FirstToken(LineBoundsMode.ExcludeWhitespace);
tokenIterator.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, firstLineCount);
temp.Next();
while (temp < comment.End)
{
tokenIterator = temp.FirstToken(LineBoundsMode.ExcludeWhitespace);
tokenIterator.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, subsequentLineCount);
temp.Next();
}
// XML can actually appear in Javadoc comments
if (comment.Javadoc)
{
comment.XML = true;
}
}
// POD comments
else if (TryToSkipPODLine(ref tokenIterator, out podLineType))
{
TokenIterator podLineStart, podLineEnd;
lineIterator.GetBounds(LineBoundsMode.CommentContent, out podLineStart, out podLineEnd);
lineIterator.Tokenizer.SetCommentParsingTypeBetween(podLineStart, podLineEnd, CommentParsingType.CommentSymbol);
if (podLineType == PODLineType.StartNaturalDocs ||
podLineType == PODLineType.StartJavadoc)
{
PossibleDocumentationComment comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
if (podLineType == PODLineType.StartJavadoc)
{
comment.Javadoc = true;
}
for (;;)
{
lineIterator.Next();
if (lineIterator.IsInBounds == false)
{
break;
}
tokenIterator = lineIterator.FirstToken(LineBoundsMode.CommentContent);
if (TryToSkipPODLine(ref tokenIterator, out podLineType) == true)
{
if (podLineType == PODLineType.End)
{
lineIterator.GetBounds(LineBoundsMode.CommentContent, out podLineStart, out podLineEnd);
lineIterator.Tokenizer.SetCommentParsingTypeBetween(podLineStart, podLineEnd, CommentParsingType.CommentSymbol);
lineIterator.Next();
}
break;
}
}
comment.End = lineIterator;
comments.Add(comment);
}
else
{
lineIterator.Next();
}
}
else
{
lineIterator.Next();
}
}
return(comments);
}
/* Function: FindIncludeInOutput
* Extracts and returns any comment content marked "include in output". All comment symbols and extra indents
* will be removed. Returns null if the comment doesn't have any.
*/
protected string FindIncludeInOutput(PossibleDocumentationComment comment)
{
Comments.LineFinder.MarkTextBoxes(comment);
LineIterator iterator = comment.Start;
// Find the "include in output" header if it exists
while (iterator < comment.End &&
iterator.Match(IncludeInOutputRegex, LineBoundsMode.CommentContent).Success == false)
{
iterator.Next();
}
if (iterator >= comment.End)
{
return(null);
}
// Find the bounds of the content excluding whitespace and the shared indent level
iterator.Next();
while (iterator < comment.End && iterator.IsEmpty(LineBoundsMode.CommentContent))
{
iterator.Next();
}
LineIterator start = iterator;
LineIterator end = iterator;
int commonIndent = 9999;
while (iterator < comment.End)
{
if (iterator.IsEmpty(LineBoundsMode.CommentContent))
{
iterator.Next();
}
else
{
if (iterator.Indent(LineBoundsMode.CommentContent) < commonIndent)
{
commonIndent = iterator.Indent(LineBoundsMode.CommentContent);
}
iterator.Next();
end = iterator;
}
}
// Build and return the comment content
if (start >= end)
{
return(null);
}
StringBuilder output = new StringBuilder();
do
{
int indentDifference = start.Indent(LineBoundsMode.CommentContent) - commonIndent;
if (indentDifference > 0)
{
output.Append(' ', indentDifference);
}
start.AppendTo(output, LineBoundsMode.CommentContent);
output.AppendLine();
start.Next();
}while (start < end);
return(output.ToString());
}
public override string OutputOf(IList <PossibleDocumentationComment> comments, IList <Topic> topics)
{
if (comments == null || comments.Count == 0)
{
return("(No comments found)");
}
int commentIndex = 0;
int topicIndex = 0;
StringBuilder output = new StringBuilder();
while (commentIndex < comments.Count)
{
PossibleDocumentationComment comment = comments[commentIndex];
output.Append("- ");
if (comment.Javadoc && comment.XML)
{
output.Append("Javadoc+XML");
}
else if (comment.Javadoc)
{
output.Append("Javadoc");
}
else if (comment.XML)
{
output.Append("XML");
}
else
{
output.Append("Plain");
}
output.Append(" comment, ");
if (comment.End.LineNumber == comment.Start.LineNumber + 1)
{
output.AppendLine("line " + comment.Start.LineNumber + ':');
}
else
{
output.AppendLine("lines " + comment.Start.LineNumber + '-' + (comment.End.LineNumber - 1) + ':');
}
output.AppendLine();
if (topics != null && topicIndex < topics.Count && topics[topicIndex].CommentLineNumber < comment.End.LineNumber)
{
do
{
Topic topic = topics[topicIndex];
output.AppendLine(" - Topic \"" + topic.Title + "\", line " + topic.CommentLineNumber + ':');
if (topic.Body != null)
{
output.AppendLine(" - " + topic.Body);
}
else
{
output.AppendLine(" - (No body detected)");
}
output.AppendLine();
topicIndex++;
}while (topicIndex < topics.Count && topics[topicIndex].CommentLineNumber < comment.End.LineNumber);
}
else
{
output.AppendLine(" - (No topics detected)");
output.AppendLine();
}
commentIndex++;
}
return(output.ToString());
}
// Group: Support Functions
// __________________________________________________________________________
/* Function: TryToGetPDBlockComment
*
* If the line iterator is on the starting symbol of a block comment, return it as a <PossibleDocumentationComment>
* and mark the symbols as <CommentParsingType.CommentSymbol>. If the iterator is not on the opening comment
* symbol or there is content after the closing comment symbol making it unsuitable as a documentation comment,
* returns null.
*
* If openingMustBeAlone is set, that means no symbol can appear immediately after the opening symbol for this
* function to succeed. This allows you to specifically detect something like /** without also matching /******.
*/
protected PossibleDocumentationComment TryToGetPDBlockComment(LineIterator lineIterator,
string openingSymbol, string closingSymbol,
bool openingMustBeAlone)
{
TokenIterator firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(openingSymbol) == false)
{
return(null);
}
if (openingMustBeAlone)
{
TokenIterator nextToken = firstToken;
nextToken.NextByCharacters(openingSymbol.Length);
if (nextToken.FundamentalType == FundamentalType.Symbol)
{
return(null);
}
}
PossibleDocumentationComment comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
for (;;)
{
if (!lineIterator.IsInBounds)
{
return(null);
}
TokenIterator closingSymbolIterator;
if (lineIterator.FindAcrossTokens(closingSymbol, false, LineBoundsMode.Everything, out closingSymbolIterator) == true)
{
closingSymbolIterator.NextByCharacters(closingSymbol.Length);
closingSymbolIterator.NextPastWhitespace();
if (closingSymbolIterator.FundamentalType != FundamentalType.LineBreak &&
closingSymbolIterator.FundamentalType != FundamentalType.Null)
{
return(null);
}
lineIterator.Next();
comment.End = lineIterator;
break;
}
lineIterator.Next();
}
// Success. Mark the symbols before returning.
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, openingSymbol.Length);
TokenIterator lastToken;
lineIterator.Previous();
lineIterator.GetBounds(LineBoundsMode.ExcludeWhitespace, out firstToken, out lastToken);
lastToken.PreviousByCharacters(closingSymbol.Length);
lastToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, closingSymbol.Length);
return(comment);
}
// Function: GetPossibleDocumentationComments
//
// Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
// comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
// comments it will return an empty list.
//
// All the comments in the returned list will have their comment symbols marked as <CommentParsingType.CommentSymbol>
// in the tokenizer. This allows further operations to be done on them in a language independent manner. If you want to also
// filter out text boxes and lines, use <Comments.LineFinder>.
//
// Default Implementation:
//
// The default implementation uses the comment symbols found in <Language> or passed to the constructor. You can override
// this function if you need to do something more sophisticated, such as interpret the POD directives in Perl.
//
// Comments must be alone on a line to be a candidate for documentation, meaning that the comment symbol must be the
// first non-whitespace character on a line, and in the case of block comments, nothing but whitespace may trail the closing
// symbol. The latter rule is important because a comment may start correctly but not end so, as in this prototype with Splint
// annotation:
//
// > int get_array(integer_t id,
// > /*@out@*/ array_t array);
//
// Speaking of which, block comments surrounded by @ symbols are not included because they're Splint comments. Not
// including them in the possible documentation comments list means the Splint comment below won't end prototype detection.
//
// > void initialize ()
// > /*@globals undef globnum,
// > undef globname @*/
// > { ... }
//
// It also goes through the code line by line in a simple manner, not accounting for things like strings, so if a language contains
// a multiline string whose content looks like a language comment it will be interpreted as one. This isn't ideal but is accepted
// as a conscious tradeoff because there are actually many different string formats (literal quotes denoted with \", literal quotes
// denoted with "", Perl's q{} forms and here doc) so you can't account for them all in a generalized way. Also, putting this in
// an independent stage even when using full language support means comments don't disappear the way prototypes do if the
// parser gets tripped up on something like an unmatched brace.
//
virtual public List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> possibleDocumentationComments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
while (lineIterator.IsInBounds)
{
PossibleDocumentationComment comment = null;
// Javadoc block comments
// We test for these before regular block comments because they are usually extended versions of them, such
// as /** and /*.
// We also test block comments in general ahead of line comments because in Lua the line comments are a
// substring of them: -- versus --[[ and ]]--.
if (javadocBlockCommentStringPairs != null)
{
for (int i = 0; comment == null && i < javadocBlockCommentStringPairs.Length; i += 2)
{
comment = TryToGetPDBlockComment(lineIterator, javadocBlockCommentStringPairs[i],
javadocBlockCommentStringPairs[i + 1], true);
}
if (comment != null)
{
comment.Javadoc = true;
}
}
// Plain block comments
if (comment == null && blockCommentStringPairs != null)
{
for (int i = 0; comment == null && i < blockCommentStringPairs.Length; i += 2)
{
comment = TryToGetPDBlockComment(lineIterator, blockCommentStringPairs[i],
blockCommentStringPairs[i + 1], false);
}
// Skip Splint comments so that they can appear in prototypes.
if (comment != null && comment.Start.FirstToken(LineBoundsMode.CommentContent).Character == '@')
{
LineIterator lastLine = comment.End;
lastLine.Previous();
TokenIterator lastToken, ignore;
lastLine.GetBounds(LineBoundsMode.CommentContent, out ignore, out lastToken);
lastToken.Previous();
if (lastToken.Character == '@')
{
comment = null;
}
}
}
// XML line comments
if (comment == null && xmlLineCommentStrings != null)
{
for (int i = 0; comment == null && i < xmlLineCommentStrings.Length; i++)
{
comment = TryToGetPDLineComment(lineIterator, xmlLineCommentStrings[i],
xmlLineCommentStrings[i], true);
}
if (comment != null)
{
comment.XML = true;
}
}
// Javadoc line comments
// We check for these even if a XML comment is found because they may share an opening symbol, such as ///.
// We change it to Javadoc if it's longer. If it's equal it's just interpreting the XML as a Javadoc start with a
// vertical line for the remainder, so leave it as XML. Unless the comment is only one line long, in which case it's
// genuinely ambiguous.
if ((comment == null || comment.XML == true) && javadocLineCommentStringPairs != null)
{
PossibleDocumentationComment javadocComment = null;
for (int i = 0; javadocComment == null && i < javadocLineCommentStringPairs.Length; i += 2)
{
javadocComment = TryToGetPDLineComment(lineIterator, javadocLineCommentStringPairs[i],
javadocLineCommentStringPairs[i + 1], true);
}
if (javadocComment != null)
{
javadocComment.Javadoc = true;
if (comment == null)
{
comment = javadocComment;
}
else
{
int javadocLength = javadocComment.End.LineNumber - javadocComment.Start.LineNumber;
int xmlLength = comment.End.LineNumber - comment.Start.LineNumber;
if (javadocLength > xmlLength)
{
comment = javadocComment;
}
else if (javadocLength == 1 && xmlLength == 1)
{
comment.Javadoc = true;
}
// else stay with the XML comment
}
}
}
// Plain line comments
if (comment == null && lineCommentStrings != null)
{
for (int i = 0; comment == null && i < lineCommentStrings.Length; i++)
{
comment = TryToGetPDLineComment(lineIterator, lineCommentStrings[i], lineCommentStrings[i], false);
}
}
// Nada.
if (comment == null)
{
lineIterator.Next();
}
else
{
// XML can actually use the Javadoc comment format in addition to its own.
if (comment.Javadoc)
{
comment.XML = true;
}
possibleDocumentationComments.Add(comment);
lineIterator = comment.End;
}
}
return(possibleDocumentationComments);
}
/* Function: TryToGetBlockComment
*
* If the iterator is on a line that starts with the opening symbol of a block comment, this function moves the iterator
* past the entire comment and returns true. If the comment is a candidate for documentation it will also return it as
* a <PossibleDocumentationComment> and mark the symbols as <CommentParsingType.CommentSymbol>. If the
* line does not start with an opening comment symbol it will return false and leave the iterator where it is.
*/
protected bool TryToGetBlockComment(ref LineIterator lineIterator, out PossibleDocumentationComment comment)
{
TokenIterator firstToken, endOfLine;
lineIterator.GetBounds(LineBoundsMode.ExcludeWhitespace, out firstToken, out endOfLine);
// Are we on a block comment?
TokenIterator lookahead = firstToken;
string closingSymbol;
if (TryToSkipOpeningBlockCommentSymbol(ref lookahead, out closingSymbol) == false)
{
comment = null;
return(false);
}
// We are. Create a possible documentation comment.
comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
// Check if we're on a Javadoc comment, which will be an extra [.
if (lookahead.Character == '[')
{
lookahead.Next();
if (lookahead.FundamentalType != FundamentalType.Symbol)
{
comment.Javadoc = true;
}
}
// Find the end of the comment, which could be on the same line as the start.
var tokenizer = lineIterator.Tokenizer;
var lineLookahead = lineIterator;
bool hadTrailingDashes = false;
for (;;)
{
TokenIterator closingSymbolIterator;
if (tokenizer.FindTokensBetween(closingSymbol, false, firstToken, endOfLine, out closingSymbolIterator) == true)
{
// Move past the end of the comment regardless of whether it's acceptable for documentation or not
lineLookahead.Next();
// Make sure nothing appears after the closing symbol on the line
closingSymbolIterator.NextByCharacters(closingSymbol.Length);
// We'll allow -- though since some people use --[[ and ]]-- for balance even though the latter is actually the
// closing comment symbol followed by a line comment.
if (closingSymbolIterator.MatchesAcrossTokens("--"))
{
hadTrailingDashes = true;
closingSymbolIterator.Next(2);
}
closingSymbolIterator.NextPastWhitespace();
if (closingSymbolIterator.FundamentalType != FundamentalType.LineBreak &&
closingSymbolIterator.FundamentalType != FundamentalType.Null)
{
comment = null;
}
else
{
comment.End = lineLookahead;
}
break;
}
lineLookahead.Next();
// If we're not in bounds that means there was an unclosed comment at the end of the file. Skip it but don't treat
// it as a documentation candidate.
if (!lookahead.IsInBounds)
{
comment = null;
break;
}
lineLookahead.GetBounds(LineBoundsMode.ExcludeWhitespace, out firstToken, out endOfLine);
}
if (comment != null)
{
// Mark the symbols before returning
firstToken = comment.Start.FirstToken(LineBoundsMode.ExcludeWhitespace);
lookahead = firstToken;
TryToSkipOpeningBlockCommentSymbol(ref lookahead, out closingSymbol);
if (comment.Javadoc)
{
lookahead.Next();
}
firstToken.SetCommentParsingTypeBetween(lookahead, CommentParsingType.CommentSymbol);
LineIterator lastLine = comment.End;
lastLine.Previous();
lastLine.GetBounds(LineBoundsMode.ExcludeWhitespace, out firstToken, out endOfLine);
lookahead = endOfLine;
if (hadTrailingDashes)
{
lookahead.Previous(2);
}
lookahead.PreviousByCharacters(closingSymbol.Length);
lookahead.SetCommentParsingTypeBetween(endOfLine, CommentParsingType.CommentSymbol);
}
// If we made it this far that means we found a comment and can move the line iterator and return true. Whether
// that comment was suitable for documentation will be determined by the comment variable, but we are moving the
// iterator and returning true either way.
lineIterator = lineLookahead;
return(true);
}
/* Function: TryToGetLineComment
*
* If the iterator is on a line that starts with a line comment symbol, this function moves the iterator past the entire
* comment and returns true. If the comment is a candidate for documentation it will also return it as a
* <PossibleDocumentationComment> and mark the symbols as <CommentParsingType.CommentSymbol>. If the
* line does not start with a line comment symbol it will return false and leave the iterator where it is.
*/
protected bool TryToGetLineComment(ref LineIterator lineIterator, out PossibleDocumentationComment comment)
{
TokenIterator firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
TokenIterator lookahead = firstToken;
// Are we on a line comment?
if (TryToSkipLineCommentSymbol(ref lookahead) == false)
{
comment = null;
return(false);
}
// We are. Create a possible documentation comment.
comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
// Check if we're a Javadoc/XML comment. We can't tell the difference from just the first line.
TokenIterator endOfCommentSymbol = lookahead;
if (lookahead.Character == '-')
{
lookahead.Next();
if (lookahead.FundamentalType != FundamentalType.Symbol)
{
endOfCommentSymbol = lookahead;
comment.Javadoc = true;
comment.XML = true;
}
}
// Mark it.
firstToken.SetCommentParsingTypeBetween(endOfCommentSymbol, CommentParsingType.CommentSymbol);
// Continue to find the rest of the comment
lineIterator.Next();
bool hasXMLishLines = false;
bool hasNonXMLishLines = false;
bool hasMultipleLines = false;
while (lineIterator.IsInBounds)
{
firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
lookahead = firstToken;
if (TryToSkipLineCommentSymbol(ref lookahead) == false)
{
break;
}
hasMultipleLines = true;
endOfCommentSymbol = lookahead;
if (lookahead.Character == '-')
{
lookahead.Next();
if (lookahead.FundamentalType != FundamentalType.Symbol)
{
hasXMLishLines = true;
endOfCommentSymbol = lookahead;
}
else
{
hasNonXMLishLines = true;
}
}
else
{
hasNonXMLishLines = true;
}
firstToken.SetCommentParsingTypeBetween(endOfCommentSymbol, CommentParsingType.CommentSymbol);
lineIterator.Next();
}
comment.End = lineIterator;
if (hasMultipleLines && comment.Javadoc)
{
if (hasXMLishLines && !hasNonXMLishLines)
{
comment.Javadoc = false;
}
else if (hasNonXMLishLines)
{
comment.XML = false;
}
}
return(true);
}