/* Function: TryToGetLineComment
* If the iterator is on a line that starts with a line comment symbol, this function moves the iterator past the entire
* comment and returns true. If the comment is a candidate for documentation it will also return it as a
* <PossibleDocumentationComment>. If the line does not start with a line comment symbol it will return false and
* leave the iterator where it is.
*/
protected bool TryToGetLineComment(ref LineIterator lineIterator, string commentSymbol,
out PossibleDocumentationComment comment)
{
TokenIterator firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(commentSymbol) == false)
{
comment = null;
return(false);
}
comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
lineIterator.Next();
// Since we're definitely returning a comment we can mark the comment symbols as we go rather than waiting until
// the end.
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, commentSymbol.Length);
while (lineIterator.IsInBounds)
{
firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(commentSymbol) == false)
{
break;
}
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, commentSymbol.Length);
lineIterator.Next();
}
comment.End = lineIterator;
return(true);
}
/* Function: GetPossibleDocumentationComments
*
* Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
* comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
* comments it will return an empty list.
*
* All the comments in the returned list will have their comment symbols marked as <CommentParsingType.CommentSymbol>
* in the tokenizer. This allows further operations to be done on them in a language independent manner. If you want to also
* filter out text boxes and lines, use <Comments.LineFinder>.
*/
override public List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> possibleDocumentationComments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
while (lineIterator.IsInBounds)
{
PossibleDocumentationComment possibleDocumentationComment = null;
if (TryToGetBlockComment(ref lineIterator, out possibleDocumentationComment) ||
TryToGetLineComment(ref lineIterator, out possibleDocumentationComment))
{
if (possibleDocumentationComment != null)
{
// XML can actually use the Javadoc comment format in addition to its own.
if (possibleDocumentationComment.Javadoc == true)
{
possibleDocumentationComment.XML = true;
}
possibleDocumentationComments.Add(possibleDocumentationComment);
}
// lineIterator should already be moved
}
else
{
lineIterator.Next();
}
}
return(possibleDocumentationComments);
}
/* Function: TryToGetPDLineComment
*
* If the line iterator is on a line comment, return it and all connected line comments as a
* <PossibleDocumentationComment> and mark the symbols as <CommentParsingType.CommentSymbol>. Returns
* null otherwise.
*
* This function takes a separate comment symbol for the first line and all remaining lines, allowing you to detect
* Javadoc line comments that start with ## and the remaining lines use #. Both symbols can be the same if this isn't
* required. If openingMustBeAlone is set, no symbol can appear immediately after the first line symbol for this
* function to succeed. This allows you to specifically detect something like ## without also matching #######.
*/
protected PossibleDocumentationComment TryToGetPDLineComment(LineIterator lineIterator,
string firstSymbol, string remainderSymbol,
bool openingMustBeAlone)
{
TokenIterator firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(firstSymbol) == false)
{
return(null);
}
if (openingMustBeAlone)
{
TokenIterator nextToken = firstToken;
nextToken.NextByCharacters(firstSymbol.Length);
if (nextToken.FundamentalType == FundamentalType.Symbol)
{
return(null);
}
}
PossibleDocumentationComment comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
lineIterator.Next();
// Since we're definitely returning a comment (barring the operation being cancelled) we can mark the comment
// symbols as we go rather than waiting until the end.
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, firstSymbol.Length);
while (lineIterator.IsInBounds)
{
firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(remainderSymbol) == false)
{
break;
}
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, remainderSymbol.Length);
lineIterator.Next();
}
comment.End = lineIterator;
return(comment);
}
/* Function: Parse
*
* Attempts to parse the passed comment into <Topics>. Returns whether it was successful, and if so, adds them
* to the list. These fields will be set:
*
* - CommentLineNumber
* - Body, if present
* - Summary, if available
*/
public bool Parse(PossibleDocumentationComment sourceComment, List <Topic> topics)
{
if (HasAnyTag(sourceComment.Start.FirstToken(LineBoundsMode.CommentContent),
sourceComment.End.FirstToken(LineBoundsMode.Everything)) == false)
{
return(false);
}
LineIterator firstBlockLine = sourceComment.Start;
while (firstBlockLine < sourceComment.End && IsFirstBlockLine(firstBlockLine) == false)
{
firstBlockLine.Next();
}
JavadocComment parsedComment = new JavadocComment();
if (sourceComment.Start < firstBlockLine)
{
parsedComment.Description = GetText(sourceComment.Start, firstBlockLine);
}
while (firstBlockLine < sourceComment.End)
{
// There may be unrecognized blocks so we have to manually advance if GetBlockTag() fails.
if (!TryToGetBlock(ref firstBlockLine, sourceComment.End, parsedComment))
{
firstBlockLine.Next();
}
}
Topic topic = GenerateTopic(parsedComment);
if (topic != null)
{
topic.CommentLineNumber = sourceComment.Start.LineNumber;
topics.Add(topic);
}
return(true);
}
/* Function: GetPossibleDocumentationComments
*
* Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
* comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
* comments it will return an empty list.
*/
protected List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> possibleDocumentationComments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
while (lineIterator.IsInBounds)
{
bool foundComment = false;
PossibleDocumentationComment possibleDocumentationComment = null;
// Block comments
if (blockCommentStringPairs != null)
{
for (int i = 0; foundComment == false && i < blockCommentStringPairs.Length; i += 2)
{
foundComment = TryToGetBlockComment(ref lineIterator, blockCommentStringPairs[i], blockCommentStringPairs[i + 1],
out possibleDocumentationComment);
}
}
// Plain line comments
if (foundComment == false && lineCommentStrings != null)
{
for (int i = 0; foundComment == false && i < lineCommentStrings.Length; i++)
{
foundComment = TryToGetLineComment(ref lineIterator, lineCommentStrings[i], out possibleDocumentationComment);
}
}
// Nada.
if (foundComment == false)
{
lineIterator.Next();
}
else
{
if (possibleDocumentationComment != null)
{
possibleDocumentationComments.Add(possibleDocumentationComment);
}
// lineIterator would have been moved already if foundComment is true
}
}
return(possibleDocumentationComments);
}
// Function: GetPossibleDocumentationComments
//
// Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
// comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
// comments it will return an empty list.
//
// All the comments in the returned list will have their comment symbols marked as <CommentParsingType.CommentSymbol>
// in the tokenizer. This allows further operations to be done on them in a language independent manner. If you want to also
// filter out text boxes and lines, use <Comments.LineFinder>.
//
// Default Implementation:
//
// The default implementation uses the comment symbols found in <Language> or passed to the constructor. You can override
// this function if you need to do something more sophisticated, such as interpret the POD directives in Perl.
//
// Comments must be alone on a line to be a candidate for documentation, meaning that the comment symbol must be the
// first non-whitespace character on a line, and in the case of block comments, nothing but whitespace may trail the closing
// symbol. The latter rule is important because a comment may start correctly but not end so, as in this prototype with Splint
// annotation:
//
// > int get_array(integer_t id,
// > /*@out@*/ array_t array);
//
// Speaking of which, block comments surrounded by @ symbols are not included because they're Splint comments. Not
// including them in the possible documentation comments list means the Splint comment below won't end prototype detection.
//
// > void initialize ()
// > /*@globals undef globnum,
// > undef globname @*/
// > { ... }
//
// It also goes through the code line by line in a simple manner, not accounting for things like strings, so if a language contains
// a multiline string whose content looks like a language comment it will be interpreted as one. This isn't ideal but is accepted
// as a conscious tradeoff because there are actually many different string formats (literal quotes denoted with \", literal quotes
// denoted with "", Perl's q{} forms and here doc) so you can't account for them all in a generalized way. Also, putting this in
// an independent stage even when using full language support means comments don't disappear the way prototypes do if the
// parser gets tripped up on something like an unmatched brace.
//
virtual public List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> possibleDocumentationComments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
while (lineIterator.IsInBounds)
{
bool foundComment = false;
PossibleDocumentationComment possibleDocumentationComment = null;
// Javadoc block comments
// We test for these before regular block comments because they are usually extended versions of them, such
// as /** and /*.
if (javadocBlockCommentStringPairs != null)
{
for (int i = 0; foundComment == false && i < javadocBlockCommentStringPairs.Length; i += 2)
{
foundComment = TryToGetBlockComment(ref lineIterator,
javadocBlockCommentStringPairs[i], javadocBlockCommentStringPairs[i + 1], true,
out possibleDocumentationComment);
}
if (possibleDocumentationComment != null)
{
possibleDocumentationComment.Javadoc = true;
}
}
// Plain block comments
// We test block comments ahead of line comments because in Lua the line comments are a substring of them: --
// versus --[[ and ]]--.
if (foundComment == false && blockCommentStringPairs != null)
{
for (int i = 0; foundComment == false && i < blockCommentStringPairs.Length; i += 2)
{
foundComment = TryToGetBlockComment(ref lineIterator,
blockCommentStringPairs[i], blockCommentStringPairs[i + 1], false,
out possibleDocumentationComment);
}
// Skip Splint comments so that they can appear in prototypes.
if (possibleDocumentationComment != null &&
possibleDocumentationComment.Start.FirstToken(LineBoundsMode.CommentContent).Character == '@')
{
LineIterator lastLine = possibleDocumentationComment.End;
lastLine.Previous();
TokenIterator lastToken, ignore;
lastLine.GetBounds(LineBoundsMode.CommentContent, out ignore, out lastToken);
lastToken.Previous();
if (lastToken.Character == '@')
{
possibleDocumentationComment = null;
}
}
}
// XML line comments
if (foundComment == false && xmlLineCommentStrings != null)
{
for (int i = 0; foundComment == false && i < xmlLineCommentStrings.Length; i++)
{
foundComment = TryToGetLineComment(ref lineIterator,
xmlLineCommentStrings[i], xmlLineCommentStrings[i], true,
out possibleDocumentationComment);
}
if (possibleDocumentationComment != null)
{
possibleDocumentationComment.XML = true;
}
}
// Ambiguous XML/Javadoc line comments
// If an XML comment is found we check the same position for Javadoc because they may share an opening
// symbol, such as ///.
if (possibleDocumentationComment != null && possibleDocumentationComment.XML == true &&
javadocLineCommentStringPairs != null)
{
LineIterator javadocLineIterator = possibleDocumentationComment.Start;
PossibleDocumentationComment possibleJavadocDocumentationComment = null;
bool foundJavadocComment = false;
for (int i = 0; foundJavadocComment == false && i < javadocLineCommentStringPairs.Length; i += 2)
{
foundJavadocComment = TryToGetLineComment(ref javadocLineIterator,
javadocLineCommentStringPairs[i], javadocLineCommentStringPairs[i + 1], true,
out possibleJavadocDocumentationComment);
}
if (possibleJavadocDocumentationComment != null)
{
// If the Javadoc comment is longer we use that instead of the XML since it may have detected the first
// line as XML and ignored the rest for not having the same symbol. For example:
//
// ## Comment
// #
// #
//
// This will be detected as a one line XML comment and a three line Javadoc comment.
if (possibleJavadocDocumentationComment.End.LineNumber >
possibleDocumentationComment.End.LineNumber)
{
possibleDocumentationComment = possibleJavadocDocumentationComment;
possibleDocumentationComment.Javadoc = true;
lineIterator = javadocLineIterator;
}
// If they're the same length...
else if (possibleJavadocDocumentationComment.End.LineNumber ==
possibleDocumentationComment.End.LineNumber)
{
// If the comments are both one line long then it's genuinely ambiguous. For example:
//
// ## Comment
//
// Is that a one line XML comment or a one line Javadoc comment? We can't tell, so mark it as
// potentially either.
if (possibleDocumentationComment.Start.LineNumber ==
possibleDocumentationComment.End.LineNumber - 1)
{
possibleDocumentationComment.Javadoc = true;
// XML should already be set to true
}
// If the comments are equal length but more than one line then it's just interpreting the XML as
// a Javadoc start with a vertical line for the remainder, so leave it as XML. For example:
//
// ## Comment
// ##
// ##
//
// That's clearly a three line XML comment and not a Javadoc comment with a vertical line.
}
// If the XML comment is longer just leave it and ignore the Javadoc one.
}
}
// Javadoc line comments
if (foundComment == false && javadocLineCommentStringPairs != null)
{
for (int i = 0; foundComment == false && i < javadocLineCommentStringPairs.Length; i += 2)
{
foundComment = TryToGetLineComment(ref lineIterator,
javadocLineCommentStringPairs[i], javadocLineCommentStringPairs[i + 1], true,
out possibleDocumentationComment);
}
if (possibleDocumentationComment != null)
{
possibleDocumentationComment.Javadoc = true;
}
}
// Plain line comments
if (foundComment == false && lineCommentStrings != null)
{
for (int i = 0; foundComment == false && i < lineCommentStrings.Length; i++)
{
foundComment = TryToGetLineComment(ref lineIterator,
lineCommentStrings[i], lineCommentStrings[i], false,
out possibleDocumentationComment);
}
}
// Nada.
if (foundComment == false)
{
lineIterator.Next();
}
else
{
if (possibleDocumentationComment != null)
{
// XML can actually use the Javadoc comment format in addition to its own.
if (possibleDocumentationComment.Javadoc == true)
{
possibleDocumentationComment.XML = true;
}
possibleDocumentationComments.Add(possibleDocumentationComment);
}
// lineIterator would have been moved already if foundComment is true
}
}
return(possibleDocumentationComments);
}
// Function: GetPossibleDocumentationComments
//
// Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
// comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
// comments it will return an empty list.
//
// All the comments in the returned list will have their comment symbols marked as <CommentParsingType.CommentSymbol>
// in the tokenizer. This allows further operations to be done on them in a language independent manner. If you want to also
// filter out text boxes and lines, use <Comments.LineFinder>.
//
override public List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> comments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
PODLineType podLineType;
while (lineIterator.IsInBounds)
{
TokenIterator tokenIterator = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
// Hash comments
if (tokenIterator.Character == '#')
{
PossibleDocumentationComment comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
// First line
if (tokenIterator.MatchesAcrossTokens("###"))
{
comment.Javadoc = false;
comment.XML = true;
}
else if (tokenIterator.MatchesAcrossTokens("##"))
{
comment.Javadoc = true;
comment.XML = false;
}
else // just "#"
{
comment.Javadoc = false;
comment.XML = false;
}
lineIterator.Next();
// Subsequent lines
while (lineIterator.IsInBounds)
{
tokenIterator = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (tokenIterator.Character != '#')
{
break;
}
if (tokenIterator.MatchesAcrossTokens("###"))
{
comment.Javadoc = false;
// XML is still possible
}
else if (tokenIterator.MatchesAcrossTokens("##"))
{
comment.Javadoc = false;
comment.XML = false;
}
else // just "#"
{
// Javadoc is still possible
comment.XML = false;
}
lineIterator.Next();
}
comment.End = lineIterator;
comments.Add(comment);
// Go back and mark the tokens
int firstLineCount, subsequentLineCount;
if (comment.XML)
{
firstLineCount = 3;
subsequentLineCount = 3;
}
else if (comment.Javadoc)
{
firstLineCount = 2;
subsequentLineCount = 1;
}
else // plain
{
firstLineCount = 1;
subsequentLineCount = 1;
}
LineIterator temp = comment.Start;
tokenIterator = temp.FirstToken(LineBoundsMode.ExcludeWhitespace);
tokenIterator.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, firstLineCount);
temp.Next();
while (temp < comment.End)
{
tokenIterator = temp.FirstToken(LineBoundsMode.ExcludeWhitespace);
tokenIterator.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, subsequentLineCount);
temp.Next();
}
// XML can actually appear in Javadoc comments
if (comment.Javadoc)
{
comment.XML = true;
}
}
// POD comments
else if (TryToSkipPODLine(ref tokenIterator, out podLineType))
{
TokenIterator podLineStart, podLineEnd;
lineIterator.GetBounds(LineBoundsMode.CommentContent, out podLineStart, out podLineEnd);
lineIterator.Tokenizer.SetCommentParsingTypeBetween(podLineStart, podLineEnd, CommentParsingType.CommentSymbol);
if (podLineType == PODLineType.StartNaturalDocs ||
podLineType == PODLineType.StartJavadoc)
{
PossibleDocumentationComment comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
if (podLineType == PODLineType.StartJavadoc)
{
comment.Javadoc = true;
}
for (;;)
{
lineIterator.Next();
if (lineIterator.IsInBounds == false)
{
break;
}
tokenIterator = lineIterator.FirstToken(LineBoundsMode.CommentContent);
if (TryToSkipPODLine(ref tokenIterator, out podLineType) == true)
{
if (podLineType == PODLineType.End)
{
lineIterator.GetBounds(LineBoundsMode.CommentContent, out podLineStart, out podLineEnd);
lineIterator.Tokenizer.SetCommentParsingTypeBetween(podLineStart, podLineEnd, CommentParsingType.CommentSymbol);
lineIterator.Next();
}
break;
}
}
comment.End = lineIterator;
comments.Add(comment);
}
else
{
lineIterator.Next();
}
}
else
{
lineIterator.Next();
}
}
return(comments);
}
/* Function: MarkTextBoxes
*
* Finds all text boxes in a comment and marks their tokens as <Tokenization.CommentParsingType.CommentDecoration>.
* Vertical lines will only be detected if they are continuous throughout the comment and horizontal lines if they
* are connected to it. Freestanding horizontal lines are *not* detected here. This function tolerates differing
* symbols on corners and where embedded horizontal lines connect to the vertical. It also tolerates tokens
* marked with <Tokenization.CommentParsingType.CommentSymbol> differing.
*
* Examples:
*
* The box below will be marked completely, including the middle horizontal line.
*
* > // +----------+
* > // | Title |
* > // +----------+
* > // | Text |
* > // +----------+
*
* The middle horizontal line below will not be marked, because it is not attached.
*
* > // +----------+
* > // | Title |
* > // | -------- |
* > // | Text |
* > // +----------+
*
* Nor will the horizontal line below since there is no vertical.
*
* > // Title
* > // ----------
* > // Text
*
* Freestanding horizontal lines are not detected because they may be intended literally, such as when part of
* a code section. If you're not in such a section use <IsHorizontalLine()> before parsing a line to filter it out.
*
* > // (start code)
* > // +-----+
* > // | box |
* > // +-----+
* > // (end code)
*/
public static void MarkTextBoxes(PossibleDocumentationComment comment)
{
char symbolA, symbolB, symbolC;
int symbolACount, symbolBCount, symbolCCount;
char leftSymbol = '\0';
char rightSymbol = '\0';
int leftSymbolCount = 0;
int rightSymbolCount = 0;
bool symbolIsAloneOnLine = false;
bool testedForVerticalLines = false;
LineIterator line = comment.Start;
TokenIterator lineStart, lineEnd;
// This should be okay to use since line numbers start at one.
IDObjects.NumberSet horizontalLines = new IDObjects.NumberSet();
// Skip leading blank lines since it's okay if they're not part of the text box.
while (line < comment.End && line.IsEmpty(LineBoundsMode.CommentContent))
{
line.Next();
}
while (line < comment.End && line.IsEmpty(LineBoundsMode.CommentContent) == false)
{
line.GetBounds(LineBoundsMode.ExcludeWhitespace, out lineStart, out lineEnd);
// Shrink the line to exclude its comment symbols, if any. We didn't do this using the line bounds mode because
// we need to know whether there was any whitespace between them and any horizontal lines.
bool commentSymbolWithoutWhitespaceAtStart = false;
bool commentSymbolWithoutWhitespaceAtEnd = false;
if (lineStart.CommentParsingType == CommentParsingType.CommentSymbol)
{
commentSymbolWithoutWhitespaceAtStart = true;
do
{
lineStart.Next();
}while (lineStart.CommentParsingType == CommentParsingType.CommentSymbol);
if (lineStart.FundamentalType == FundamentalType.Whitespace)
{
commentSymbolWithoutWhitespaceAtStart = false;
do
{
lineStart.Next();
}while (lineStart.FundamentalType == FundamentalType.Whitespace);
}
}
lineEnd.Previous();
if (lineEnd.CommentParsingType == CommentParsingType.CommentSymbol)
{
commentSymbolWithoutWhitespaceAtEnd = true;
do
{
lineEnd.Previous();
}while (lineEnd.CommentParsingType == CommentParsingType.CommentSymbol);
if (lineEnd.FundamentalType == FundamentalType.Whitespace)
{
commentSymbolWithoutWhitespaceAtEnd = false;
do
{
lineEnd.Previous();
}while (lineEnd.FundamentalType == FundamentalType.Whitespace);
}
}
lineEnd.Next();
// Horizontal line detection
bool isHorizontalLine = false;
CountSymbolLine(ref lineStart, lineEnd, out symbolA, out symbolB, out symbolC,
out symbolACount, out symbolBCount, out symbolCCount);
if (commentSymbolWithoutWhitespaceAtStart == true &&
commentSymbolWithoutWhitespaceAtEnd == true &&
symbolACount >= 4 && symbolBCount == 0)
{
isHorizontalLine = true;
}
else if (commentSymbolWithoutWhitespaceAtStart == true &&
symbolACount >= 4 &&
(symbolBCount == 0 || (symbolBCount <= 3 && symbolCCount == 0)))
{
isHorizontalLine = true;
}
else if (commentSymbolWithoutWhitespaceAtEnd == true &&
((symbolACount >= 1 && symbolACount <= 3 && symbolBCount >= 4 && symbolCCount == 0) ||
(symbolACount >= 4 && symbolBCount == 0)))
{
isHorizontalLine = true;
}
else if ((symbolACount >= 4 && symbolBCount == 0) ||
(symbolACount >= 1 && symbolACount <= 3 && symbolBCount >= 4 && symbolCCount <= 3))
{
isHorizontalLine = true;
}
// The horizontal line has to be the only thing on the line to count.
if (isHorizontalLine && lineStart == lineEnd)
{
horizontalLines.Add(line.LineNumber);
}
// Vertical line detection
else if (testedForVerticalLines == false)
{
// We permit the very first line to be different to allow for this:
// /** text
// * text
// */
//
// However, don't skip the first line if it's a one line comment or we wouldn't be able to handle this:
// ### text
//
if (line != comment.Start ||
(comment.End.LineNumber - comment.Start.LineNumber) == 1)
{
if (CountEdgeSymbols(line, out leftSymbol, out rightSymbol, out leftSymbolCount, out rightSymbolCount,
out symbolIsAloneOnLine) == false)
{
return;
}
testedForVerticalLines = true;
}
}
else // testedForVerticalLines == true
{
char lineLeftSymbol, lineRightSymbol;
int lineLeftSymbolCount, lineRightSymbolCount;
bool lineSymbolIsAloneOnLine;
CountEdgeSymbols(line, out lineLeftSymbol, out lineRightSymbol, out lineLeftSymbolCount, out lineRightSymbolCount,
out lineSymbolIsAloneOnLine);
// Account for a lone symbol being the right symbol.
if (lineSymbolIsAloneOnLine == true && symbolIsAloneOnLine == false && leftSymbolCount == 0 && rightSymbolCount > 0)
{
if (lineLeftSymbol != rightSymbol || lineLeftSymbolCount != rightSymbolCount)
{
return;
}
}
else if (lineSymbolIsAloneOnLine == false && symbolIsAloneOnLine == true && lineLeftSymbolCount == 0 && lineRightSymbolCount > 0)
{
if (lineRightSymbol != leftSymbol || lineRightSymbolCount != leftSymbolCount)
{
return;
}
rightSymbol = leftSymbol;
leftSymbol = '\0';
rightSymbolCount = leftSymbolCount;
leftSymbolCount = 0;
}
// Otherwise it's okay to do a straight compare.
else
{
if (lineLeftSymbol != leftSymbol || lineLeftSymbolCount != leftSymbolCount)
{
leftSymbol = '\0';
leftSymbolCount = 0;
}
if (lineRightSymbol != rightSymbol || lineRightSymbolCount != rightSymbolCount)
{
rightSymbol = '\0';
rightSymbolCount = 0;
}
if (leftSymbolCount == 0 && rightSymbolCount == 0)
{
return;
}
}
// Turn off the overall alone flag if this line didn't have it.
if (lineSymbolIsAloneOnLine == false)
{
symbolIsAloneOnLine = false;
}
}
line.Next();
}
// If we stopped because we hit a blank line, this comment is only acceptable for marking text boxes if all the lines
// left are blank.
while (line < comment.End && line.IsEmpty(LineBoundsMode.CommentContent))
{
line.Next();
}
if (line != comment.End)
{
return;
}
// If we made it this far without returning it means we have a valid text box which we have to mark as comment decoration.
line = comment.Start;
while (line < comment.End)
{
line.GetBounds(LineBoundsMode.CommentContent, out lineStart, out lineEnd);
if (horizontalLines.Contains(line.LineNumber))
{
while (lineStart < lineEnd)
{
lineStart.CommentParsingType = CommentParsingType.CommentDecoration;
lineStart.Next();
}
}
else if (lineEnd > lineStart)
{
// We test the characters against the symbols to account for any exceptions we allowed to go through
// in previous code.
for (int i = 0; i < leftSymbolCount; i++)
{
if (lineStart.Character == leftSymbol)
{
lineStart.CommentParsingType = CommentParsingType.CommentDecoration;
lineStart.Next();
}
}
lineEnd.Previous();
for (int i = 0; i < rightSymbolCount; i++)
{
if (lineEnd.Character == rightSymbol)
{
lineEnd.CommentParsingType = CommentParsingType.CommentDecoration;
lineEnd.Previous();
}
}
}
line.Next();
}
}
/* Function: FindIncludeInOutput
* Extracts and returns any comment content marked "include in output". All comment symbols and extra indents
* will be removed. Returns null if the comment doesn't have any.
*/
protected string FindIncludeInOutput(PossibleDocumentationComment comment)
{
Comments.LineFinder.MarkTextBoxes(comment);
LineIterator iterator = comment.Start;
// Find the "include in output" header if it exists
while (iterator < comment.End &&
iterator.Match(IncludeInOutputRegex, LineBoundsMode.CommentContent).Success == false)
{
iterator.Next();
}
if (iterator >= comment.End)
{
return(null);
}
// Find the bounds of the content excluding whitespace and the shared indent level
iterator.Next();
while (iterator < comment.End && iterator.IsEmpty(LineBoundsMode.CommentContent))
{
iterator.Next();
}
LineIterator start = iterator;
LineIterator end = iterator;
int commonIndent = 9999;
while (iterator < comment.End)
{
if (iterator.IsEmpty(LineBoundsMode.CommentContent))
{
iterator.Next();
}
else
{
if (iterator.Indent(LineBoundsMode.CommentContent) < commonIndent)
{
commonIndent = iterator.Indent(LineBoundsMode.CommentContent);
}
iterator.Next();
end = iterator;
}
}
// Build and return the comment content
if (start >= end)
{
return(null);
}
StringBuilder output = new StringBuilder();
do
{
int indentDifference = start.Indent(LineBoundsMode.CommentContent) - commonIndent;
if (indentDifference > 0)
{
output.Append(' ', indentDifference);
}
start.AppendTo(output, LineBoundsMode.CommentContent);
output.AppendLine();
start.Next();
}while (start < end);
return(output.ToString());
}
/* Function: TryToGetBlock
* If the iterator is on a line that starts with one of the <BlockTags>, parses it, adds its content to the comment,
* moves the iterator past it, and returns true. If it is not at the start of a tag block it will return false and change
* nothing.
*/
protected bool TryToGetBlock(ref LineIterator lineIterator, LineIterator limit, JavadocComment comment)
{
if (lineIterator >= limit)
{
return(false);
}
// Get the complete content across multiple lines.
string tag;
TokenIterator startOfContent;
if (TryToGetFirstBlockLine(lineIterator, out tag, out startOfContent) == false)
{
return(false);
}
for (;;)
{
lineIterator.Next();
if (lineIterator >= limit || IsFirstBlockLine(lineIterator))
{
break;
}
}
TokenIterator endOfContent = lineIterator.FirstToken(LineBoundsMode.Everything);
// Any "@tag item description", possibly in a list
if (tag == "exception" ||
tag == "param" ||
tag == "throws")
{
TokenIterator iterator = startOfContent;
string symbol = null;
TryToGetBlockSymbol(ref iterator, endOfContent, out symbol);
iterator.NextPastWhitespace();
string description = GetText(iterator, endOfContent);
description = Normalize(description);
if (symbol == null || symbol == "" || description == null || description == "")
{
return(false);
}
var commentBlock = comment.GetListBlock(tag);
commentBlock.Add(symbol, description);
return(true);
}
// Any "@tag description", possibly in a list
else if (tag == "author" ||
tag == "deprecated" ||
tag == "since" ||
tag == "version")
{
string description = GetText(startOfContent, endOfContent);
description = Normalize(description);
if (description == null || description == "")
{
return(false);
}
if (tag == "deprecated")
{
if (comment.Deprecated == null)
{
comment.Deprecated = description;
}
else
{
comment.Deprecated += description;
}
}
else
{
var commentBlock = comment.GetListBlock(tag);
commentBlock.Add(null, description);
}
return(true);
}
// Any "@tag description" that can't be in a list
else if (tag == "return")
{
string description = GetText(startOfContent, endOfContent);
description = Normalize(description);
if (description == null || description == "")
{
return(false);
}
var commentBlock = comment.GetTextBlock(tag);
commentBlock.Text.Append(description);
return(true);
}
else if (tag == "see")
{
string description = null;
TokenIterator iterator = startOfContent;
// @see "Description"
// @see <a href="link">Description</a>
if (iterator.Character == '"' ||
iterator.Character == '<')
{
// There's not symbol so interpret the whole thing as the description. We'll let GetText() handle the HTML link.
description = GetText(iterator, endOfContent);
description = Normalize(description);
}
// @see Class.Class#Member
// @see Class.Class#Member Description
else
{
string symbol = GetJavadocLinkSymbol(ref iterator);
iterator.NextPastWhitespace();
description = GetSimpleText(iterator, endOfContent);
description = Normalize(description);
if (description == null || description == "")
{
description = "<p><link type=\"naturaldocs\" originaltext=\"" + symbol.EntityEncode() + "\"></p>";
}
else
{
description = "<p><link type=\"naturaldocs\" originaltext=\"" + description.EntityEncode() + " at " + symbol.EntityEncode() + "\"></p>";
}
}
if (description == null || description == "")
{
return(false);
}
var commentBlock = comment.GetListBlock(tag);
commentBlock.Add(null, description);
return(true);
}
// Ignored blocks
// - serial
// - serialField
// - serialData
else
{
return(true);
}
}
// Group: Support Functions
// __________________________________________________________________________
/* Function: TryToGetPDBlockComment
*
* If the line iterator is on the starting symbol of a block comment, return it as a <PossibleDocumentationComment>
* and mark the symbols as <CommentParsingType.CommentSymbol>. If the iterator is not on the opening comment
* symbol or there is content after the closing comment symbol making it unsuitable as a documentation comment,
* returns null.
*
* If openingMustBeAlone is set, that means no symbol can appear immediately after the opening symbol for this
* function to succeed. This allows you to specifically detect something like /** without also matching /******.
*/
protected PossibleDocumentationComment TryToGetPDBlockComment(LineIterator lineIterator,
string openingSymbol, string closingSymbol,
bool openingMustBeAlone)
{
TokenIterator firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
if (firstToken.MatchesAcrossTokens(openingSymbol) == false)
{
return(null);
}
if (openingMustBeAlone)
{
TokenIterator nextToken = firstToken;
nextToken.NextByCharacters(openingSymbol.Length);
if (nextToken.FundamentalType == FundamentalType.Symbol)
{
return(null);
}
}
PossibleDocumentationComment comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
for (;;)
{
if (!lineIterator.IsInBounds)
{
return(null);
}
TokenIterator closingSymbolIterator;
if (lineIterator.FindAcrossTokens(closingSymbol, false, LineBoundsMode.Everything, out closingSymbolIterator) == true)
{
closingSymbolIterator.NextByCharacters(closingSymbol.Length);
closingSymbolIterator.NextPastWhitespace();
if (closingSymbolIterator.FundamentalType != FundamentalType.LineBreak &&
closingSymbolIterator.FundamentalType != FundamentalType.Null)
{
return(null);
}
lineIterator.Next();
comment.End = lineIterator;
break;
}
lineIterator.Next();
}
// Success. Mark the symbols before returning.
firstToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, openingSymbol.Length);
TokenIterator lastToken;
lineIterator.Previous();
lineIterator.GetBounds(LineBoundsMode.ExcludeWhitespace, out firstToken, out lastToken);
lastToken.PreviousByCharacters(closingSymbol.Length);
lastToken.SetCommentParsingTypeByCharacters(CommentParsingType.CommentSymbol, closingSymbol.Length);
return(comment);
}
// Function: GetPossibleDocumentationComments
//
// Goes through the file looking for comments that could possibly contain documentation and returns them as a list. These
// comments are not guaranteed to have documentation in them, just to be acceptable candidates for them. If there are no
// comments it will return an empty list.
//
// All the comments in the returned list will have their comment symbols marked as <CommentParsingType.CommentSymbol>
// in the tokenizer. This allows further operations to be done on them in a language independent manner. If you want to also
// filter out text boxes and lines, use <Comments.LineFinder>.
//
// Default Implementation:
//
// The default implementation uses the comment symbols found in <Language> or passed to the constructor. You can override
// this function if you need to do something more sophisticated, such as interpret the POD directives in Perl.
//
// Comments must be alone on a line to be a candidate for documentation, meaning that the comment symbol must be the
// first non-whitespace character on a line, and in the case of block comments, nothing but whitespace may trail the closing
// symbol. The latter rule is important because a comment may start correctly but not end so, as in this prototype with Splint
// annotation:
//
// > int get_array(integer_t id,
// > /*@out@*/ array_t array);
//
// Speaking of which, block comments surrounded by @ symbols are not included because they're Splint comments. Not
// including them in the possible documentation comments list means the Splint comment below won't end prototype detection.
//
// > void initialize ()
// > /*@globals undef globnum,
// > undef globname @*/
// > { ... }
//
// It also goes through the code line by line in a simple manner, not accounting for things like strings, so if a language contains
// a multiline string whose content looks like a language comment it will be interpreted as one. This isn't ideal but is accepted
// as a conscious tradeoff because there are actually many different string formats (literal quotes denoted with \", literal quotes
// denoted with "", Perl's q{} forms and here doc) so you can't account for them all in a generalized way. Also, putting this in
// an independent stage even when using full language support means comments don't disappear the way prototypes do if the
// parser gets tripped up on something like an unmatched brace.
//
virtual public List <PossibleDocumentationComment> GetPossibleDocumentationComments(Tokenizer source)
{
List <PossibleDocumentationComment> possibleDocumentationComments = new List <PossibleDocumentationComment>();
LineIterator lineIterator = source.FirstLine;
while (lineIterator.IsInBounds)
{
PossibleDocumentationComment comment = null;
// Javadoc block comments
// We test for these before regular block comments because they are usually extended versions of them, such
// as /** and /*.
// We also test block comments in general ahead of line comments because in Lua the line comments are a
// substring of them: -- versus --[[ and ]]--.
if (javadocBlockCommentStringPairs != null)
{
for (int i = 0; comment == null && i < javadocBlockCommentStringPairs.Length; i += 2)
{
comment = TryToGetPDBlockComment(lineIterator, javadocBlockCommentStringPairs[i],
javadocBlockCommentStringPairs[i + 1], true);
}
if (comment != null)
{
comment.Javadoc = true;
}
}
// Plain block comments
if (comment == null && blockCommentStringPairs != null)
{
for (int i = 0; comment == null && i < blockCommentStringPairs.Length; i += 2)
{
comment = TryToGetPDBlockComment(lineIterator, blockCommentStringPairs[i],
blockCommentStringPairs[i + 1], false);
}
// Skip Splint comments so that they can appear in prototypes.
if (comment != null && comment.Start.FirstToken(LineBoundsMode.CommentContent).Character == '@')
{
LineIterator lastLine = comment.End;
lastLine.Previous();
TokenIterator lastToken, ignore;
lastLine.GetBounds(LineBoundsMode.CommentContent, out ignore, out lastToken);
lastToken.Previous();
if (lastToken.Character == '@')
{
comment = null;
}
}
}
// XML line comments
if (comment == null && xmlLineCommentStrings != null)
{
for (int i = 0; comment == null && i < xmlLineCommentStrings.Length; i++)
{
comment = TryToGetPDLineComment(lineIterator, xmlLineCommentStrings[i],
xmlLineCommentStrings[i], true);
}
if (comment != null)
{
comment.XML = true;
}
}
// Javadoc line comments
// We check for these even if a XML comment is found because they may share an opening symbol, such as ///.
// We change it to Javadoc if it's longer. If it's equal it's just interpreting the XML as a Javadoc start with a
// vertical line for the remainder, so leave it as XML. Unless the comment is only one line long, in which case it's
// genuinely ambiguous.
if ((comment == null || comment.XML == true) && javadocLineCommentStringPairs != null)
{
PossibleDocumentationComment javadocComment = null;
for (int i = 0; javadocComment == null && i < javadocLineCommentStringPairs.Length; i += 2)
{
javadocComment = TryToGetPDLineComment(lineIterator, javadocLineCommentStringPairs[i],
javadocLineCommentStringPairs[i + 1], true);
}
if (javadocComment != null)
{
javadocComment.Javadoc = true;
if (comment == null)
{
comment = javadocComment;
}
else
{
int javadocLength = javadocComment.End.LineNumber - javadocComment.Start.LineNumber;
int xmlLength = comment.End.LineNumber - comment.Start.LineNumber;
if (javadocLength > xmlLength)
{
comment = javadocComment;
}
else if (javadocLength == 1 && xmlLength == 1)
{
comment.Javadoc = true;
}
// else stay with the XML comment
}
}
}
// Plain line comments
if (comment == null && lineCommentStrings != null)
{
for (int i = 0; comment == null && i < lineCommentStrings.Length; i++)
{
comment = TryToGetPDLineComment(lineIterator, lineCommentStrings[i], lineCommentStrings[i], false);
}
}
// Nada.
if (comment == null)
{
lineIterator.Next();
}
else
{
// XML can actually use the Javadoc comment format in addition to its own.
if (comment.Javadoc)
{
comment.XML = true;
}
possibleDocumentationComments.Add(comment);
lineIterator = comment.End;
}
}
return(possibleDocumentationComments);
}
/* Function: TryToGetLineComment
*
* If the iterator is on a line that starts with a line comment symbol, this function moves the iterator past the entire
* comment and returns true. If the comment is a candidate for documentation it will also return it as a
* <PossibleDocumentationComment> and mark the symbols as <CommentParsingType.CommentSymbol>. If the
* line does not start with a line comment symbol it will return false and leave the iterator where it is.
*/
protected bool TryToGetLineComment(ref LineIterator lineIterator, out PossibleDocumentationComment comment)
{
TokenIterator firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
TokenIterator lookahead = firstToken;
// Are we on a line comment?
if (TryToSkipLineCommentSymbol(ref lookahead) == false)
{
comment = null;
return(false);
}
// We are. Create a possible documentation comment.
comment = new PossibleDocumentationComment();
comment.Start = lineIterator;
// Check if we're a Javadoc/XML comment. We can't tell the difference from just the first line.
TokenIterator endOfCommentSymbol = lookahead;
if (lookahead.Character == '-')
{
lookahead.Next();
if (lookahead.FundamentalType != FundamentalType.Symbol)
{
endOfCommentSymbol = lookahead;
comment.Javadoc = true;
comment.XML = true;
}
}
// Mark it.
firstToken.SetCommentParsingTypeBetween(endOfCommentSymbol, CommentParsingType.CommentSymbol);
// Continue to find the rest of the comment
lineIterator.Next();
bool hasXMLishLines = false;
bool hasNonXMLishLines = false;
bool hasMultipleLines = false;
while (lineIterator.IsInBounds)
{
firstToken = lineIterator.FirstToken(LineBoundsMode.ExcludeWhitespace);
lookahead = firstToken;
if (TryToSkipLineCommentSymbol(ref lookahead) == false)
{
break;
}
hasMultipleLines = true;
endOfCommentSymbol = lookahead;
if (lookahead.Character == '-')
{
lookahead.Next();
if (lookahead.FundamentalType != FundamentalType.Symbol)
{
hasXMLishLines = true;
endOfCommentSymbol = lookahead;
}
else
{
hasNonXMLishLines = true;
}
}
else
{
hasNonXMLishLines = true;
}
firstToken.SetCommentParsingTypeBetween(endOfCommentSymbol, CommentParsingType.CommentSymbol);
lineIterator.Next();
}
comment.End = lineIterator;
if (hasMultipleLines && comment.Javadoc)
{
if (hasXMLishLines && !hasNonXMLishLines)
{
comment.Javadoc = false;
}
else if (hasNonXMLishLines)
{
comment.XML = false;
}
}
return(true);
}