internal ContinuationToken(Token virtualConcatenatedToken, int startIndex, int stopIndex, ITokensLine tokensLine, bool isContinuationFromPreviousLine, bool isContinuedOnNextLine)
: base(virtualConcatenatedToken.TokenType, startIndex, stopIndex, virtualConcatenatedToken.UsesVirtualSpaceAtEndOfLine, tokensLine)
{
// Store the concatenated source text
MultilineContinuationText = virtualConcatenatedToken.Text;
IsContinuationFromPreviousLine = isContinuationFromPreviousLine;
IsContinuedOnNextLine = isContinuedOnNextLine;
// Copy the delimiter properties
if (virtualConcatenatedToken.UsesDelimiters)
{
UsesDelimiters = virtualConcatenatedToken.UsesDelimiters;
HasOpeningDelimiter = virtualConcatenatedToken.HasOpeningDelimiter;
HasClosingDelimiter = virtualConcatenatedToken.HasClosingDelimiter;
ExpectedClosingDelimiter = virtualConcatenatedToken.ExpectedClosingDelimiter;
}
// Copy the literal value
LiteralValue = virtualConcatenatedToken.LiteralValue;
// Set specific Channel to enable filtering of all tokens participating in the continuation, except the first one
if (IsContinuationFromPreviousLine)
{
Channel = CHANNEL_ContinuationTokens;
}
}
/// <summary>
/// Sets the current iterator position to point to a specific token.
/// After a call to this method, GetNextToken returns the token FOLLOWING startToken.
/// </summary>
public void SeekToToken(Token startToken)
{
// Find line for the start token
currentPosition.LineIndex = tokensLines.IndexOf(startToken.TokensLine, startToken.TokensLine.LineIndex);
currentLine = startToken.TokensLine;
// Find index in line for the start token
currentPosition.TokenIndexInLine = currentLine.SourceTokens.IndexOf(startToken);
currentToken = startToken;
}
/// <summary>
/// Resets the iterator position : before the first token of the document
/// </summary>
public virtual void Reset()
{
currentPosition.LineIndex = 0;
if (tokensLines.Count > 0)
{
currentLine = tokensLines[currentPosition.LineIndex];
}
currentPosition.TokenIndexInLine = -1;
currentToken = null;
}
/// <summary>
/// Constructor for tokens without delimiters, using the virtual space at the end of the line
/// </summary>
internal Token(TokenType tokenType, int startIndex, int stopIndex, bool usesVirtualSpaceAtEndOfLine, ITokensLine tokensLine)
{
TokenType = tokenType;
TokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
HasError = false;
SetInitialChannelFromTokenFamily();
this.startIndex = startIndex;
this.stopIndex = stopIndex;
this.tokensLine = tokensLine;
UsesDelimiters = false;
HasClosingDelimiter = false;
UsesVirtualSpaceAtEndOfLine = usesVirtualSpaceAtEndOfLine;
}
/// <summary>
/// Constructor for tokens without delimiters, using the virtual space at the end of the line
/// </summary>
internal Token(TokenType tokenType, int startIndex, int stopIndex, bool usesVirtualSpaceAtEndOfLine, ITokensLine tokensLine)
{
TokenType = tokenType;
TokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
HasError = false;
SetInitialChannelFromTokenFamily();
this.startIndex = startIndex;
this.stopIndex = stopIndex;
this.tokensLine = tokensLine;
UsesDelimiters = false;
HasClosingDelimiter = false;
UsesVirtualSpaceAtEndOfLine = usesVirtualSpaceAtEndOfLine;
}
public static string BuildResultString(ITokensLine tokensLine)
{
StringBuilder tokensText = new StringBuilder();
foreach (Token token in tokensLine.SourceTokens)
{
tokensText.AppendLine(token.ToString());
}
StringBuilder diagnosticsText = new StringBuilder();
foreach (Diagnostic diagnostic in tokensLine.ScannerDiagnostics)
{
diagnosticsText.AppendLine(diagnostic.ToString());
}
return tokensText.ToString() + diagnosticsText.ToString();
}
protected override void ColorizeLine(DocumentLine line)
{
TypeCobolCompilerService compilerService = CurrentContext.TextView.GetService(typeof(TypeCobolCompilerService)) as TypeCobolCompilerService;
if (compilerService != null)
{
int lineIndex = line.LineNumber - 1;
if (compilerService.CompilationUnit.TokensDocumentSnapshot.Lines.Count > lineIndex)
{
ITokensLine tokensLine = compilerService.CompilationUnit.TokensDocumentSnapshot.Lines[lineIndex];
int lineStartOffset = line.Offset;
// Areas
if (!tokensLine.SequenceNumber.IsEmpty)
{
ApplyTextAreaStyle(lineStartOffset, tokensLine.SequenceNumber, TokenStyles.GetAreaStyle(TextAreaType.SequenceNumber));
}
if (!tokensLine.Indicator.IsEmpty)
{
if (tokensLine.Type == CobolTextLineType.Comment)
{
ApplyTextAreaStyle(lineStartOffset, tokensLine.Indicator, TokenStyles.GetAreaStyle(TextAreaType.Comment));
}
else if (tokensLine.Type == CobolTextLineType.Debug || tokensLine.Type == CobolTextLineType.Continuation)
{
ApplyTextAreaStyle(lineStartOffset, tokensLine.Indicator, TokenStyles.GetAreaStyle(TextAreaType.Indicator));
}
}
if (!tokensLine.Comment.IsEmpty)
{
ApplyTextAreaStyle(lineStartOffset, tokensLine.Comment, TokenStyles.GetAreaStyle(TextAreaType.Comment));
}
// Tokens
foreach (Token token in tokensLine.SourceTokens)
{
HighlightingColor tokenStyle = TokenStyles.GetTokenStyle(token);
if (tokenStyle != null && token.Length > 0)
{
ApplyTokenStyle(lineStartOffset, token, tokenStyle);
}
}
}
}
}
public static string BuildResultString(ITokensLine tokensLine)
{
StringBuilder tokensText = new StringBuilder();
foreach (Token token in tokensLine.SourceTokens)
{
tokensText.AppendLine(token.ToString());
}
StringBuilder diagnosticsText = new StringBuilder();
foreach (Diagnostic diagnostic in tokensLine.ScannerDiagnostics)
{
diagnosticsText.AppendLine(diagnostic.ToString());
}
return(tokensText.ToString() + diagnosticsText.ToString());
}
/// <summary>
/// Constructor for tokens with delimiters, using the virtual space at the end of the line
/// </summary>
internal Token(TokenType tokenType, int startIndex, int stopIndex, bool usesVirtualSpaceAtEndOfLine, ITokensLine tokensLine, bool hasOpeningDelimiter, bool hasClosingDelimiter, char expectedClosingDelimiter)
{
TokenType = tokenType;
TokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
SetInitialChannelFromTokenFamily();
HasError = false;
this.startIndex = startIndex;
this.stopIndex = stopIndex;
this.tokensLine = tokensLine;
UsesDelimiters = true;
HasOpeningDelimiter = hasOpeningDelimiter;
HasClosingDelimiter = hasClosingDelimiter;
ExpectedClosingDelimiter = expectedClosingDelimiter;
UsesVirtualSpaceAtEndOfLine = usesVirtualSpaceAtEndOfLine;
}
/// <summary>
/// Get the next token, with or without applying the channel filter
/// </summary>
public Token NextToken(bool applyChannelFilter)
{
// If document is empty, immediately return EndOfFile
if (currentLine == null)
{
return(Token.END_OF_FILE);
}
// While we can find a next token
currentToken = null;
while (currentToken == null)
{
// try to find the next token on the same line
currentPosition.TokenIndexInLine++;
// but if we reached the end of the current line ...
while (currentPosition.TokenIndexInLine >= currentLine.SourceTokens.Count)
{
// .. advance to next line
currentPosition.LineIndex++;
currentPosition.TokenIndexInLine = 0;
if (currentPosition.LineIndex < tokensLines.Count)
{
// TO DO - OPTIMIZATION : should we use here an Enumerator on tokensLines ?
currentLine = tokensLines[currentPosition.LineIndex];
}
// and if we reached the last line of the document ...
else
{
// return EndOfFile
currentLine = null;
return(Token.END_OF_FILE);
}
}
// Check if the next token found matches the filter criteria
Token nextTokenCandidate = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
if (!applyChannelFilter || nextTokenCandidate.Channel == channelFilter)
{
currentToken = nextTokenCandidate;
}
}
// found a next token matching the filter criteria
return(currentToken);
}
public void BeginParsingFile(TextSourceInfo textSourceInfo, ITokensLinesIterator tokensCountIterator)
{
CurrentFileInfo = new ParsedFileInfo(textSourceInfo.Name, parserRulesCount, parserDecisionsCount);
// Only for CodeElementsParser
if (tokensCountIterator != null)
{
ITokensLine lastLine = null;
Token token = null;
while ((token = tokensCountIterator.NextToken()) != Token.END_OF_FILE)
{
CurrentFileInfo.TokensCount++;
if (token.TokensLine != lastLine)
{
CurrentFileInfo.LinesCount++;
lastLine = token.TokensLine;
}
}
}
}
/// <summary>
/// Constructor for tokens with delimiters, using the virtual space at the end of the line
/// </summary>
internal Token(TokenType tokenType, int startIndex, int stopIndex, bool usesVirtualSpaceAtEndOfLine, ITokensLine tokensLine, bool hasOpeningDelimiter, bool hasClosingDelimiter, char expectedClosingDelimiter)
{
TokenType = tokenType;
TokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
SetInitialChannelFromTokenFamily();
HasError = false;
this.startIndex = startIndex;
this.stopIndex = stopIndex;
this.tokensLine = tokensLine;
UsesDelimiters = true;
HasOpeningDelimiter = hasOpeningDelimiter;
HasClosingDelimiter = hasClosingDelimiter;
ExpectedClosingDelimiter = expectedClosingDelimiter;
UsesVirtualSpaceAtEndOfLine = usesVirtualSpaceAtEndOfLine;
}
/// <summary>
/// Sets the current iterator position to a previous position returned by GetCurrentPosition.
/// After a call to this method, GetNextToken returns the token FOLLOWING the current position.
/// </summary>
public void SeekToPosition(object iteratorPosition)
{
currentPosition = (TokensLineIteratorPosition)iteratorPosition;
if (currentPosition.LineIndex >= 0 && currentPosition.LineIndex < tokensLines.Count)
{
currentLine = tokensLines[currentPosition.LineIndex];
}
else
{
currentLine = null;
}
if (currentPosition.TokenIndexInLine >= 0 && currentLine != null)
{
currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
}
else
{
currentToken = null;
}
}
/// <summary>
/// Returns all tokens (no channel filter), line per line, selected between startToken and stopToken.
/// This method takes a snapshot of the current position before its execution and restores it before returning.
/// </summary>
public MultilineTokensGroupSelection SelectAllTokensBetween(Token startToken, Token stopToken)
{
// Check parameters
ITokensLine startLine = startToken.TokensLine;
int startLineIndex = tokensLines.IndexOf(startLine, startLine.LineIndex);
ITokensLine stopLine = stopToken.TokensLine;
int stopLineIndex = tokensLines.IndexOf(stopLine, stopLine.LineIndex);
if (startLineIndex < 0 || stopLineIndex < 0 || stopLineIndex < startLineIndex ||
((startLineIndex == stopLineIndex) && (stopToken.StartIndex < startToken.StartIndex)))
{
throw new InvalidOperationException("Invalid start or stop token : line or columns number do not define a valid selection interval");
}
// Save iterator position (to restore it before return)
SaveCurrentPositionSnapshot();
// Seek to the first token at the start of the line where startToken appears
currentPosition.LineIndex = startLineIndex;
currentLine = startToken.TokensLine;
currentPosition.TokenIndexInLine = 0;
currentToken = currentLine.SourceTokens[0];
// List of tokens not selected on the first line before startToken
IList <Token> tokensOnFirstLineBeforeStartToken = null;
if (currentToken != startToken)
{
tokensOnFirstLineBeforeStartToken = new List <Token>();
do
{
tokensOnFirstLineBeforeStartToken.Add(currentToken);
}while (NextToken(false) != startToken);
}
// List of tokens selected ...
int numberOfSelectedLines = stopLineIndex - startLineIndex + 1;
IList <Token>[] selectedTokensOnSeveralLines = new IList <Token> [numberOfSelectedLines];
// ... on the first line
IList <Token> tokensSelectedOnFirstLine = new List <Token>();
selectedTokensOnSeveralLines[0] = tokensSelectedOnFirstLine;
if (numberOfSelectedLines == 1)
{
if (currentToken == stopToken)
{
tokensSelectedOnFirstLine.Add(currentToken);
}
else
{
do
{
tokensSelectedOnFirstLine.Add(currentToken);
}while (NextToken(false) != stopToken);
tokensSelectedOnFirstLine.Add(currentToken);
}
NextToken(false);
}
else
{
do
{
tokensSelectedOnFirstLine.Add(currentToken);
}while (NextToken(false).TokensLine == startLine);
}
// ... on intermediate lines
// => optimization : reuse the complete lists of the source lines
if (numberOfSelectedLines > 2)
{
for (int intermediateLineIndex = startLineIndex + 1; intermediateLineIndex < stopLineIndex; intermediateLineIndex++)
{
// TO DO - OPTIMIZATION - IMPORTANT : here, we should use an Enumerator on tokensLines
selectedTokensOnSeveralLines[intermediateLineIndex - startLineIndex] = tokensLines[intermediateLineIndex].SourceTokens;
}
currentPosition.LineIndex = stopLineIndex;
currentLine = tokensLines[currentPosition.LineIndex];
currentPosition.TokenIndexInLine = 0;
currentToken = currentLine.SourceTokens[0];
}
// ... on the last line
if (numberOfSelectedLines > 1)
{
IList <Token> tokensSelectedOnLastLine = new List <Token>();
selectedTokensOnSeveralLines[numberOfSelectedLines - 1] = tokensSelectedOnLastLine;
if (currentToken == stopToken)
{
tokensSelectedOnLastLine.Add(currentToken);
}
else
{
do
{
tokensSelectedOnLastLine.Add(currentToken);
}while (NextToken(false) != stopToken);
tokensSelectedOnLastLine.Add(currentToken);
}
NextToken(false);
}
// List of tokens not selected on the last line after stopToken
IList <Token> tokensOnLastLineAfterStopToken = null;
if (currentToken != null && currentToken.TokensLine == stopToken.TokensLine)
{
Token lastTokenOfLastLine = currentLine.SourceTokens[currentLine.SourceTokens.Count - 1];
if (stopToken != lastTokenOfLastLine)
{
tokensOnLastLineAfterStopToken = new List <Token>();
if (currentToken == lastTokenOfLastLine)
{
tokensOnLastLineAfterStopToken.Add(currentToken);
}
else
{
do
{
tokensOnLastLineAfterStopToken.Add(currentToken);
}while (NextToken(false) != lastTokenOfLastLine);
tokensOnLastLineAfterStopToken.Add(currentToken);
}
}
}
// Restore initial iterator position before return
ReturnToLastPositionSnapshot();
return(new MultilineTokensGroupSelection(startLineIndex,
tokensOnFirstLineBeforeStartToken,
selectedTokensOnSeveralLines,
tokensOnLastLineAfterStopToken));
}
/// <summary>
/// Get the previous token, with or without applying the channel filter
/// </summary>
public Token PreviousToken(bool applyChannelFilter)
{
// If the iterator already points before the first token :
// impossible to get previous token, do nothing
if (currentPosition.LineIndex == 0 && currentPosition.TokenIndexInLine == -1)
{
return(null);
}
// Look for previous tokens until we find one that matches the filter
for (; ;)
{
// Try to get the previous token on the same line
currentPosition.TokenIndexInLine -= 1;
// Current token was not the first token on his line
if (currentPosition.TokenIndexInLine >= 0)
{
// => simply return previous token on the same line
currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
}
// Current token was the first token on the line
// => look for a previous token on the preceding line
else
{
// If there is a preceding line
if (currentPosition.LineIndex > 0)
{
// => find the first preceding line which is not empty
while (currentPosition.LineIndex > 0)
{
currentPosition.LineIndex = currentPosition.LineIndex - 1;
// TO DO - OPTIMIZATION - IMPORTANT : here, we should use an Enumerator on tokensLines
currentLine = tokensLines[currentPosition.LineIndex];
if (currentLine.SourceTokens.Count > 0)
{
break;
}
}
// If such a line was found :
if (currentLine.SourceTokens.Count > 0)
{
// Select the last token on this line
currentPosition.TokenIndexInLine = currentLine.SourceTokens.Count - 1;
currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
}
else
{
// => position the iterator before the first token of the document
currentPosition.TokenIndexInLine = -1;
currentToken = null;
break;
}
}
// If there is no preceding line
else
{
// => position the iterator before the first token of the document
currentPosition.TokenIndexInLine = -1;
currentToken = null;
break;
}
}
// Check if the previous token found matches the filter criteria
if (!applyChannelFilter || currentToken == null | currentToken?.Channel == channelFilter)
{
break;
}
}
return(currentToken);
}
internal void CorrectTokensLine(ITokensLine tokensLine, int startIndex, int stopIndex)
{
this.tokensLine = tokensLine;
this.startIndex = startIndex;
this.stopIndex = stopIndex;
}
/// <summary>
/// Constructor for tokens without delimiters
/// </summary>
public Token(TokenType tokenType, int startIndex, int stopIndex, ITokensLine tokensLine) :
this(tokenType, startIndex, stopIndex, false, tokensLine)
{
}
/// <summary>
/// Get the next token, with or without applying the channel filter
/// </summary>
public Token NextToken(bool applyChannelFilter)
{
// If document is empty, immediately return EndOfFile
if (currentLine == null)
{
return Token.END_OF_FILE;
}
// While we can find a next token
currentToken = null;
while (currentToken == null)
{
// try to find the next token on the same line
currentPosition.TokenIndexInLine++;
// but if we reached the end of the current line ...
while (currentPosition.TokenIndexInLine >= currentLine.SourceTokens.Count)
{
// .. advance to next line
currentPosition.LineIndex++;
currentPosition.TokenIndexInLine = 0;
if (currentPosition.LineIndex < tokensLines.Count)
{
// TO DO - OPTIMIZATION : should we use here an Enumerator on tokensLines ?
currentLine = tokensLines[currentPosition.LineIndex];
}
// and if we reached the last line of the document ...
else
{
// return EndOfFile
currentLine = null;
return Token.END_OF_FILE;
}
}
// Check if the next token found matches the filter criteria
Token nextTokenCandidate = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
if (!applyChannelFilter || nextTokenCandidate.Channel == channelFilter)
{
currentToken = nextTokenCandidate;
}
}
// found a next token matching the filter criteria
return currentToken;
}
internal void CorrectTokensLine(ITokensLine tokensLine, int startIndex, int stopIndex)
{
this.tokensLine = tokensLine;
this.startIndex = startIndex;
this.stopIndex = stopIndex;
}
/// <summary>
/// Returns all tokens (no channel filter), line per line, selected between startToken and stopToken.
/// This method takes a snapshot of the current position before its execution and restores it before returning.
/// </summary>
public MultilineTokensGroupSelection SelectAllTokensBetween(Token startToken, Token stopToken)
{
// Check parameters
ITokensLine startLine = startToken.TokensLine;
int startLineIndex = tokensLines.IndexOf(startLine, startLine.InitialLineIndex);
ITokensLine stopLine = stopToken.TokensLine;
int stopLineIndex = tokensLines.IndexOf(stopLine, stopLine.InitialLineIndex);
if (startLineIndex < 0 || stopLineIndex < 0 || stopLineIndex < startLineIndex ||
((startLineIndex == stopLineIndex) && (stopToken.StartIndex < startToken.StartIndex)))
{
throw new InvalidOperationException("Invalid start or stop token : line or columns number do not define a valid selection interval");
}
// Save iterator position (to restore it before return)
SaveCurrentPositionSnapshot();
// Seek to the first token at the start of the line where startToken appears
currentPosition.LineIndex = startLineIndex;
currentLine = startToken.TokensLine;
currentPosition.TokenIndexInLine = 0;
currentToken = currentLine.SourceTokens[0];
// List of tokens not selected on the first line before startToken
IList<Token> tokensOnFirstLineBeforeStartToken = null;
if(currentToken != startToken)
{
tokensOnFirstLineBeforeStartToken = new List<Token>();
do
{
tokensOnFirstLineBeforeStartToken.Add(currentToken);
}
while (NextToken(false) != startToken);
}
// List of tokens selected ...
int numberOfSelectedLines = stopLineIndex - startLineIndex + 1;
IList<Token>[] selectedTokensOnSeveralLines = new IList<Token>[numberOfSelectedLines];
// ... on the first line
IList<Token> tokensSelectedOnFirstLine = new List<Token>();
selectedTokensOnSeveralLines[0] = tokensSelectedOnFirstLine;
if(numberOfSelectedLines == 1)
{
if (currentToken == stopToken)
{
tokensSelectedOnFirstLine.Add(currentToken);
}
else
{
do
{
tokensSelectedOnFirstLine.Add(currentToken);
}
while (NextToken(false) != stopToken);
tokensSelectedOnFirstLine.Add(currentToken);
}
NextToken(false);
}
else
{
do
{
tokensSelectedOnFirstLine.Add(currentToken);
}
while (NextToken(false).TokensLine == startLine);
}
// ... on intermediate lines
// => optimization : reuse the complete lists of the source lines
if(numberOfSelectedLines > 2)
{
for(int intermediateLineIndex = startLineIndex+1 ; intermediateLineIndex < stopLineIndex ; intermediateLineIndex++)
{
// TO DO - OPTIMIZATION - IMPORTANT : here, we should use an Enumerator on tokensLines
selectedTokensOnSeveralLines[intermediateLineIndex - startLineIndex] = tokensLines[intermediateLineIndex].SourceTokens;
}
currentPosition.LineIndex = stopLineIndex;
currentLine = tokensLines[currentPosition.LineIndex];
currentPosition.TokenIndexInLine = 0;
currentToken = currentLine.SourceTokens[0];
}
// ... on the last line
if (numberOfSelectedLines > 1)
{
IList<Token> tokensSelectedOnLastLine = new List<Token>();
selectedTokensOnSeveralLines[numberOfSelectedLines - 1] = tokensSelectedOnLastLine;
if (currentToken == stopToken)
{
tokensSelectedOnLastLine.Add(currentToken);
}
else
{
do
{
tokensSelectedOnLastLine.Add(currentToken);
}
while (NextToken(false) != stopToken);
tokensSelectedOnLastLine.Add(currentToken);
}
NextToken(false);
}
// List of tokens not selected on the last line after stopToken
IList<Token> tokensOnLastLineAfterStopToken = null;
if (currentToken != null && currentToken.TokensLine == stopToken.TokensLine)
{
Token lastTokenOfLastLine = currentLine.SourceTokens[currentLine.SourceTokens.Count - 1];
if (stopToken != lastTokenOfLastLine)
{
tokensOnLastLineAfterStopToken = new List<Token>();
if (currentToken == lastTokenOfLastLine)
{
tokensOnLastLineAfterStopToken.Add(currentToken);
}
else
{
do
{
tokensOnLastLineAfterStopToken.Add(currentToken);
}
while (NextToken(false) != lastTokenOfLastLine);
tokensOnLastLineAfterStopToken.Add(currentToken);
}
}
}
// Restore initial iterator position before return
ReturnToLastPositionSnapshot();
return new MultilineTokensGroupSelection(startLineIndex,
tokensOnFirstLineBeforeStartToken,
selectedTokensOnSeveralLines,
tokensOnLastLineAfterStopToken);
}
/// <summary>
/// Sets the current iterator position to point to a specific token.
/// After a call to this method, GetNextToken returns the token FOLLOWING startToken.
/// </summary>
public void SeekToToken(Token startToken)
{
// Find line for the start token
currentPosition.LineIndex = tokensLines.IndexOf(startToken.TokensLine, startToken.TokensLine.InitialLineIndex);
currentLine = startToken.TokensLine;
// Find index in line for the start token
currentPosition.TokenIndexInLine = currentLine.SourceTokens.IndexOf(startToken);
currentToken = startToken;
}
/// <summary>
/// Sets the current iterator position to a previous position returned by GetCurrentPosition.
/// After a call to this method, GetNextToken returns the token FOLLOWING the current position.
/// </summary>
public void SeekToPosition(object iteratorPosition)
{
currentPosition = (TokensLineIteratorPosition)iteratorPosition;
if (currentPosition.LineIndex >= 0 && currentPosition.LineIndex < tokensLines.Count)
{
currentLine = tokensLines[currentPosition.LineIndex];
}
else
{
currentLine = null;
}
if (currentPosition.TokenIndexInLine >= 0 && currentLine != null)
{
currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
}
else
{
currentToken = null;
}
}
/// <summary>
/// Resets the iterator position : before the first token of the document
/// </summary>
public void Reset()
{
currentPosition.LineIndex = 0;
if (tokensLines.Count > 0)
{
currentLine = tokensLines[currentPosition.LineIndex];
}
currentPosition.TokenIndexInLine = -1;
currentToken = null;
}
/// <summary>
/// Get the previous token, with or without applying the channel filter
/// </summary>
public Token PreviousToken(bool applyChannelFilter)
{
// If the iterator already points before the first token :
// impossible to get previous token, do nothing
if (currentPosition.LineIndex == 0 && currentPosition.TokenIndexInLine == -1)
{
return null;
}
// Look for previous tokens until we find one that matches the filter
for (; ; )
{
// Try to get the previous token on the same line
currentPosition.TokenIndexInLine -= 1;
// Current token was not the first token on his line
if (currentPosition.TokenIndexInLine >= 0)
{
// => simply return previous token on the same line
currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
}
// Current token was the first token on the line
// => look for a previous token on the preceding line
else
{
// If there is a preceding line
if (currentPosition.LineIndex > 0)
{
// => find the first preceding line which is not empty
while (currentPosition.LineIndex > 0)
{
currentPosition.LineIndex = currentPosition.LineIndex - 1;
// TO DO - OPTIMIZATION - IMPORTANT : here, we should use an Enumerator on tokensLines
currentLine = tokensLines[currentPosition.LineIndex];
if (currentLine.SourceTokens.Count > 0) break;
}
// If such a line was found :
if (currentLine.SourceTokens.Count > 0)
{
// Select the last token on this line
currentPosition.TokenIndexInLine = currentLine.SourceTokens.Count - 1;
currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
}
else
{
// => position the iterator before the first token of the document
currentPosition.TokenIndexInLine = -1;
currentToken = null;
break;
}
}
// If there is no preceding line
else
{
// => position the iterator before the first token of the document
currentPosition.TokenIndexInLine = -1;
currentToken = null;
break;
}
}
// Check if the previous token found matches the filter criteria
if (!applyChannelFilter || currentToken == null | currentToken.Channel == channelFilter)
{
break;
}
}
return currentToken;
}
/// <summary>
/// Constructor for tokens without delimiters
/// </summary>
internal Token(TokenType tokenType, int startIndex, int stopIndex, ITokensLine tokensLine)
: this(tokenType, startIndex, stopIndex, false, tokensLine)
{
}
/// <summary>
/// Constructor for tokens with delimiters
/// </summary>
internal Token(TokenType tokenType, int startIndex, int stopIndex, ITokensLine tokensLine, bool hasOpeningDelimiter, bool hasClosingDelimiter, char expectedClosingDelimiter) :
this(tokenType, startIndex, stopIndex, false, tokensLine, hasOpeningDelimiter, hasClosingDelimiter, expectedClosingDelimiter)
{
}
/// <summary>
/// Constructor for tokens with delimiters
/// </summary>
internal Token(TokenType tokenType, int startIndex, int stopIndex, ITokensLine tokensLine, bool hasOpeningDelimiter, bool hasClosingDelimiter, char expectedClosingDelimiter)
: this(tokenType, startIndex, stopIndex, false, tokensLine, hasOpeningDelimiter, hasClosingDelimiter, expectedClosingDelimiter)
{
}
internal ContinuationToken(Token virtualConcatenatedToken, int startIndex, int stopIndex, ITokensLine tokensLine, bool isContinuationFromPreviousLine, bool isContinuedOnNextLine) :
base(virtualConcatenatedToken.TokenType, startIndex, stopIndex, virtualConcatenatedToken.UsesVirtualSpaceAtEndOfLine, tokensLine)
{
// Store the concatenated source text
MultilineContinuationText = virtualConcatenatedToken.Text;
IsContinuationFromPreviousLine = isContinuationFromPreviousLine;
IsContinuedOnNextLine = isContinuedOnNextLine;
// Copy the delimiter properties
if (virtualConcatenatedToken.UsesDelimiters)
{
UsesDelimiters = virtualConcatenatedToken.UsesDelimiters;
HasOpeningDelimiter = virtualConcatenatedToken.HasOpeningDelimiter;
HasClosingDelimiter = virtualConcatenatedToken.HasClosingDelimiter;
ExpectedClosingDelimiter = virtualConcatenatedToken.ExpectedClosingDelimiter;
}
// Copy the literal value
LiteralValue = virtualConcatenatedToken.LiteralValue;
// Set specific Channel to enable filtering of all tokens participating in the continuation, except the first one
if (IsContinuationFromPreviousLine)
{
Channel = CHANNEL_ContinuationTokens;
}
}
private string GetIndent(ITokensLine line, int firstTokenStartIndex)
{
var lineStartIndex = line.SequenceNumberText.Length + 1;// +1 for line.IndicatorChar
return line.SourceText.Substring(0, firstTokenStartIndex-lineStartIndex);
}
private string GetIndent(ITokensLine line, int firstTokenStartIndex)
{
var lineStartIndex = line.SequenceNumberText.Length + 1; // +1 for line.IndicatorChar
return(line.SourceText.Substring(0, firstTokenStartIndex - lineStartIndex));
}
private int GetLine(ITokensLine line)
{
if (line != null) return Lines.IndexOf(line, line.InitialLineIndex)+1;
throw new System.ArgumentNullException("this.Line must be set from the source document snapshot");
}
public MissingToken(TokenType missingTokenType, string informationTextForMissingToken, ITokensLine lineWhereTokenIsMissing, int charPositionWhereTokenIsMissing) :
base(missingTokenType, charPositionWhereTokenIsMissing, charPositionWhereTokenIsMissing - 1, lineWhereTokenIsMissing)
{
InformationText = informationTextForMissingToken;
}