private void ProcessOpenStringLine(int tokenType)
{
if (_currentToken == null)
{
_currentToken =
new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
tokenType, Channel, this._tokenStartCharIndex, -1)
{
Column = _tokenStartCharPositionInLine,
TokenIndent = _indents.Peek() + 1
};
}
_currentToken.StopIndex = this.CharIndex - 1;
_currentToken.StopLine = Line;
_currentToken.StopColumn = Column - 1;
if (_input.La(1) == -1 || InWsaMode)
{
Emit(_currentToken);
ExitInValueMode();
//Emitting NEWLINE if not in WSA
if (!InWsaMode)
{
EmitIndentationToken(NEWLINE, CharIndex, CharIndex);
}
}
else
{
Skip(); EnterInOsMode();
}
}
public override void Reset()
{
InvalidTokens = new List <IToken>();
_indents = new Stack <int>(new[] { 0 });
_tokens = new Queue <IToken>();
_wsaStack = new Stack <int>();
_currentToken = null;
base.Reset();
}
private void StartSqs()
{
var token =
new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
SQS, Channel, _tokenStartCharIndex, -1)
{
TokenIndent = _indents.Peek() + 1,
Column = _tokenStartCharPositionInLine + 1,
StopIndex = CharIndex - 1
};
Emit(token);
}
private void StartDqsMl()
{
_currentToken =
new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), DQS,
Channel,
_tokenStartCharIndex, -1)
{
TokenIndent = _indents.Peek() + 1,
Column = _tokenStartCharPositionInLine + 1
};
PushMode(IN_DQS);
EndDqsIfEofOrWsa();
}
/// <summary>
/// COLON token is emitted only if colon follows an ID with no space between them.
/// Otherwise token ARRAY_ITEM is emitted by ARRAY_ITEM lexer rule.
/// </summary>
/// <param name="tokenType"></param>
private void EmitIdWithColon(int tokenType)
{
if (InputStream.La(-1) == ':')
{
var token =
new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
tokenType, Channel, _tokenStartCharIndex, CharIndex - 2)
{
Line = _tokenStartLine,
StopLine = _tokenStartLine,
Column = _tokenStartCharPositionInLine,
StopColumn = Column - 2
};
Emit(token);
token = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
COLON, Channel, CharIndex - 1, CharIndex - 1)
{
Line = _tokenStartLine,
StopLine = _tokenStartLine,
Column = Column - 1,
StopColumn = Column - 1
};
Emit(token);
}
else
{
var token =
new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
tokenType, Channel, _tokenStartCharIndex, CharIndex - 1)
{
Line = _tokenStartLine,
StopLine = _tokenStartLine,
Column = _tokenStartCharPositionInLine,
StopColumn = Column - 1
};
Emit(token);
}
}
public override IToken NextToken()
{
//Return previously generated tokens first
if (_tokens.Count > 0)
{
return(_tokens.Dequeue());
}
Token = null;
//Checking if EOF and still mode = IN_VALUE then we need to generate OPEN_VALUE
//Scenario: Open value ends with EOF
if (_input.La(1) == Eof && _mode == 1)
{
if (_currentToken == null)//Empty Open String
{
//Creating Token for Empty OpenString
_currentToken =
new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
OPEN_STRING, Channel, -1, -1)
{
Text = ""
};
Emit(_currentToken);
}
ExitInValueMode();
}
if (_tokens.Count > 0)
{
//Return generated tokens
return(_tokens.Dequeue());
}
//Run regular path if there no extra tokens generated in queue
return(base.NextToken());
}
//Open String Indents/Dedents processing
private void OsIndentDedent(int tokenType, int mlTokenType)
{
var currentIndent = InWsaMode ? 0 : _indents.Peek();
var indent = CalcOsIndent();
//If dedent found check if this is comment and ignore it
if (!(indent > currentIndent) && CurrentLineEndsWithComment())
{
return;
}
if (indent == currentIndent)
{
//Emitting NEWLINE if OS is not ended by ==
if (_input.La(-1) != '=')
{
if (_currentToken == null)//Empty Open String
{
//Creating Token for Empty OpenString
_currentToken =
new MalinaToken(
new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
tokenType, Channel, -1, -1)
{
Text = ""
};
}
Emit(_currentToken);
if (!InWsaMode)
{
EmitIndentationToken(NEWLINE, CharIndex - indent - 1, CharIndex - indent - 1);
}
}
else
{
if (_currentToken == null)
{
//Creating Token
_currentToken = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), OPEN_STRING, Channel, _tokenStartCharIndex, -1);
}
//if value was ended with == then we need to add \n
_currentToken.StopIndex = CharIndex - Column - 1;
_currentToken.StopLine = this._tokenStartLine;
_currentToken.Type = mlTokenType;
_currentToken.StopColumn++;
Emit(_currentToken);
}
ExitInValueMode();
}
else if (indent > currentIndent)
{
if (_currentToken == null)
{
//If Open String starts with empty string then ignore first line
var offset = 0;
if (InputStream.La(_tokenStartCharIndex - CharIndex) == '\r')
{
offset++;
if (InputStream.La(_tokenStartCharIndex - CharIndex + 1) == '\n')
{
offset++;
}
}
else if (InputStream.La(_tokenStartCharIndex - CharIndex) == '\n')
{
offset++;
}
if (offset > 0)
{
offset += _indents.Peek() + 1;
}
_currentToken =
new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
tokenType, Channel, this._tokenStartCharIndex + offset, -1)
{
TokenIndent = _indents.Peek() + 1
};
}
if (_input.La(-1) == '=')//If indent ends with == then include it because it was consumed by fragment OpenStringEol
{
_currentToken.StopIndex = this.CharIndex - 1;
_currentToken.StopLine = Line;
_currentToken.StopColumn = Column - 1;
}
//Indent before EOF and in WsaMode should end OS
if (_input.La(1) == -1 || InWsaMode)
{
Emit(_currentToken);
ExitInValueMode();
//Emitting NEWLINE if not in WSA
if (!InWsaMode)
{
EmitIndentationToken(NEWLINE, CharIndex, CharIndex);
}
}
_currentToken.Type = mlTokenType;
Skip();
EnterInOsMode();
}
else
{
if (_currentToken == null)//Empty Open String Scenario
{
//Creating Token for Empty OpenString
_currentToken =
new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream),
tokenType, Channel, -1, -1)
{
Text = ""
};
}
//Adding 1 NEWLINE before DEDENTS
Emit(_currentToken);
EmitIndentationToken(NEWLINE, CharIndex - indent - 1, CharIndex - indent - 1);
//Emitting 1 or more DEDENTS
while (_indents.Count > 1 && _indents.Peek() > indent)
{
EmitIndentationToken(DEDENT, CharIndex - indent, CharIndex - 1);
_indents.Pop();
}
ExitInValueMode();
}
}
private void StartNewMultiLineToken()
{
_currentToken = null;
}
