private void ProcessOpenStringLine(int tokenType) { if (_currentToken == null) { _currentToken = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), tokenType, Channel, this._tokenStartCharIndex, -1) { Column = _tokenStartCharPositionInLine, TokenIndent = _indents.Peek() + 1 }; } _currentToken.StopIndex = this.CharIndex - 1; _currentToken.StopLine = Line; _currentToken.StopColumn = Column - 1; if (_input.La(1) == -1 || InWsaMode) { Emit(_currentToken); ExitInValueMode(); //Emitting NEWLINE if not in WSA if (!InWsaMode) { EmitIndentationToken(NEWLINE, CharIndex, CharIndex); } } else { Skip(); EnterInOsMode(); } }
public override void Reset() { InvalidTokens = new List <IToken>(); _indents = new Stack <int>(new[] { 0 }); _tokens = new Queue <IToken>(); _wsaStack = new Stack <int>(); _currentToken = null; base.Reset(); }
private void StartSqs() { var token = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), SQS, Channel, _tokenStartCharIndex, -1) { TokenIndent = _indents.Peek() + 1, Column = _tokenStartCharPositionInLine + 1, StopIndex = CharIndex - 1 }; Emit(token); }
private void StartDqsMl() { _currentToken = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), DQS, Channel, _tokenStartCharIndex, -1) { TokenIndent = _indents.Peek() + 1, Column = _tokenStartCharPositionInLine + 1 }; PushMode(IN_DQS); EndDqsIfEofOrWsa(); }
/// <summary> /// COLON token is emitted only if colon follows an ID with no space between them. /// Otherwise token ARRAY_ITEM is emitted by ARRAY_ITEM lexer rule. /// </summary> /// <param name="tokenType"></param> private void EmitIdWithColon(int tokenType) { if (InputStream.La(-1) == ':') { var token = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), tokenType, Channel, _tokenStartCharIndex, CharIndex - 2) { Line = _tokenStartLine, StopLine = _tokenStartLine, Column = _tokenStartCharPositionInLine, StopColumn = Column - 2 }; Emit(token); token = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), COLON, Channel, CharIndex - 1, CharIndex - 1) { Line = _tokenStartLine, StopLine = _tokenStartLine, Column = Column - 1, StopColumn = Column - 1 }; Emit(token); } else { var token = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), tokenType, Channel, _tokenStartCharIndex, CharIndex - 1) { Line = _tokenStartLine, StopLine = _tokenStartLine, Column = _tokenStartCharPositionInLine, StopColumn = Column - 1 }; Emit(token); } }
public override IToken NextToken() { //Return previously generated tokens first if (_tokens.Count > 0) { return(_tokens.Dequeue()); } Token = null; //Checking if EOF and still mode = IN_VALUE then we need to generate OPEN_VALUE //Scenario: Open value ends with EOF if (_input.La(1) == Eof && _mode == 1) { if (_currentToken == null)//Empty Open String { //Creating Token for Empty OpenString _currentToken = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), OPEN_STRING, Channel, -1, -1) { Text = "" }; Emit(_currentToken); } ExitInValueMode(); } if (_tokens.Count > 0) { //Return generated tokens return(_tokens.Dequeue()); } //Run regular path if there no extra tokens generated in queue return(base.NextToken()); }
//Open String Indents/Dedents processing private void OsIndentDedent(int tokenType, int mlTokenType) { var currentIndent = InWsaMode ? 0 : _indents.Peek(); var indent = CalcOsIndent(); //If dedent found check if this is comment and ignore it if (!(indent > currentIndent) && CurrentLineEndsWithComment()) { return; } if (indent == currentIndent) { //Emitting NEWLINE if OS is not ended by == if (_input.La(-1) != '=') { if (_currentToken == null)//Empty Open String { //Creating Token for Empty OpenString _currentToken = new MalinaToken( new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), tokenType, Channel, -1, -1) { Text = "" }; } Emit(_currentToken); if (!InWsaMode) { EmitIndentationToken(NEWLINE, CharIndex - indent - 1, CharIndex - indent - 1); } } else { if (_currentToken == null) { //Creating Token _currentToken = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), OPEN_STRING, Channel, _tokenStartCharIndex, -1); } //if value was ended with == then we need to add \n _currentToken.StopIndex = CharIndex - Column - 1; _currentToken.StopLine = this._tokenStartLine; _currentToken.Type = mlTokenType; _currentToken.StopColumn++; Emit(_currentToken); } ExitInValueMode(); } else if (indent > currentIndent) { if (_currentToken == null) { //If Open String starts with empty string then ignore first line var offset = 0; if (InputStream.La(_tokenStartCharIndex - CharIndex) == '\r') { offset++; if (InputStream.La(_tokenStartCharIndex - CharIndex + 1) == '\n') { offset++; } } else if (InputStream.La(_tokenStartCharIndex - CharIndex) == '\n') { offset++; } if (offset > 0) { offset += _indents.Peek() + 1; } _currentToken = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), tokenType, Channel, this._tokenStartCharIndex + offset, -1) { TokenIndent = _indents.Peek() + 1 }; } if (_input.La(-1) == '=')//If indent ends with == then include it because it was consumed by fragment OpenStringEol { _currentToken.StopIndex = this.CharIndex - 1; _currentToken.StopLine = Line; _currentToken.StopColumn = Column - 1; } //Indent before EOF and in WsaMode should end OS if (_input.La(1) == -1 || InWsaMode) { Emit(_currentToken); ExitInValueMode(); //Emitting NEWLINE if not in WSA if (!InWsaMode) { EmitIndentationToken(NEWLINE, CharIndex, CharIndex); } } _currentToken.Type = mlTokenType; Skip(); EnterInOsMode(); } else { if (_currentToken == null)//Empty Open String Scenario { //Creating Token for Empty OpenString _currentToken = new MalinaToken(new Tuple <ITokenSource, ICharStream>(this, (this as ITokenSource).InputStream), tokenType, Channel, -1, -1) { Text = "" }; } //Adding 1 NEWLINE before DEDENTS Emit(_currentToken); EmitIndentationToken(NEWLINE, CharIndex - indent - 1, CharIndex - indent - 1); //Emitting 1 or more DEDENTS while (_indents.Count > 1 && _indents.Peek() > indent) { EmitIndentationToken(DEDENT, CharIndex - indent, CharIndex - 1); _indents.Pop(); } ExitInValueMode(); } }
private void StartNewMultiLineToken() { _currentToken = null; }