private Tokens TokenizeExpandingHeredocContent(HeredocTokenizer/*!*/ heredoc) { MutableStringBuilder content; int c = Peek(); if (c == '#') { Skip(c); switch (Peek()) { case '$': case '@': MarkSingleLineTokenEnd(); return StringEmbeddedVariableBegin(); case '{': Skip('{'); MarkSingleLineTokenEnd(); return StringEmbeddedCodeBegin(); } content = new MutableStringBuilder(_encoding); content.Append('#'); } else { content = new MutableStringBuilder(_encoding); } bool isIndented = (heredoc.Properties & StringType.IndentedHeredoc) != 0; do { // read string content upto the end of the line: int tmp = 0; c = ReadStringContent(content, heredoc.Properties, '\n', 0, ref tmp); // stop reading on end-of-file or just before an embedded expression: #$, #$, #{ if (c != '\n') { break; } // adds \n content.Append((char)ReadNormalizeEndOfLine()); // TODO: RefillBuffer(); // first char on the next line: if (Peek() == -1) { break; } } while (!LineContentEquals(heredoc.Label, isIndented)); _tokenValue.SetStringContent(content); MarkMultiLineTokenEnd(); return Tokens.StringContent; }
private StringBuilder/*!*/ ReadNonexpandingHeredocContent(HeredocTokenizer/*!*/ heredoc) { bool isIndented = (heredoc.Properties & StringType.IndentedHeredoc) != 0; var result = new StringBuilder(); // reads lines until the line contains heredoc label do { int end = _lineLength; if (end > 0) { switch (_lineBuffer[end - 1]) { case '\n': if (--end == 0 || _lineBuffer[end - 1] != '\r') { end++; break; } --end; break; case '\r': --end; break; } } result.Append(_lineBuffer, 0, end); if (end < _lineLength) { result.Append('\n'); } _bufferPos = _lineLength; // force new line load: RefillBuffer(); if (Peek() == -1) { // eof reached before end of heredoc: return result; } } while (!LineContentEquals(heredoc.Label, isIndented)); // return to the end of line, next token will be StringEnd spanning over the end-of-heredoc label: _bufferPos = 0; return result; }
internal Tokens TokenizeHeredoc(HeredocTokenizer/*!*/ heredoc) { StringType stringKind = heredoc.Properties; bool isIndented = (stringKind & StringType.IndentedHeredoc) != 0; MarkTokenStart(); if (Peek() == -1) { ReportError(Errors.UnterminatedHereDoc, heredoc.Label); MarkSingleLineTokenEnd(); HeredocRestore(heredoc); _unterminatedToken = true; return Tokens.StringEnd; } // label reached - it becomes a string-end token: // (note that label is single line, MRI allows multiline, but such label is never matched) if (is_bol() && LineContentEquals(heredoc.Label, isIndented)) { // seek to the end of the line: SeekRelative(heredoc.Label.Length); MarkSingleLineTokenEnd(); HeredocRestore(heredoc); // Zero-width token end immediately follows the heredoc opening label. // Prevents parser confusion when merging locations. // // [<<END][zero-width string end] ... other tokens ... // ... heredoc content tokens ... // END // MarkTokenStart(); MarkSingleLineTokenEnd(); return Tokens.StringEnd; } if ((stringKind & StringType.ExpandsEmbedded) == 0) { StringBuilder str = ReadNonexpandingHeredocContent(heredoc); // do not restore buffer, the next token query will invoke 'if (EOF)' or 'if (line contains label)' above: SetStringToken(str.ToString()); MarkMultiLineTokenEnd(); return Tokens.StringContent; } return TokenizeExpandingHeredocContent(heredoc); }
private void HeredocRestore(HeredocTokenizer/*!*/ here) { _lineBuffer = here.ResumeLine; _lineLength = here.ResumeLineLength; _bufferPos = here.ResumePosition; _heredocEndLine = _currentLine; _heredocEndLineIndex = _currentLineIndex; _currentLine = here.FirstLine; _currentLineIndex = here.FirstLineIndex; }
private Tokens TokenizeExpandingHeredocContent(HeredocTokenizer/*!*/ heredoc) { newtok(); int c = nextc(); if (c == '#') { c = nextc(); switch (c) { case '$': case '@': pushback(c); MarkSingleLineTokenEnd(); return StringEmbeddedVariableBegin(); case '{': MarkSingleLineTokenEnd(); return StringEmbeddedCodeBegin(); } tokadd('#'); } bool isIndented = (heredoc.Properties & StringType.IndentedHeredoc) != 0; pushback(c); bool hasUnicodeEscape = false; do { // read string content upto the end of the line: int tmp = 0; c = ReadStringContent(heredoc.Properties, '\n', 0, ref tmp, ref hasUnicodeEscape); // stop reading on end-of-file or just before an embedded expression: #$, #$, #{ if (c != '\n') { break; } // adds \n tokadd((char)nextc()); // first char on the next line: if (peekc() == -1) { break; } } while (!LineContentEquals(heredoc.Label, isIndented)); _tokenValue.SetString(tok(), hasUnicodeEscape); MarkMultiLineTokenEnd(); return Tokens.StringContent; }
internal Tokens TokenizeHeredoc(HeredocTokenizer/*!*/ heredoc) { StringType stringKind = heredoc.Properties; bool isIndented = (stringKind & StringType.IndentedHeredoc) != 0; int c = peekc(); MarkTokenStart(); if (c == -1) { ReportError(Errors.UnterminatedHereDoc, heredoc.Label); MarkSingleLineTokenEnd(); HeredocRestore(heredoc); UnterminatedToken = true; return Tokens.StringEnd; } // label reached - it becomes a string-end token: // (note that label is single line, MRI allows multiline, but such label is never matched) if (is_bol() && LineContentEquals(heredoc.Label, isIndented)) { // TODO: reads the entire label: do { c = nextc(); } while (c != '\n' && c != -1); pushback(c); MarkSingleLineTokenEnd(); HeredocRestore(heredoc); // Zero-width token end immediately follows the heredoc opening label. // Prevents parser confusion when merging locations. // // [<<END][zero-width string end] ... other tokens ... // ... heredoc content tokens ... // END // MarkTokenStart(); MarkSingleLineTokenEnd(); return Tokens.StringEnd; } if ((stringKind & StringType.ExpandsEmbedded) == 0) { StringBuilder str = ReadNonexpandingHeredocContent(heredoc); // do not restore buffer, the next token query will invoke 'if (EOF)' or 'if (line contains label)' above: _tokenValue.SetString(str.ToString(), false); MarkMultiLineTokenEnd(); return Tokens.StringContent; } return TokenizeExpandingHeredocContent(heredoc); // obsolete: //MarkMultiLineTokenEnd(); //HeredocRestore(heredoc); //_currentString = new StringTerminator(StringType.FinalWordSeparator, 0, 0); //_tokenValue.SetString(str); //return Tokens.StringContent; }