public override IToken Emit() { if ((Type == ANTLRv4Lexer.OPTIONS || Type == ANTLRv4Lexer.TOKENS || Type == ANTLRv4Lexer.CHANNELS) && CurrentRuleType == TokenConstants.InvalidType) { // enter prequel construct ending with an RBRACE CurrentRuleType = PREQUEL_CONSTRUCT; } else if (Type == ANTLRv4Lexer.RBRACE && CurrentRuleType == PREQUEL_CONSTRUCT) { // exit prequel construct CurrentRuleType = TokenConstants.InvalidType; } else if (Type == ANTLRv4Lexer.AT && CurrentRuleType == TokenConstants.InvalidType) { // enter action CurrentRuleType = ANTLRv4Lexer.AT; } else if (Type == ANTLRv4Lexer.END_ACTION && CurrentRuleType == ANTLRv4Lexer.AT) { // exit action CurrentRuleType = TokenConstants.InvalidType; } else if (Type == ANTLRv4Lexer.ID) { char firstChar = stream.GetText(Interval.Of(TokenStartCharIndex, TokenStartCharIndex))[0]; if (char.IsUpper(firstChar)) { Type = ANTLRv4Lexer.TOKEN_REF; } else { Type = ANTLRv4Lexer.RULE_REF; } if (CurrentRuleType == TokenConstants.InvalidType) { // if outside of rule def CurrentRuleType = Type; // set to inside lexer or parser rule } } else if (Type == ANTLRv4Lexer.SEMI) { // exit rule def CurrentRuleType = TokenConstants.InvalidType; } return(base.Emit()); }
protected virtual bool TokenEndsAtEndOfLine(ITextSnapshot snapshot, ITokenSourceWithState <TState> lexer, IToken token) { ICharStream charStream = lexer.CharStream; if (charStream != null) { int nextCharIndex = token.StopIndex + 1; if (nextCharIndex >= charStream.Size) { return(true); } int c = charStream.GetText(new Interval(token.StopIndex + 1, token.StopIndex + 1))[0]; return(c == '\r' || c == '\n'); } ITextSnapshotLine line = snapshot.GetLineFromPosition(token.StopIndex + 1); return(line.End <= token.StopIndex + 1 && line.EndIncludingLineBreak >= token.StopIndex + 1); }
public string GetText(Interval interval) { return(stream.GetText(interval)); }
/** Get the text matched so far for the current token. */ public String GetText(ICharStream input) { // index is first lookahead char, don't include. return(input.GetText(Interval.Of(startIndex, input.Index - 1))); }
public virtual string GetText(ICharStream input) { // index is first lookahead char, don't include. return input.GetText(Interval.Of(startIndex, input.Index - 1)); }
public static string GetValueFromValueInterval(ICharStream charStream, Interval valueInterval, int valueIndent, ValueType valueType) { if (valueInterval.Length == 0) { return(null); //Node has no value } if (valueInterval.a == -1) { return(""); //Node has empty value } var _sb = new StringBuilder(); //Splitting text. Getting array of text lines var lines = charStream.GetText(valueInterval).Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None); var first = true; var firstEmptyLine = true; //If true then previous line was not empty therefor newline shouldn't be added foreach (var item in lines) { string line = TrimEndOfOpenStringLine(item, valueType); if (first) { _sb.Append(line); first = false; continue; } //adding first line without appending new line symbol //Counting indent of line var lineIndent = line.TakeWhile(c => c == ' ' || c == '\t').Count(); //Ignore dedented comments inside open string if (lineIndent < valueIndent && line.Substring(lineIndent).StartsWith("//")) { continue; } if (line.Length <= valueIndent) //this is just empty line { if (valueType == ValueType.FreeOpenString) //Folded string { if (firstEmptyLine) { firstEmptyLine = false; continue; //Ignore first empty line for folded string } } _sb.AppendLine(); continue; } line = line.Substring(valueIndent); if (valueType == ValueType.FreeOpenString && firstEmptyLine) { _sb.Append(" "); } if (valueType != ValueType.FreeOpenString || !firstEmptyLine) { _sb.AppendLine(); } firstEmptyLine = true; //reseting the flag for folded string logic _sb.Append(line); //Removing indents } return(_sb.ToString()); }
/// <summary> /// program_shorthand : SYM_DOLLAR ~EOL* ; /// </summary> override public Shell.Types.IShellReturnable VisitProgram_shorthand(ShellParser.Program_shorthandContext context) { string execString = tokenSource.GetText( new Antlr4.Runtime.Misc.Interval( context.Start.StartIndex, context.Stop.StopIndex ) ) .Substring(1); List <string> argsList = execString.Split(' ').ToList(); if (argsList.First().Length == 0) { argsList.RemoveAt(0); } List <string> nargs = new List <string>(); // Join any arguments if they are valid substitutions for (int i = 0; i < argsList.Count; i++) { bool added = false; var arg = argsList[i]; for (int j = i + 1; j < argsList.Count; j++) { if (arg.StartsWith("\"") && argsList[j].EndsWith("\"")) { nargs.Add(String.Join(" ", argsList.GetRange(i, j))); i = j + 1; added = true; break; } else if (arg.StartsWith('`') && argsList[j].EndsWith('`')) { nargs.Add(String.Join(" ", argsList.GetRange(i, j))); i = j + 1; added = true; break; } } if (!added) { nargs.Add(arg); } } argsList = nargs; Process process = new Process(); process.StartInfo.FileName = argsList.First(); argsList = argsList.Skip(1).Select( str => { // perform any substitutions if (str.StartsWith("\"") && str.EndsWith("\"")) { return(Utility.GetString(str, this).ToString()); } else if (str.StartsWith('`') && str.EndsWith('`')) { return(str[1..^ 1]); }
public static void Check_CobolCharStream() { // Test file properties string relativePath = @"Compiler\Parser\Samples"; string textName = "MSVCOUT"; DocumentFormat documentFormat = DocumentFormat.RDZReferenceFormat; // Compile test file CompilationDocument compilationDocument = ParserUtils.ScanCobolFile(relativePath, textName, documentFormat); // Create a token iterator on top of tokens lines TokensLinesIterator tokensIterator = new TokensLinesIterator( compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name, compilationDocument.TokensDocumentSnapshot.Lines, null, Token.CHANNEL_SourceTokens); // Crate an Antlr compatible token source on top a the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name, tokensIterator); tokenSource.NextToken(); // Get underlying CharStream ICharStream charStream = tokenSource.InputStream; if (charStream.Index != 0) { throw new Exception("Char stream index should start at 0"); } if (charStream.La(0) != 0) { throw new Exception("La(0) should be 0"); } if (charStream.La(1) != '0') { throw new Exception("La(1) should be 0"); } if (charStream.La(4) != '1') { throw new Exception("La(4) should be 1"); } if (charStream.La(5) != '6') { throw new Exception("La(5) should be 6"); } charStream.Consume(); if (charStream.Index != 1) { throw new Exception("Char stream index should be 1 after consume"); } if (charStream.La(4) != '6') { throw new Exception("La(4) should be 6 after consume"); } if (charStream.La(80) != IntStreamConstants.Eof) { throw new Exception("La(80) should be Eof"); } charStream.Seek(12); if (charStream.Index != 12) { throw new Exception("Char stream index should be 12 after seek"); } if (charStream.La(-1) != ':') { throw new Exception("La(-1) should be : after seek"); } if (charStream.La(1) != 'M') { throw new Exception("La(1) should be M after seek"); } // should do nothing int marker = charStream.Mark(); charStream.Release(marker); if (charStream.La(2) != 'S') { throw new Exception("La(2) should be S after release"); } string text = charStream.GetText(new Interval(11, 18)); if (text != ":MSVCOUT") { throw new Exception("Char stream GetText method KO"); } if (charStream.Size != 80) { throw new Exception("Char stream size KO"); } }
public string GetText(Interval interval) => internalStream.GetText(interval);
public string GetText([NotNull] Interval interval) { return(stream.GetText(interval)); }
public String GetText(Interval interval) { return(_source.GetText(interval)); }