protected internal virtual int ExecATN(ICharStream input, DFAState ds0)
{
//System.out.println("enter exec index "+input.index()+" from "+ds0.configs);
int t = input.La(1);
DFAState s = ds0;
// s is current/from DFA state
while (true)
{
// while more work
// As we move src->trg, src->trg, we keep track of the previous trg to
// avoid looking up the DFA state again, which is expensive.
// If the previous target was already part of the DFA, we might
// be able to avoid doing a reach operation upon t. If s!=null,
// it means that semantic predicates didn't prevent us from
// creating a DFA state. Once we know s!=null, we check to see if
// the DFA state has an edge already for t. If so, we can just reuse
// it's configuration set; there's no point in re-computing it.
// This is kind of like doing DFA simulation within the ATN
// simulation because DFA simulation is really just a way to avoid
// computing reach/closure sets. Technically, once we know that
// we have a previously added DFA state, we could jump over to
// the DFA simulator. But, that would mean popping back and forth
// a lot and making things more complicated algorithmically.
// This optimization makes a lot of sense for loops within DFA.
// A character will take us back to an existing DFA state
// that already has lots of edges out of it. e.g., .* in comments.
DFAState target = GetExistingTargetState(s, t);
if (target == null)
{
target = ComputeTargetState(input, s, t);
}
if (target == Error)
{
break;
}
if (target.isAcceptState)
{
CaptureSimState(prevAccept, input, target);
if (t == IntStreamConstants.Eof)
{
break;
}
}
if (t != IntStreamConstants.Eof)
{
Consume(input);
t = input.La(1);
}
s = target;
}
// flip; current DFA target becomes new src/from state
return(FailOrAccept(prevAccept, input, s.configs, t));
}
public JavaUnicodeInputStream([NotNull] ICharStream source)
{
if (source == null)
{
throw new ArgumentNullException("source");
}
this._source = source;
this._la1 = source.La(1);
}
public static int FindChar(int start, int stop, ICharStream input, char c)
{
for (int i = start - input.Index + 1; i <= stop - input.Index; i++)
{
if (input.La(i) == c)
{
return(input.Index + i);
}
}
return(-1);
}
public static bool ConsumeMlComment(this ICharStream stream, IPairFactory pairFactory, Pair parent)
{
if (stream.Next != '\"')
{
return(false);
}
if (stream.La(2) != '\"')
{
return(false);
}
if (stream.La(3) != '\"')
{
return(false);
}
stream.Consume();
var begin = new CharLocation(stream);
stream.Consume();
stream.Consume();
while (!(stream.Next == '\"' && stream.La(2) == '\"' && stream.La(3) == '\"') && stream.Next != -1)
{
stream.Consume();
}
if (stream.Next == '\"')
{
stream.Consume();
stream.Consume();
stream.Consume();
}
var comment = pairFactory.ProcessComment((ITextSource)stream, 2, new Interval(begin, new CharLocation(stream)));
if (comment != null)
{
pairFactory.AppendChild(parent, comment);
}
return(true);
}
public static bool ConsumeSlComment(this ICharStream stream, IPairFactory pairFactory, Pair parent)
{
if (stream.Next != '\'')
{
return(false);
}
if (stream.La(2) != '\'')
{
return(false);
}
if (stream.La(3) != '\'')
{
return(false);
}
stream.Consume();
var begin = new CharLocation(stream);
stream.Consume();
stream.Consume();
var c = stream.Next;
while (!c.IsNewLineCharacter() && c != -1)
{
stream.Consume();
c = stream.Next;
}
var comment = pairFactory.ProcessComment((ITextSource)stream, 1, new Interval(begin, new CharLocation(stream)));
if (comment != null)
{
pairFactory.AppendChild(parent, comment);
}
return(true);
}
protected internal virtual void Accept(ICharStream input, LexerActionExecutor lexerActionExecutor, int startIndex, int index, int line, int charPos)
{
// seek to after last char in token
input.Seek(index);
this.line = line;
this.charPositionInLine = charPos;
if (input.La(1) != IntStreamConstants.Eof)
{
Consume(input);
}
if (lexerActionExecutor != null && recog != null)
{
lexerActionExecutor.Execute(recog, input, startIndex);
}
}
public virtual void Consume(ICharStream input)
{
int curChar = input.La(1);
if (curChar == '\n')
{
_line++;
charPositionInLine = 0;
}
else
{
charPositionInLine++;
}
input.Consume();
}
protected internal virtual void Accept(ICharStream input, int ruleIndex, int actionIndex
, int index, int line, int charPos)
{
if (actionIndex >= 0 && recog != null)
{
recog.Action(null, ruleIndex, actionIndex);
}
// seek to after last char in token
input.Seek(index);
this.line = line;
this.charPositionInLine = charPos;
if (input.La(1) != IntStreamConstants.Eof)
{
Consume(input);
}
}
public int La(int i)
{
int c = stream.La(i);
if (c \ <= 0)
{
return(c);
}
char o = (char)c;
if (upper)
{
return((int)char.ToUpperInvariant(o));
}
return((int)char.ToLowerInvariant(o));
}
public int La(int i)
{
int c = stream.La(i);
if (c <= 0)
{
return(c);
}
char o = (char)c;
if (convertFromNonUS)
{
if (o == ';')
{
return((int)',');
}
if (o == ',')
{
return((int)'.');
}
}
return((int)char.ToUpperInvariant(o));
}
public static int LA(this ICharStream self, int i)
{
return(self.La(i: i));
}
protected internal virtual int ExecATN(ICharStream input, DFAState ds0)
{
//System.out.println("enter exec index "+input.index()+" from "+ds0.configs);
int t = input.La(1);
DFAState s = ds0;
// s is current/from DFA state
while (true)
{
// while more work
// As we move src->trg, src->trg, we keep track of the previous trg to
// avoid looking up the DFA state again, which is expensive.
// If the previous target was already part of the DFA, we might
// be able to avoid doing a reach operation upon t. If s!=null,
// it means that semantic predicates didn't prevent us from
// creating a DFA state. Once we know s!=null, we check to see if
// the DFA state has an edge already for t. If so, we can just reuse
// it's configuration set; there's no point in re-computing it.
// This is kind of like doing DFA simulation within the ATN
// simulation because DFA simulation is really just a way to avoid
// computing reach/closure sets. Technically, once we know that
// we have a previously added DFA state, we could jump over to
// the DFA simulator. But, that would mean popping back and forth
// a lot and making things more complicated algorithmically.
// This optimization makes a lot of sense for loops within DFA.
// A character will take us back to an existing DFA state
// that already has lots of edges out of it. e.g., .* in comments.
DFAState target = GetExistingTargetState(s, t);
if (target == null)
{
target = ComputeTargetState(input, s, t);
}
if (target == Error)
{
break;
}
if (target.isAcceptState)
{
CaptureSimState(prevAccept, input, target);
if (t == IntStreamConstants.Eof)
{
break;
}
}
if (t != IntStreamConstants.Eof)
{
Consume(input);
t = input.La(1);
}
s = target;
}
// flip; current DFA target becomes new src/from state
return FailOrAccept(prevAccept, input, s.configs, t);
}
public static int LA(this ICharStream stream, int i)
{
return(stream.La(i));
}
/// <summary>
/// Consumes indent of the multiline string line.
/// Returns true if indent is greater than current indent (ml string continues)
/// </summary>
/// <returns></returns>
private void ParseMlStringIndent()
{
var currentPair = _lineState.CurrentPair;
var indent = GetIndent(_lineState.Indent, currentPair, out var endedByComment);
if (_input.Next != -1 &&
indent > _lineState.Indent && // Indent is greater than current indent (ml string continues).
!endedByComment //Dedent in comments ends ML string
)
{
ParseMlValue();
return;
}
// At this point we know that ml string is ended.
var valueStart = currentPair.ValueQuotesType;
if (valueStart > 0)
{
//Quoted string ended by indentation (missing quote)
currentPair.MissingValueQuote = true;
//newPair = AppendCurrentPair();
ReportMLSSyntaxError(1, new Interval(currentPair.ValueInterval.End, currentPair.ValueInterval.End),
valueStart);
}
else
{
//Multiline Open string
if (indent == _lineState.Indent && _input.Next == '=' && _input.La(2) == '=' && _input.La(3) == '=')
{
//Checking if it is ended with ===
_input.Consume(); _input.Consume(); _input.Consume(); //Consuming ===
currentPair.ValueInterval = new Interval(currentPair.ValueInterval.Begin, new CharLocation(_input));
}
}
var newPair = AppendCurrentPair(); //Creating the new pair and appending it to parent
//Reporting end of pair
if (_input.Next != -1)
{
_pairFactory.EndPair(newPair, new Interval(GetPairEnd((IMappedPair)newPair)));
}
else
{
_pairFactory.EndPair(newPair,
indent <= _lineState.Indent
? new Interval(GetPairEnd((IMappedPair)newPair))
: new Interval(_input),
_lineState.State == ParserStateEnum.Value ||
indent > _lineState.Indent); //Special case used in completion. Value context. True- means value is ended by EOF but not by dedent.
}
_lineState.Indent = indent; //Saving new value of indent in the lineState
//Ending pair with bigger indents.
while (_pairStack.Peek().Indent >= indent)
{
EndPair(new Interval(_input));
}
//Checking for BlockIndentationMismatch error
if (_input.Next != -1 && //ignore indent mismatch in the EOF
_pairStack.Peek().BlockIndent != indent)
{
ReportBlockIndentationMismatch(new Interval(new CharLocation(_input.Line, 1, _input.Index - indent), new CharLocation(_input)));
}
_lineState.State = ParserStateEnum.PairDelimiter;
}
public void Consume()
{
if (_la1 != '\\')
{
_source.Consume();
_la1 = _source.La(1);
_range = Math.Max(_range, _source.Index);
_slashCount = 0;
return;
}
// make sure the next character has been processed
this.La(1);
if (_escapeListIndex >= _escapeIndexes.Count || _escapeIndexes[_escapeListIndex] != Index)
{
_source.Consume();
_slashCount++;
}
else
{
int indirectionLevel = _escapeIndirectionLevels[_escapeListIndex];
for (int i = 0; i < 6 + indirectionLevel; i++)
{
_source.Consume();
}
_escapeListIndex++;
_slashCount = 0;
}
_la1 = _source.La(1);
Debug.Assert(_range >= Index);
}
protected internal virtual void Accept(ICharStream input, LexerActionExecutor lexerActionExecutor, int startIndex, int index, int line, int charPos)
{
// seek to after last char in token
input.Seek(index);
this.line = line;
this.charPositionInLine = charPos;
if (input.La(1) != IntStreamConstants.Eof)
{
Consume(input);
}
if (lexerActionExecutor != null && recog != null)
{
lexerActionExecutor.Execute(recog, input, startIndex);
}
}
public virtual void Consume(ICharStream input)
{
int curChar = input.La(1);
if (curChar == '\n')
{
line++;
charPositionInLine = 0;
}
else
{
charPositionInLine++;
}
input.Consume();
}
protected internal virtual void Accept(ICharStream input, int ruleIndex, int actionIndex
, int index, int line, int charPos)
{
if (actionIndex >= 0 && recog != null)
{
recog.Action(null, ruleIndex, actionIndex);
}
// seek to after last char in token
input.Seek(index);
this.line = line;
this.charPositionInLine = charPos;
if (input.La(1) != IntStreamConstants.Eof)
{
Consume(input);
}
}
public static void Check_CobolCharStream()
{
// Test file properties
string relativePath = @"Compiler\Parser\Samples";
string textName = "MSVCOUT";
DocumentFormat documentFormat = DocumentFormat.RDZReferenceFormat;
// Compile test file
CompilationDocument compilationDocument = ParserUtils.ScanCobolFile(relativePath, textName, documentFormat);
// Create a token iterator on top of tokens lines
TokensLinesIterator tokensIterator = new TokensLinesIterator(
compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name,
compilationDocument.TokensDocumentSnapshot.Lines,
null,
Token.CHANNEL_SourceTokens);
// Crate an Antlr compatible token source on top a the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name,
tokensIterator);
tokenSource.NextToken();
// Get underlying CharStream
ICharStream charStream = tokenSource.InputStream;
if (charStream.Index != 0)
{
throw new Exception("Char stream index should start at 0");
}
if (charStream.La(0) != 0)
{
throw new Exception("La(0) should be 0");
}
if (charStream.La(1) != '0')
{
throw new Exception("La(1) should be 0");
}
if (charStream.La(4) != '1')
{
throw new Exception("La(4) should be 1");
}
if (charStream.La(5) != '6')
{
throw new Exception("La(5) should be 6");
}
charStream.Consume();
if (charStream.Index != 1)
{
throw new Exception("Char stream index should be 1 after consume");
}
if (charStream.La(4) != '6')
{
throw new Exception("La(4) should be 6 after consume");
}
if (charStream.La(80) != IntStreamConstants.Eof)
{
throw new Exception("La(80) should be Eof");
}
charStream.Seek(12);
if (charStream.Index != 12)
{
throw new Exception("Char stream index should be 12 after seek");
}
if (charStream.La(-1) != ':')
{
throw new Exception("La(-1) should be : after seek");
}
if (charStream.La(1) != 'M')
{
throw new Exception("La(1) should be M after seek");
}
// should do nothing
int marker = charStream.Mark();
charStream.Release(marker);
if (charStream.La(2) != 'S')
{
throw new Exception("La(2) should be S after release");
}
string text = charStream.GetText(new Interval(11, 18));
if (text != ":MSVCOUT")
{
throw new Exception("Char stream GetText method KO");
}
if (charStream.Size != 80)
{
throw new Exception("Char stream size KO");
}
}
/// <summary>
/// Return a token from this source; i.e., match a token on the char
/// stream.
/// </summary>
/// <remarks>
/// Return a token from this source; i.e., match a token on the char
/// stream.
/// </remarks>
public virtual IToken NextToken()
{
if (_input == null)
{
throw new InvalidOperationException("nextToken requires a non-null input stream.");
}
// Mark start location in char stream so unbuffered streams are
// guaranteed at least have text of current token
int tokenStartMarker = _input.Mark();
try
{
while (true)
{
if (_hitEOF)
{
EmitEOF();
return(_token);
}
_token = null;
_channel = TokenConstants.DefaultChannel;
_tokenStartCharIndex = _input.Index;
_tokenStartCharPositionInLine = Interpreter.Column;
_tokenStartLine = Interpreter.Line;
_text = null;
do
{
_type = TokenConstants.InvalidType;
// System.out.println("nextToken line "+tokenStartLine+" at "+((char)input.LA(1))+
// " in mode "+mode+
// " at index "+input.index());
int ttype;
try
{
ttype = Interpreter.Match(_input, _mode);
}
catch (LexerNoViableAltException e)
{
NotifyListeners(e);
// report error
Recover(e);
ttype = TokenTypes.Skip;
}
if (_input.La(1) == IntStreamConstants.Eof)
{
_hitEOF = true;
}
if (_type == TokenConstants.InvalidType)
{
_type = ttype;
}
if (_type == TokenTypes.Skip)
{
goto outer_continue;
}
}while (_type == TokenTypes.More);
if (_token == null)
{
Emit();
}
return(_token);
outer_continue :;
}
}
finally
{
// make sure we release marker after match or
// unbuffered char stream will keep buffering
_input.Release(tokenStartMarker);
}
}
protected internal virtual int ExecATN(ICharStream input, DFAState ds0)
{
//System.out.println("enter exec index "+input.index()+" from "+ds0.configs);
int t = input.La(1);
DFAState s = ds0;
// s is current/from DFA state
while (true)
{
// while more work
// As we move src->trg, src->trg, we keep track of the previous trg to
// avoid looking up the DFA state again, which is expensive.
// If the previous target was already part of the DFA, we might
// be able to avoid doing a reach operation upon t. If s!=null,
// it means that semantic predicates didn't prevent us from
// creating a DFA state. Once we know s!=null, we check to see if
// the DFA state has an edge already for t. If so, we can just reuse
// it's configuration set; there's no point in re-computing it.
// This is kind of like doing DFA simulation within the ATN
// simulation because DFA simulation is really just a way to avoid
// computing reach/closure sets. Technically, once we know that
// we have a previously added DFA state, we could jump over to
// the DFA simulator. But, that would mean popping back and forth
// a lot and making things more complicated algorithmically.
// This optimization makes a lot of sense for loops within DFA.
// A character will take us back to an existing DFA state
// that already has lots of edges out of it. e.g., .* in comments.
ATNConfigSet closure = s.configs;
DFAState target = null;
target = s.GetTarget(t);
if (target == Error)
{
break;
}
#if !PORTABLE
if (debug && target != null)
{
System.Console.Out.WriteLine("reuse state " + s.stateNumber + " edge to " + target
.stateNumber);
}
#endif
if (target == null)
{
ATNConfigSet reach = new OrderedATNConfigSet();
// if we don't find an existing DFA state
// Fill reach starting from closure, following t transitions
GetReachableConfigSet(input, closure, reach, t);
if (reach.Count == 0)
{
// we got nowhere on t from s
// we reached state associated with closure for sure, so
// make sure it's defined. worst case, we define s0 from
// start state configs.
DFAState from = s != null ? s : AddDFAState(closure);
// we got nowhere on t, don't throw out this knowledge; it'd
// cause a failover from DFA later.
AddDFAEdge(from, t, Error);
break;
}
// stop when we can't match any more char
// Add an edge from s to target DFA found/created for reach
target = AddDFAEdge(s, t, reach);
}
if (target.isAcceptState)
{
CaptureSimState(prevAccept, input, target);
if (t == IntStreamConstants.Eof)
{
break;
}
}
if (t != IntStreamConstants.Eof)
{
Consume(input);
t = input.La(1);
}
s = target;
}
// flip; current DFA target becomes new src/from state
return FailOrAccept(prevAccept, input, s.configs, t);
}
