public void ResetAcceptPosition(ICharStream input, int index, int line, int charPositionInLine)
{
input.Seek(index);
this.line = line;
this.charPositionInLine = charPositionInLine;
Consume(input);
}
/**
* Evaluate a predicate specified in the lexer.
*
* <p>If {@code speculative} is {@code true}, this method was called before
* {@link #consume} for the matched character. This method should call
* {@link #consume} before evaluating the predicate to ensure position
* sensitive values, including {@link Lexer#getText}, {@link Lexer#getLine},
* and {@link Lexer#getCharPositionInLine}, properly reflect the current
* lexer state. This method should restore {@code input} and the simulator
* to the original state before returning (i.e. undo the actions made by the
* call to {@link #consume}.</p>
*
* @param input The input stream.
* @param ruleIndex The rule containing the predicate.
* @param predIndex The index of the predicate within the rule.
* @param speculative {@code true} if the current index in {@code input} is
* one character before the predicate's location.
*
* @return {@code true} if the specified predicate evaluates to
* {@code true}.
*/
protected bool EvaluatePredicate(ICharStream input, int ruleIndex, int predIndex, bool speculative)
{
// assume true if no recognizer was provided
if (recog == null)
{
return(true);
}
if (!speculative)
{
return(recog.Sempred(null, ruleIndex, predIndex));
}
int savedCharPositionInLine = charPositionInLine;
int savedLine = thisLine;
int index = input.Index;
int marker = input.Mark();
try
{
Consume(input);
return(recog.Sempred(null, ruleIndex, predIndex));
}
finally
{
charPositionInLine = savedCharPositionInLine;
thisLine = savedLine;
input.Seek(index);
input.Release(marker);
}
}
public string GetText(Interval interval)
{
StringBuilder buf = new StringBuilder();
int start = interval.a;
int stop = interval.b;
int index = stream.Index;
stream.Seek(0);
for (int i = start; i < (stop + 1); i++)
{
int t = stream.LA(i + 1);
char c = (char)t;
buf.Append(c);
}
stream.Seek(index);
return(buf.ToString());
}
protected internal virtual void Accept(ICharStream input, LexerActionExecutor lexerActionExecutor, int startIndex, int index, int line, int charPos)
{
// seek to after last char in token
input.Seek(index);
this._line = line;
this.charPositionInLine = charPos;
if (lexerActionExecutor != null && recog != null)
{
lexerActionExecutor.Execute(recog, input, startIndex);
}
}
/// <summary>
/// Execute the actions encapsulated by this executor within the context of a
/// particular
/// <see cref="Antlr4.Runtime.Lexer"/>
/// .
/// <p>This method calls
/// <see cref="Antlr4.Runtime.IIntStream.Seek(int)"/>
/// to set the position of the
/// <paramref name="input"/>
///
/// <see cref="Antlr4.Runtime.ICharStream"/>
/// prior to calling
/// <see cref="ILexerAction.Execute(Antlr4.Runtime.Lexer)"/>
/// on a position-dependent action. Before the
/// method returns, the input position will be restored to the same position
/// it was in when the method was invoked.</p>
/// </summary>
/// <param name="lexer">The lexer instance.</param>
/// <param name="input">
/// The input stream which is the source for the current token.
/// When this method is called, the current
/// <see cref="Antlr4.Runtime.IIntStream.Index()"/>
/// for
/// <paramref name="input"/>
/// should be the start of the following token, i.e. 1
/// character past the end of the current token.
/// </param>
/// <param name="startIndex">
/// The token start index. This value may be passed to
/// <see cref="Antlr4.Runtime.IIntStream.Seek(int)"/>
/// to set the
/// <paramref name="input"/>
/// position to the beginning
/// of the token.
/// </param>
public virtual void Execute(Lexer lexer, ICharStream input, int startIndex)
{
bool requiresSeek = false;
int stopIndex = input.Index;
try
{
foreach (ILexerAction lexerAction in lexerActions)
{
ILexerAction action = lexerAction;
if (action is LexerIndexedCustomAction)
{
int offset = ((LexerIndexedCustomAction)action).Offset;
input.Seek(startIndex + offset);
action = ((LexerIndexedCustomAction)action).Action;
requiresSeek = (startIndex + offset) != stopIndex;
}
else
{
if (action.IsPositionDependent)
{
input.Seek(stopIndex);
requiresSeek = false;
}
}
action.Execute(lexer);
}
}
finally
{
if (requiresSeek)
{
input.Seek(stopIndex);
}
}
}
protected internal virtual void Accept(ICharStream input, int ruleIndex, int actionIndex
, int index, int line, int charPos)
{
if (actionIndex >= 0 && recog != null)
{
recog.Action(null, ruleIndex, actionIndex);
}
// seek to after last char in token
input.Seek(index);
this.line = line;
this.charPositionInLine = charPos;
if (input.La(1) != IntStreamConstants.Eof)
{
Consume(input);
}
}
protected void Accept(ICharStream input, LexerActionExecutor lexerActionExecutor,
int startIndex, int index, int line, int charPos)
{
if (debug)
{
ConsoleWriteLine("ACTION " + lexerActionExecutor);
}
// seek to after last char in token
input.Seek(index);
this.thisLine = line;
this.charPositionInLine = charPos;
if (lexerActionExecutor != null && recog != null)
{
lexerActionExecutor.Execute(recog, input, startIndex);
}
}
public override void Reset()
{
base.Reset(); // reset all recognizer state variables
// wack Lexer state variables
if (input != null)
{
input.Seek(0); // rewind the input
}
if (state == null)
{
return; // no shared state work to do
}
state.token = null;
state.type = TokenConstants.INVALID_TOKEN_TYPE;
state.channel = TokenConstants.DEFAULT_CHANNEL;
state.tokenStartCharIndex = -1;
state.tokenStartCharPositionInLine = -1;
state.tokenStartLine = -1;
state.text = null;
}
public override void Reset()
{
base.Reset(); // reset all recognizer state variables
// wack Lexer state variables
if (input != null)
{
input.Seek(0); // rewind the input
}
if (state == null)
{
return; // no shared state work to do
}
state.token = null;
state.type = TokenTypes.Invalid;
state.channel = TokenChannels.Default;
state.tokenStartCharIndex = -1;
state.tokenStartCharPositionInLine = -1;
state.tokenStartLine = -1;
state.text = null;
}
public virtual void Reset()
{
// wack Lexer state variables
if (_input != null)
{
_input.Seek(0);
}
// rewind the input
_token = null;
_type = TokenConstants.InvalidType;
_channel = TokenConstants.DefaultChannel;
_tokenStartCharIndex = -1;
_tokenStartColumn = -1;
_tokenStartLine = -1;
_text = null;
_hitEOF = false;
_mode = Antlr4.Runtime.Lexer.DEFAULT_MODE;
_modeStack.Clear();
Interpreter.Reset();
}
public void Seek(int index)
{
if (index > _range)
{
throw new NotSupportedException();
}
_source.Seek(index);
_la1 = _source.La(1);
_slashCount = 0;
while (_source.La(-_slashCount - 1) == '\\')
{
_slashCount++;
}
_escapeListIndex = _escapeIndexes.BinarySearch(_source.Index);
if (_escapeListIndex < 0)
{
_escapeListIndex = -_escapeListIndex - 1;
}
}
protected internal virtual void Accept(ICharStream input, int ruleIndex, int actionIndex
, int index, int line, int charPos)
{
if (actionIndex >= 0 && recog != null)
{
recog.Action(null, ruleIndex, actionIndex);
}
// seek to after last char in token
input.Seek(index);
this.line = line;
this.charPositionInLine = charPos;
if (input.La(1) != IntStreamConstants.Eof)
{
Consume(input);
}
}
/// <summary>
/// Execute the actions encapsulated by this executor within the context of a
/// particular
/// <see cref="Antlr4.Runtime.Lexer"/>
/// .
/// <p>This method calls
/// <see cref="Antlr4.Runtime.IIntStream.Seek(int)"/>
/// to set the position of the
/// <code>input</code>
///
/// <see cref="Antlr4.Runtime.ICharStream"/>
/// prior to calling
/// <see cref="ILexerAction.Execute(Antlr4.Runtime.Lexer)"/>
/// on a position-dependent action. Before the
/// method returns, the input position will be restored to the same position
/// it was in when the method was invoked.</p>
/// </summary>
/// <param name="lexer">The lexer instance.</param>
/// <param name="input">
/// The input stream which is the source for the current token.
/// When this method is called, the current
/// <see cref="Antlr4.Runtime.IIntStream.Index()"/>
/// for
/// <code>input</code>
/// should be the start of the following token, i.e. 1
/// character past the end of the current token.
/// </param>
/// <param name="startIndex">
/// The token start index. This value may be passed to
/// <see cref="Antlr4.Runtime.IIntStream.Seek(int)"/>
/// to set the
/// <code>input</code>
/// position to the beginning
/// of the token.
/// </param>
public virtual void Execute(Lexer lexer, ICharStream input, int startIndex)
{
bool requiresSeek = false;
int stopIndex = input.Index;
try
{
foreach (ILexerAction lexerAction in lexerActions)
{
ILexerAction action = lexerAction;
if (action is LexerIndexedCustomAction)
{
int offset = ((LexerIndexedCustomAction)action).Offset;
input.Seek(startIndex + offset);
action = ((LexerIndexedCustomAction)action).Action;
requiresSeek = (startIndex + offset) != stopIndex;
}
else
{
if (action.IsPositionDependent)
{
input.Seek(stopIndex);
requiresSeek = false;
}
}
action.Execute(lexer);
}
}
finally
{
if (requiresSeek)
{
input.Seek(stopIndex);
}
}
}
public void Seek(int index)
{
stream.Seek(index);
}
protected void parseSources(ParserFactory factory, IEnumerable <InputDescriptor> sources)
{
Stopwatch startTime = Stopwatch.StartNew();
Thread.VolatileWrite(ref tokenCount, 0);
int sourceCount = 0;
int inputSize = 0;
#if NET40PLUS
BlockingCollection <int> threadIdentifiers = new BlockingCollection <int>();
for (int i = 0; i < NUMBER_OF_THREADS; i++)
{
threadIdentifiers.Add(i);
}
ICollection <Task <int> > results = new List <Task <int> >();
QueuedTaskScheduler executorServiceHost = new QueuedTaskScheduler(NUMBER_OF_THREADS);
TaskScheduler executorService = executorServiceHost.ActivateNewQueue();
#else
ICollection <Func <int> > results = new List <Func <int> >();
#endif
foreach (InputDescriptor inputDescriptor in sources)
{
ICharStream input = inputDescriptor.GetInputStream();
sourceCount++;
input.Seek(0);
inputSize += input.Size;
#if NET40PLUS
Task <int> futureChecksum = Task.Factory.StartNew <int>(new Callable_1(input, factory, threadIdentifiers).call, CancellationToken.None, TaskCreationOptions.None, executorService);
#else
Func <int> futureChecksum = new Callable_1(input, factory).call;
#endif
results.Add(futureChecksum);
}
Checksum checksum = new CRC32();
foreach (var future in results)
{
#if NET40PLUS
int value = future.Result;
#else
int value = future();
#endif
if (COMPUTE_CHECKSUM)
{
updateChecksum(checksum, value);
}
}
#if NET40PLUS
executorServiceHost.Dispose();
#endif
Console.Out.WriteLine("Total parse time for {0} files ({1} KB, {2} tokens, checksum 0x{3:X8}): {4}ms",
sourceCount,
inputSize / 1024,
Thread.VolatileRead(ref tokenCount),
COMPUTE_CHECKSUM ? checksum.Value : 0,
startTime.ElapsedMilliseconds);
if (sharedLexers.Length > 0)
{
Lexer lexer = sharedLexers[0];
LexerATNSimulator lexerInterpreter = lexer.Interpreter;
DFA[] modeToDFA = lexerInterpreter.atn.modeToDFA;
if (SHOW_DFA_STATE_STATS)
{
int states = 0;
int configs = 0;
HashSet <ATNConfig> uniqueConfigs = new HashSet <ATNConfig>();
for (int i = 0; i < modeToDFA.Length; i++)
{
DFA dfa = modeToDFA[i];
if (dfa == null || dfa.states == null)
{
continue;
}
states += dfa.states.Count;
foreach (DFAState state in dfa.states.Values)
{
configs += state.configs.Count;
uniqueConfigs.UnionWith(state.configs);
}
}
Console.Out.WriteLine("There are {0} lexer DFAState instances, {1} configs ({2} unique), {3} prediction contexts.", states, configs, uniqueConfigs.Count, lexerInterpreter.atn.ContextCacheSize);
}
}
if (RUN_PARSER && sharedParsers.Length > 0)
{
Parser parser = sharedParsers[0];
// make sure the individual DFAState objects actually have unique ATNConfig arrays
ParserATNSimulator interpreter = parser.Interpreter;
DFA[] decisionToDFA = interpreter.atn.decisionToDFA;
if (SHOW_DFA_STATE_STATS)
{
int states = 0;
int configs = 0;
HashSet <ATNConfig> uniqueConfigs = new HashSet <ATNConfig>();
for (int i = 0; i < decisionToDFA.Length; i++)
{
DFA dfa = decisionToDFA[i];
if (dfa == null || dfa.states == null)
{
continue;
}
states += dfa.states.Count;
foreach (DFAState state in dfa.states.Values)
{
configs += state.configs.Count;
uniqueConfigs.UnionWith(state.configs);
}
}
Console.Out.WriteLine("There are {0} parser DFAState instances, {1} configs ({2} unique), {3} prediction contexts.", states, configs, uniqueConfigs.Count, interpreter.atn.ContextCacheSize);
}
int localDfaCount = 0;
int globalDfaCount = 0;
int localConfigCount = 0;
int globalConfigCount = 0;
int[] contextsInDFAState = new int[0];
for (int i = 0; i < decisionToDFA.Length; i++)
{
DFA dfa = decisionToDFA[i];
if (dfa == null || dfa.states == null)
{
continue;
}
if (SHOW_CONFIG_STATS)
{
foreach (DFAState state in dfa.states.Keys)
{
if (state.configs.Count >= contextsInDFAState.Length)
{
Array.Resize(ref contextsInDFAState, state.configs.Count + 1);
}
if (state.IsAcceptState)
{
bool hasGlobal = false;
foreach (ATNConfig config in state.configs)
{
if (config.ReachesIntoOuterContext)
{
globalConfigCount++;
hasGlobal = true;
}
else
{
localConfigCount++;
}
}
if (hasGlobal)
{
globalDfaCount++;
}
else
{
localDfaCount++;
}
}
contextsInDFAState[state.configs.Count]++;
}
}
if (EXPORT_LARGEST_CONFIG_CONTEXTS)
{
foreach (DFAState state in dfa.states.Keys)
{
foreach (ATNConfig config in state.configs)
{
string configOutput = config.ToDotString();
if (configOutput.Length <= configOutputSize)
{
continue;
}
configOutputSize = configOutput.Length;
writeFile(tmpdir, "d" + dfa.decision + ".s" + state.stateNumber + ".a" + config.Alt + ".config.dot", configOutput);
}
}
}
}
if (SHOW_CONFIG_STATS && currentPass == 0)
{
Console.Out.WriteLine(" DFA accept states: {0} total, {1} with only local context, {2} with a global context", localDfaCount + globalDfaCount, localDfaCount, globalDfaCount);
Console.Out.WriteLine(" Config stats: {0} total, {1} local, {2} global", localConfigCount + globalConfigCount, localConfigCount, globalConfigCount);
if (SHOW_DFA_STATE_STATS)
{
for (int i = 0; i < contextsInDFAState.Length; i++)
{
if (contextsInDFAState[i] != 0)
{
Console.Out.WriteLine(" {0} configs = {1}", i, contextsInDFAState[i]);
}
}
}
}
}
}
protected void Accept(ICharStream input, LexerActionExecutor lexerActionExecutor,
int startIndex, int index, int line, int charPos)
{
if (debug)
{
Console.WriteLine("ACTION " + lexerActionExecutor);
}
// seek to after last char in token
input.Seek(index);
this.thisLine = line;
this.charPositionInLine = charPos;
if (lexerActionExecutor != null && recog != null)
{
lexerActionExecutor.Execute(recog, input, startIndex);
}
}
public void Seek(int index) => internalStream.Seek(index);
public static void Check_CobolCharStream()
{
// Test file properties
string relativePath = @"Compiler\Parser\Samples";
string textName = "MSVCOUT";
DocumentFormat documentFormat = DocumentFormat.RDZReferenceFormat;
// Compile test file
CompilationDocument compilationDocument = ParserUtils.ScanCobolFile(relativePath, textName, documentFormat);
// Create a token iterator on top of tokens lines
TokensLinesIterator tokensIterator = new TokensLinesIterator(
compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name,
compilationDocument.TokensDocumentSnapshot.Lines,
null,
Token.CHANNEL_SourceTokens);
// Crate an Antlr compatible token source on top a the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
compilationDocument.TokensDocumentSnapshot.TextSourceInfo.Name,
tokensIterator);
tokenSource.NextToken();
// Get underlying CharStream
ICharStream charStream = tokenSource.InputStream;
if (charStream.Index != 0)
{
throw new Exception("Char stream index should start at 0");
}
if (charStream.La(0) != 0)
{
throw new Exception("La(0) should be 0");
}
if (charStream.La(1) != '0')
{
throw new Exception("La(1) should be 0");
}
if (charStream.La(4) != '1')
{
throw new Exception("La(4) should be 1");
}
if (charStream.La(5) != '6')
{
throw new Exception("La(5) should be 6");
}
charStream.Consume();
if (charStream.Index != 1)
{
throw new Exception("Char stream index should be 1 after consume");
}
if (charStream.La(4) != '6')
{
throw new Exception("La(4) should be 6 after consume");
}
if (charStream.La(80) != IntStreamConstants.Eof)
{
throw new Exception("La(80) should be Eof");
}
charStream.Seek(12);
if (charStream.Index != 12)
{
throw new Exception("Char stream index should be 12 after seek");
}
if (charStream.La(-1) != ':')
{
throw new Exception("La(-1) should be : after seek");
}
if (charStream.La(1) != 'M')
{
throw new Exception("La(1) should be M after seek");
}
// should do nothing
int marker = charStream.Mark();
charStream.Release(marker);
if (charStream.La(2) != 'S')
{
throw new Exception("La(2) should be S after release");
}
string text = charStream.GetText(new Interval(11, 18));
if (text != ":MSVCOUT")
{
throw new Exception("Char stream GetText method KO");
}
if (charStream.Size != 80)
{
throw new Exception("Char stream size KO");
}
}
/// <summary>Evaluate a predicate specified in the lexer.</summary>
/// <remarks>
/// Evaluate a predicate specified in the lexer.
/// <p/>
/// If
/// <code>speculative</code>
/// is
/// <code>true</code>
/// , this method was called before
/// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream)
/// </see>
/// for the matched character. This method should call
/// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream)
/// </see>
/// before evaluating the predicate to ensure position
/// sensitive values, including
/// <see cref="Antlr4.Runtime.Lexer.Text()">Antlr4.Runtime.Lexer.Text()</see>
/// ,
/// <see cref="Antlr4.Runtime.Lexer.Line()">Antlr4.Runtime.Lexer.Line()</see>
/// ,
/// and
/// <see cref="Antlr4.Runtime.Lexer.Column()">Antlr4.Runtime.Lexer.Column()</see>
/// , properly reflect the current
/// lexer state. This method should restore
/// <code>input</code>
/// and the simulator
/// to the original state before returning (i.e. undo the actions made by the
/// call to
/// <see cref="Consume(Antlr4.Runtime.ICharStream)">Consume(Antlr4.Runtime.ICharStream)
/// </see>
/// .
/// </remarks>
/// <param name="input">The input stream.</param>
/// <param name="ruleIndex">The rule containing the predicate.</param>
/// <param name="predIndex">The index of the predicate within the rule.</param>
/// <param name="speculative">
///
/// <code>true</code>
/// if the current index in
/// <code>input</code>
/// is
/// one character before the predicate's location.
/// </param>
/// <returns>
///
/// <code>true</code>
/// if the specified predicate evaluates to
/// <code>true</code>
/// .
/// </returns>
protected internal virtual bool EvaluatePredicate(ICharStream input, int ruleIndex
, int predIndex, bool speculative)
{
// assume true if no recognizer was provided
if (recog == null)
{
return true;
}
if (!speculative)
{
return recog.Sempred(null, ruleIndex, predIndex);
}
int savedCharPositionInLine = charPositionInLine;
int savedLine = line;
int index = input.Index;
int marker = input.Mark();
try
{
Consume(input);
return recog.Sempred(null, ruleIndex, predIndex);
}
finally
{
charPositionInLine = savedCharPositionInLine;
line = savedLine;
input.Seek(index);
input.Release(marker);
}
}
private ErrorList InternalCompile(ICharStream stream)
{
var errors = new ErrorList();
var lexer = TemplateLexer.Create(stream);
var tStream = new CommonTokenStream(lexer);
var parser = new TemplateParser(tStream) {Errors = errors};
var docResult = parser.document();
_template = new DynamicMethod(
string.Format("__template_{0}", Name),
typeof (void),
new[] {typeof (VirtualTemplate)},
typeof (VirtualTemplate),
true);
var emit = _template.GetILGenerator();
var ctx = new Context(emit) {OptimizeLevel = OptimizeLevel};
var doc = new Document(docResult.Tree);
try
{
var e = doc.Generate(ctx);
errors.AddRange(e);
_image = (RunTemplate) _template.CreateDelegate(typeof (RunTemplate));
Assembly = ctx.Sink.Build();
}
catch (Exception e)
{
errors.ErrorUnhandledCompileError(e);
}
stream.Seek(0);
Source = stream.ToString();
return errors;
}
protected internal virtual void Accept(ICharStream input, LexerActionExecutor lexerActionExecutor, int startIndex, int index, int line, int charPos)
{
// seek to after last char in token
input.Seek(index);
this.line = line;
this.charPositionInLine = charPos;
if (lexerActionExecutor != null && recog != null)
{
lexerActionExecutor.Execute(recog, input, startIndex);
}
}