public AssertTokenizer/*!*/ Next() {
_actualToken = _tokenizer.GetNextToken();
_actualValue = _tokenizer.TokenValue;
_actualSpan = _tokenizer.TokenSpan;
_allTokens.Add(_actualToken);
_allValues.Add(_actualValue);
return this;
}
private Arguments /*!*/ RequireNoBlockArg(TokenValue arguments)
{
if (arguments.Block != null)
{
ErrorSink.Add(_sourceUnit, "block argument should not be given", arguments.Block.Location, -1, Severity.Error);
arguments.Block = null;
}
return(arguments.Arguments);
}
public static StringLiteral /*!*/ MakeStringLiteral(TokenValue token, SourceSpan location)
{
return(new StringLiteral(token.StringContent, location));
}
private WhenClause /*!*/ MakeWhenClause(TokenValue whenArgs, Statements /*!*/ statements, SourceSpan location)
{
return(new WhenClause(whenArgs.ArgumentCount != 0 ? PopArguments(whenArgs.ArgumentCount) : null, whenArgs.Expression, statements, location));
}
private static SuperCall /*!*/ MakeSuperCall(TokenValue args, SourceSpan location)
{
Debug.Assert(args.Arguments != null);
return(new SuperCall(args.Arguments, args.Block, location));
}
private static MethodCall /*!*/ MakeMethodCall(Expression target, string /*!*/ methodName, TokenValue args, Block block, SourceSpan location)
{
Debug.Assert(args.Block == null);
return(new MethodCall(target, methodName, args.Arguments, block, location));
}
private static MethodCall /*!*/ MakeMethodCall(Expression target, string /*!*/ methodName, TokenValue args, SourceSpan location)
{
return(new MethodCall(target, methodName, args.Arguments, args.Block, location));
}
private static SuperCall /*!*/ MakeSuperCall(TokenValue args, SourceSpan location)
{
return(new SuperCall(args.Arguments, args.Block, location));
}
private static ArrayItemAccess /*!*/ MakeArrayItemAccess(Expression /*!*/ array, TokenValue args, SourceSpan location)
{
return(new ArrayItemAccess(array, args.Arguments, args.Block, location));
}
public override void Initialize(object state, TextReader/*!*/ reader, SourceUnit sourceUnit, SourceLocation initialLocation) {
ContractUtils.RequiresNotNull(reader, "reader");
_sourceUnit = sourceUnit;
_input = reader;
_initialLocation = initialLocation;
_currentLine = _initialLocation.Line;
_currentLineIndex = _initialLocation.Index;
_tokenSpan = new SourceSpan(initialLocation, initialLocation);
SetState(LexicalState.EXPR_BEG);
_tokenValue = new TokenValue();
_eofReached = false;
_unterminatedToken = false;
DumpBeginningOfUnit();
}
public Tokens GetNextToken() {
//if (_buffer == null) throw new InvalidOperationException("Uninitialized");
if (_input == null) throw new InvalidOperationException("Uninitialized");
#if DEBUG
_tokenValue = new TokenValue();
#endif
Tokens result = Tokenize();
if (result == Tokens.EndOfFile) {
_eofReached = true;
}
return result;
}
public void Initialize(object state, TextReader/*!*/ reader, SourceUnit sourceUnit, SourceLocation initialLocation, int bufferCapacity) {
ContractUtils.RequiresNotNull(reader, "reader");
//if (state != null) {
// if (!(state is State)) throw new ArgumentException();
// _state = new State((State)state);
//} else {
// _state = new State(null);
//}
_sourceUnit = sourceUnit;
//_buffer = new TokenizerBuffer(sourceReader, initialLocation, bufferCapacity, true);
_input = reader;
_initialLocation = initialLocation;
_tokenSpan = new SourceSpan(initialLocation, initialLocation);
_tokenValue = new TokenValue();
_eofReached = false;
UnterminatedToken = false;
DumpBeginningOfUnit();
}
public Tokenizer(bool verbatim, ErrorSink/*!*/ errorSink) {
ContractUtils.RequiresNotNull(errorSink, "errorSink");
_bigIntParser = new BignumParser();
_errorSink = errorSink;
_sourceUnit = null;
_parser = null;
_verbatim = verbatim;
_compatibility = RubyCompatibility.Default;
// _buffer = null;
_initialLocation = SourceLocation.Invalid;
_tokenSpan = SourceSpan.Invalid;
_tokenValue = new TokenValue();
_bufferPos = 0;
// TODO:
_input = null;
}