void tokenize_file (CompilationUnit file, CompilerContext ctx)
{
Stream input;
try {
input = File.OpenRead (file.Name);
} catch {
Report.Error (2001, "Source file `" + file.Name + "' could not be found");
return;
}
using (input){
SeekableStreamReader reader = new SeekableStreamReader (input, RootContext.Encoding);
Tokenizer lexer = new Tokenizer (reader, file, ctx);
int token, tokens = 0, errors = 0;
while ((token = lexer.token ()) != Token.EOF){
tokens++;
if (token == Token.ERROR)
errors++;
}
Console.WriteLine ("Tokenized: " + tokens + " found " + errors + " errors");
}
return;
}
void tokenize_file (SourceFile sourceFile, ModuleContainer module)
{
Stream input;
try {
input = File.OpenRead (sourceFile.Name);
} catch {
Report.Error (2001, "Source file `" + sourceFile.Name + "' could not be found");
return;
}
using (input){
SeekableStreamReader reader = new SeekableStreamReader (input, ctx.Settings.Encoding);
var file = new CompilationSourceFile (module, sourceFile);
Tokenizer lexer = new Tokenizer (reader, file);
int token, tokens = 0, errors = 0;
while ((token = lexer.token ()) != Token.EOF){
tokens++;
if (token == Token.ERROR)
errors++;
}
Console.WriteLine ("Tokenized: " + tokens + " found " + errors + " errors");
}
return;
}
void tokenize_file (SourceFile sourceFile, ModuleContainer module, ParserSession session)
{
Stream input;
try {
input = File.OpenRead (sourceFile.Name);
} catch {
Report.Error (2001, "Source file `" + sourceFile.Name + "' could not be found");
return;
}
using (input) {
SeekableStreamReader reader = new SeekableStreamReader (input, ctx.Settings.Encoding);
var file = new CompilationSourceFile (module, sourceFile);
if (sourceFile.FileType == SourceFileType.CSharp) {
Tokenizer lexer = new Tokenizer (reader, file, session);
int token, tokens = 0, errors = 0;
while ((token = lexer.token ()) != Token.EOF){
tokens++;
if (token == Token.ERROR)
errors++;
}
} else {
Mono.PlayScript.Tokenizer lexer = new Mono.PlayScript.Tokenizer (reader, file, session);
lexer.ParsingPlayScript = sourceFile.PsExtended;
int token, tokens = 0, errors = 0;
while ((token = lexer.token ()) != Mono.PlayScript.Token.EOF){
tokens++;
if (token == Mono.PlayScript.Token.ERROR)
errors++;
}
}
}
return;
}
//
// Deambiguates the input string to determine if we
// want to process a statement or if we want to
// process a compilation unit.
//
// This is done using a top-down predictive parser,
// since the yacc/jay parser can not deambiguage this
// without more than one lookahead token. There are very
// few ambiguities.
//
InputKind ToplevelOrStatement (SeekableStreamReader seekable)
{
Tokenizer tokenizer = new Tokenizer (seekable, source_file);
int t = tokenizer.token ();
switch (t){
case Token.EOF:
return InputKind.EOF;
// These are toplevels
case Token.EXTERN:
case Token.OPEN_BRACKET:
case Token.ABSTRACT:
case Token.CLASS:
case Token.ENUM:
case Token.INTERFACE:
case Token.INTERNAL:
case Token.NAMESPACE:
case Token.PRIVATE:
case Token.PROTECTED:
case Token.PUBLIC:
case Token.SEALED:
case Token.STATIC:
case Token.STRUCT:
return InputKind.CompilationUnit;
// Definitely expression
case Token.FIXED:
case Token.BOOL:
case Token.BYTE:
case Token.CHAR:
case Token.DECIMAL:
case Token.DOUBLE:
case Token.FLOAT:
case Token.INT:
case Token.LONG:
case Token.NEW:
case Token.OBJECT:
case Token.SBYTE:
case Token.SHORT:
case Token.STRING:
case Token.UINT:
case Token.ULONG:
return InputKind.StatementOrExpression;
// These need deambiguation help
case Token.USING:
t = tokenizer.token ();
if (t == Token.EOF)
return InputKind.EOF;
if (t == Token.IDENTIFIER)
return InputKind.CompilationUnit;
return InputKind.StatementOrExpression;
// Distinguish between:
// delegate opt_anonymous_method_signature block
// delegate type
case Token.DELEGATE:
t = tokenizer.token ();
if (t == Token.EOF)
return InputKind.EOF;
if (t == Token.OPEN_PARENS || t == Token.OPEN_BRACE)
return InputKind.StatementOrExpression;
return InputKind.CompilationUnit;
// Distinguih between:
// unsafe block
// unsafe as modifier of a type declaration
case Token.UNSAFE:
t = tokenizer.token ();
if (t == Token.EOF)
return InputKind.EOF;
if (t == Token.OPEN_PARENS)
return InputKind.StatementOrExpression;
return InputKind.CompilationUnit;
// These are errors: we list explicitly what we had
// from the grammar, ERROR and then everything else
case Token.READONLY:
case Token.OVERRIDE:
case Token.ERROR:
return InputKind.Error;
// This catches everything else allowed by
// expressions. We could add one-by-one use cases
// if needed.
default:
return InputKind.StatementOrExpression;
}
}
public CSharpParser (SeekableStreamReader reader, CompilationSourceFile file, Report report, ParserSession session)
{
this.file = file;
current_container = current_namespace = file;
this.module = file.Module;
this.compiler = file.Compiler;
this.settings = compiler.Settings;
this.report = report;
lang_version = settings.Version;
yacc_verbose_flag = settings.VerboseParserFlag;
doc_support = settings.DocumentationFile != null;
lexer = new Tokenizer (reader, file, session);
oob_stack = new Stack<object> ();
lbag = session.LocationsBag;
use_global_stacks = session.UseJayGlobalArrays;
parameters_bucket = session.ParametersStack;
}
public CSharpParser (SeekableStreamReader reader, CompilationUnit file, CompilerContext ctx)
{
if (RootContext.EvalMode)
undo = new Undo ();
this.file = file;
this.compiler = ctx;
current_namespace = new NamespaceEntry (ctx, null, file, null);
current_class = current_namespace.SlaveDeclSpace;
current_container = current_class.PartialContainer; // == RootContest.ToplevelTypes
oob_stack.Clear ();
lexer = new Tokenizer (reader, file, ctx);
use_global_stacks = true;
}
public Position (Tokenizer t)
{
position = t.reader.Position;
line = t.line;
ref_line = t.ref_line;
col = t.col;
hidden = t.hidden;
putback_char = t.putback_char;
previous_col = t.previous_col;
if (t.ifstack != null && t.ifstack.Count != 0)
ifstack = (Stack)t.ifstack.Clone ();
parsing_generic_less_than = t.parsing_generic_less_than;
current_token = t.current_token;
}
public CSharpParser (SeekableStreamReader reader, CompilationSourceFile file)
{
this.file = file;
current_namespace = file.NamespaceContainer;
this.module = current_namespace.Module;
this.compiler = module.Compiler;
this.settings = compiler.Settings;
lang_version = settings.Version;
doc_support = settings.Documentation != null;
current_class = current_namespace.SlaveDeclSpace;
current_container = current_class.PartialContainer; // == RootContest.ToplevelTypes
oob_stack.Clear ();
lexer = new Tokenizer (reader, file, compiler);
use_global_stacks = true;
}
public CSharpParser (SeekableStreamReader reader, CompilationSourceFile file, Report report)
{
this.file = file;
current_container = current_namespace = file;
this.module = file.Module;
this.compiler = file.Compiler;
this.settings = compiler.Settings;
this.report = report;
lang_version = settings.Version;
yacc_verbose_flag = settings.VerboseParserFlag;
doc_support = settings.DocumentationFile != null;
oob_stack.Clear ();
lexer = new Tokenizer (reader, file);
#if FULL_AST
lbag = new LocationsBag ();
#else
lbag = null;
#endif
use_global_stacks = true;
}