public void TestCompleteBuffer()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"ID : 'a'..'z'+;\n" +
"INT : '0'..'9'+;\n" +
"SEMI : ';';\n" +
"ASSIGN : '=';\n" +
"PLUS : '+';\n" +
"MULT : '*';\n" +
"WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
ICharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
Interpreter lexEngine = new Interpreter(g, input);
BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
int i = 1;
IToken t = tokens.LT(i);
while (t.Type != CharStreamConstants.EndOfFile)
{
i++;
t = tokens.LT(i);
}
tokens.LT(i++); // push it past end
tokens.LT(i++);
string result = tokens.ToString();
string expecting = "x = 3 * 0 + 2 * 0;";
Assert.AreEqual(expecting, result);
}
public string GetTextWithWhitespaceUntab(
BufferedTokenStream tokenStream,
ParserRuleContext context,
int untabLevels = -1)
{
return(_stringUtilService.UntabString(GetTextWithWhitespace(tokenStream, context), untabLevels));
}
public void TestLookback()
{
Grammar g = new Grammar(
"lexer grammar t;\n" +
"ID : 'a'..'z'+;\n" +
"INT : '0'..'9'+;\n" +
"SEMI : ';';\n" +
"ASSIGN : '=';\n" +
"PLUS : '+';\n" +
"MULT : '*';\n" +
"WS : ' '+;\n");
// Tokens: 012345678901234567
// Input: x = 3 * 0 + 2 * 0;
ICharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
Interpreter lexEngine = new Interpreter(g, input);
BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
tokens.Consume(); // get x into buffer
IToken t = tokens.LT(-1);
Assert.AreEqual("x", t.Text);
tokens.Consume();
tokens.Consume(); // consume '='
t = tokens.LT(-3);
Assert.AreEqual("x", t.Text);
t = tokens.LT(-2);
Assert.AreEqual(" ", t.Text);
t = tokens.LT(-1);
Assert.AreEqual("=", t.Text);
}
public BreadcrumbControllerInjector(
IStringUtilService stringUtilService,
ICSharpParserService cSharpParserService,
IBreadcrumbCommandParserService breadcrumbCommandParserService,
ICSharpCommonStgService cSharpCommonStgService,
BufferedTokenStream tokenStream,
ControllerDictionary controllerDictionary,
string breadcrumbServiceNamespace,
string controllerRootNamespace,
string defaultAreaBreadcrumbServiceRootName,
string tabString)
{
_stringUtilService = stringUtilService;
_cSharpParserService = cSharpParserService;
_breadcrumbCommandParserService = breadcrumbCommandParserService;
_cSharpCommonStgService = cSharpCommonStgService;
Tokens = tokenStream;
Rewriter = new TokenStreamRewriter(tokenStream);
ControllerDict = controllerDictionary;
_breadcrumbServiceNamespace = breadcrumbServiceNamespace;
_controllerRootNamespace = controllerRootNamespace;
_defaultAreaBreadcrumbServiceRootName = defaultAreaBreadcrumbServiceRootName;
_tabString = tabString;
_currentNamespace = new Stack <string>();
_currentClass = new Stack <string>();
_isControllerClass = new Stack <bool>();
_isControllerClass.Push(false);
_isClassModified = new Stack <bool>();
_isClassModified.Push(false);
IsModified = false;
}
public ServiceConstructorInjector Create(
BufferedTokenStream tokenStream,
string constructorClassName,
string constructorClassNamespace,
string serviceIdentifier,
string serviceNamespace,
string serviceInterfaceType,
FieldDeclaration fieldDeclaration,
FixedParameter constructorParameter,
SimpleAssignment constructorAssignment,
ConstructorDeclaration constructorDeclaration,
string tabString = null)
{
return(new ServiceConstructorInjector(
_stringUtilService,
_cSharpParserService,
_cSharpCommonStgService,
_logger,
tokenStream,
constructorClassName,
constructorClassNamespace,
serviceIdentifier,
serviceNamespace,
serviceInterfaceType,
fieldDeclaration,
constructorParameter,
constructorAssignment,
constructorDeclaration,
tabString));
}
/// <summary>
/// Returns `true` if no line terminator exists after any encountered
/// parameters beyond the specified token offset and the next on the
/// `HIDDEN` channel.
/// </summary>
protected bool noTerminatorAfterParams(int tokenOffset)
{
BufferedTokenStream stream = (BufferedTokenStream)tokenStream;
int leftParams = 1;
int rightParams = 0;
if (LT(stream, tokenOffset).Type == GoLexer.L_PAREN)
{
// Scan past parameters
while (leftParams != rightParams)
{
tokenOffset++;
int tokenType = LT(stream, tokenOffset).Type;
if (tokenType == GoLexer.L_PAREN)
{
leftParams++;
}
else if (tokenType == GoLexer.R_PAREN)
{
rightParams++;
}
}
tokenOffset++;
return(noTerminatorBetween(tokenOffset));
}
return(true);
}
public List <TypeParameter> ParseVariantTypeParameterList(
BufferedTokenStream tokenStream,
CSharpParser.Variant_type_parameterContext[] variantTypeParameters,
CSharpParser.Type_parameter_constraints_clauseContext[] constraintsClauses)
{
var typeParamList = new List <TypeParameter>();
var typeParamDict = new Dictionary <string, TypeParameter>();
if (variantTypeParameters != null)
{
foreach (var variantTypeParameter in variantTypeParameters)
{
var typeParam = variantTypeParameter.identifier().GetText();
var typeParameter = new TypeParameter()
{
TypeParam = typeParam,
VarianceAnnotation = variantTypeParameter?.variance_annotation()?.GetText()
};
typeParamDict.Add(typeParam, typeParameter);
typeParamList.Add(typeParameter);
}
}
if (constraintsClauses != null)
{
foreach (var constraintClause in constraintsClauses)
{
var identifier = constraintClause.identifier().GetText();
var typeParameter = typeParamDict[identifier];
var constraints = constraintClause.type_parameter_constraints();
var primaryConstraint = constraints?.primary_constraint()?.GetText();
if (primaryConstraint != null)
{
typeParameter.Constraints.Add(primaryConstraint);
}
var secondaryConstraints = constraints?.secondary_constraints()?.secondary_constraint();
if (secondaryConstraints != null)
{
foreach (var secondaryConstraint in secondaryConstraints)
{
typeParameter.Constraints.Add(secondaryConstraint.GetText());
}
}
var constructorConstraint = constraints?.constructor_constraint()?.GetText();
if (constructorConstraint != null)
{
typeParameter.Constraints.Add(constructorConstraint);
}
}
}
return(typeParamList);
}
public XDocument(BufferedTokenStream tokenstream, ITextSnapshot snapshot, IList <string> includeFiles)
{
TokenStream = tokenstream;
SnapShot = snapshot;
Entities = null;
Lines = new Dictionary <int, IList <XSharpToken> >();
IncludeFiles = includeFiles;
}
public string GetTextWithWhitespaceMinified(BufferedTokenStream tokenStream, ParserRuleContext context)
{
if (context is null)
{
return(null);
}
return(_stringUtilService.MinifyString(GetTextWithWhitespace(tokenStream, context)));
}
private static void GetLexerErrors(XSharpLexer lexer, BufferedTokenStream tokenStream, List <ParseErrorData> parseErrors)
{
// get lexer errors
foreach (var error in lexer.LexErrors)
{
parseErrors.Add(error);
}
}
public ServiceClassInjector(
IStringUtilService stringUtilService,
ICSharpParserService cSharpParserService,
IServiceCommandParserService serviceCommandParserService,
BufferedTokenStream tokenStream,
string serviceClassInterfaceName,
ServiceFile serviceFile,
string tabString = null)
{
_stringUtilService = stringUtilService;
_cSharpParserService = cSharpParserService;
_serviceCommandParserService = serviceCommandParserService;
Tokens = tokenStream;
Rewriter = new TokenStreamRewriter(tokenStream);
_serviceClassInterfaceName = serviceClassInterfaceName;
_serviceFile = serviceFile;
_tabString = tabString;
_currentNamespace = new Stack <string>();
_currentClass = new Stack <string>();
_isCorrectClass = new Stack <bool>();
_isCorrectClass.Push(false);
_hasServiceNamespace = false;
_hasServiceClass = false;
_hasServiceConstructor = false;
IsModified = false;
_usingSet = _serviceFile.UsingDirectives.ToHashSet();
_ctorParamDict = new Dictionary <string, FixedParameter>();
foreach (var fixedParam in
_serviceFile.ServiceDeclaration.Body.ConstructorDeclaration.FormalParameterList.FixedParameters)
{
_ctorParamDict.Add($"{fixedParam.Type} {fixedParam.Identifier}", fixedParam);
}
_fieldDict = new Dictionary <string, FieldDeclaration>();
foreach (var fieldDec in _serviceFile.ServiceDeclaration.Body.FieldDeclarations)
{
_fieldDict.Add($"{fieldDec.Type} {fieldDec?.VariableDeclarator?.Identifier}", fieldDec);
}
_ctorAssignmentDict = new Dictionary <string, SimpleAssignment>();
var statements = _serviceFile.ServiceDeclaration?.Body?.ConstructorDeclaration?.Body?.Statements;
if (statements != null)
{
foreach (var statement in statements)
{
if (statement.SimpleAssignment != null)
{
var sa = statement.SimpleAssignment;
_ctorAssignmentDict.Add($"{sa.LeftHandSide}={sa.RightHandSide};", sa);
}
}
}
}
public BreadcrumbControllerScraper(
BufferedTokenStream tokenStream)
{
Tokens = tokenStream;
_currentNamespace = new Stack <string>();
_currentClass = new Stack <string>();
_isControllerClass = new Stack <bool>();
_isControllerClass.Push(false);
Results = new ControllerDictionary();
}
public static void Main(string[] args)
{
AntlrInputStream ain = new AntlrInputStream("(a b cd ef (ged))");
SExprLexer lexer = new SExprLexer(ain);
BufferedTokenStream tokens = new BufferedTokenStream(lexer);
tokens.Fill();
SExprParser parser = new SExprParser(tokens);
Console.WriteLine(parser.main());
}
public ServiceInterfaceScraper Create(BufferedTokenStream tokenStream,
string serviceInterfaceName,
string serviceNamespace,
List <TypeParameter> typeParameters)
{
return(new ServiceInterfaceScraper(
_cSharpParserService,
tokenStream,
serviceInterfaceName,
serviceNamespace,
typeParameters));
}
static void Main(string[] args)
{
var a = new StreamReader("./root.txt");
var antlrInput = new AntlrInputStream(a);
var lexer = new CalcLexer(antlrInput);
var tokens = new BufferedTokenStream(lexer);
var parser = new CalcParser(tokens);
lexer.PushLexerState();
IParseTree tree = parser.compileUnit();
Console.WriteLine(tree.ToStringTree());
}
public int lastHiddenTokenType()
{
BufferedTokenStream bts = (BufferedTokenStream)this.InputStream;
IList <IToken> hidden = bts.GetHiddenTokensToLeft(bts.Index);
if (hidden == null || hidden.Count == 0)
{
return(0);
}
else
{
return(hidden[hidden.Count - 1].Type);
}
}
public int nextHiddenTokenType()
{
BufferedTokenStream bts = (BufferedTokenStream)this.InputStream;
IList <IToken> hidden = bts.GetHiddenTokensToRight(bts.Index - 1);
if (hidden == null || hidden.Count == 0)
{
return(0);
}
else
{
return(hidden[0].Type);
}
}
public ServiceStartupRegistration Create(
BufferedTokenStream tokenStream,
string rootNamespace,
StartupRegistrationInfo startupRegInfo,
string tabString = null)
{
return(Create(
tokenStream,
rootNamespace,
new List <StartupRegistrationInfo>()
{
startupRegInfo
},
tabString));
}
public ServiceInterfaceInjector Create(
BufferedTokenStream tokenStream,
string serviceClassInterfaceName,
ServiceFile serviceFile,
string tabString)
{
return(new ServiceInterfaceInjector(
_stringUtilService,
_cSharpParserService,
_serviceCommandParserService,
tokenStream,
serviceClassInterfaceName,
serviceFile,
tabString));
}
public ServiceStartupRegistration Create(
BufferedTokenStream tokenStream,
string rootNamespace,
List <StartupRegistrationInfo> startupRegInfoList,
string tabString = null)
{
return(new ServiceStartupRegistration(
_stringUtilService,
_cSharpParserService,
_serviceCommandStgService,
tokenStream,
rootNamespace,
startupRegInfoList,
tabString));
}
public BreadcrumbClassInjector Create(
BufferedTokenStream tokenStream,
List <string> usingDirectives,
string breadcrumbNamespace,
BreadcrumbServiceDeclaration breadcrumbDeclaration,
string tabString)
{
return(new BreadcrumbClassInjector(
_stringUtilService,
_cSharpParserService,
_breadcrumbCommandParserService,
tokenStream,
usingDirectives,
breadcrumbNamespace,
breadcrumbDeclaration,
tabString));
}
public ServiceClassScraper(
ICSharpParserService cSharpParserService,
BufferedTokenStream tokenStream,
string serviceClassName,
string serviceNamespace,
List <TypeParameter> typeParameters)
{
_cSharpParserService = cSharpParserService;
Tokens = tokenStream;
ServiceClassName = serviceClassName;
TypeParameters = typeParameters;
_serviceNamespace = serviceNamespace;
Rewriter = new TokenStreamRewriter(tokenStream);
Results = new ServiceFile();
_currentNamespace = new Stack <string>();
HasServiceClass = false;
}
public static bool Lex(string sourceText, string fileName, CSharpParseOptions options, IErrorListener listener,
out ITokenStream tokens)
{
tokens = null;
var parseErrors = ParseErrorData.NewBag();
try
{
var lexer = XSharpLexer.Create(sourceText, fileName, options);
lexer.Options = options;
var tokenStream = lexer.GetTokenStream();
tokenStream.Fill();
tokens = tokenStream;
GetLexerErrors(lexer, tokenStream, parseErrors);
#region Determine if we need to preprocess
bool mustPreprocess = true;
if (options.NoStdDef)
{
mustPreprocess = lexer.MustBeProcessed || lexer.HasPreprocessorTokens;
}
#endregion
XSharpPreprocessor pp = null;
BufferedTokenStream ppStream = null;
pp = new XSharpPreprocessor(lexer, tokenStream, options, fileName, Encoding.Unicode, SourceHashAlgorithm.None, parseErrors);
if (mustPreprocess)
{
var ppTokens = pp.PreProcess();
ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, ppTokens));
}
else
{
// No Standard Defs and no preprocessor tokens in the lexer
// so we bypass the preprocessor and use the lexer token stream
ppStream = new CommonTokenStream(new XSharpListTokenSource(lexer, tokenStream.GetTokens()));
}
ppStream.Fill();
}
catch (Exception)
{
}
ReportErrors(parseErrors, listener);
return(tokens != null);
}
public static void Main(string[] args)
{
Syms.init("calc.tokens");
string gdata;
using (var r = new StreamReader("terminals.txt"))
{
gdata = r.ReadToEnd();
}
var tokenizer = new Tokenizer(gdata);
string idata = Console.ReadLine();
tokenizer.setInput(idata);
IList <IToken> tokens = new List <IToken>();
while (true)
{
Token t = tokenizer.next();
if (t.Symbol == "$")
{
break; //at end
}
//CommonToken is defined in the ANTLR runtime
CommonToken T = new CommonToken(Syms.stringToInt[t.Symbol], t.Lexeme);
T.Line = t.Line;
tokens.Add(T);
}
var antlrtokenizer = new BufferedTokenStream(new ListTokenSource(tokens));
var parser = new calcParser(antlrtokenizer);
parser.BuildParseTree = true;
//optional: parser.ErrorHandler = new BailErrorStrategy ();
//'start' should be the name of the grammar's start symbol
var listener = new MyListener();
var walker = new ParseTreeWalker();
var antlrroot = parser.start();
walker.Walk(listener, antlrroot);
double v = Annotations.ptp.Get(antlrroot);
Console.WriteLine($"{v}");
}
public BreadcrumbControllerInjector Create(
BufferedTokenStream tokenStream,
ControllerDictionary controllerDictionary,
string breadcrumbServiceNamespace,
string controllerRootNamespace,
string defaultAreaBreadcrumbServiceRootName,
string tabString)
{
return(new BreadcrumbControllerInjector(
_stringUtilService,
_cSharpParserService,
_breadcrumbCommandParserService,
_cSharpCommonStgService,
tokenStream,
controllerDictionary,
breadcrumbServiceNamespace,
controllerRootNamespace,
defaultAreaBreadcrumbServiceRootName,
tabString));
}
/// <summary>
/// Returns `true` if no line terminator exists between the specified
/// token offset and the prior one on the `HIDDEN` channel.
/// </summary>
protected bool noTerminatorBetween(int tokenOffset)
{
BufferedTokenStream stream = (BufferedTokenStream)tokenStream;
IList <IToken> tokens = stream.GetHiddenTokensToLeft(LT(stream, tokenOffset).TokenIndex);
if (tokens == null)
{
return(true);
}
foreach (IToken token in tokens)
{
if (token.Text.Contains("\n"))
{
return(false);
}
}
return(true);
}
public ServiceStartupRegistration(
IStringUtilService stringUtilService,
ICSharpParserService cSharpParserService,
IServiceCommandStgService serviceCommandStgService,
BufferedTokenStream tokenStream,
string rootNamespace,
List <StartupRegistrationInfo> startupRegInfoList,
string tabString = null)
{
_stringUtilService = stringUtilService;
_cSharpParserService = cSharpParserService;
_serviceCommandStgService = serviceCommandStgService;
Tokens = tokenStream;
Rewriter = new TokenStreamRewriter(tokenStream);
_rootNamespace = rootNamespace;
_startupRegInfoList = startupRegInfoList;
_tabString = tabString;
_currentNamespace = new Stack <string>();
_currentClass = new Stack <string>();
IsModified = false;
}
public FormalParameterList ParseFormalParameterList(
BufferedTokenStream tokenStream,
CSharpParser.Formal_parameter_listContext formalParameterList)
{
FormalParameterList formalParamList = null;
if (formalParameterList != null)
{
formalParamList = new FormalParameterList();
var fixedParameters = formalParameterList?.fixed_parameters()?.fixed_parameter();
if (fixedParameters != null)
{
foreach (var fixedParameter in fixedParameters)
{
formalParamList.FixedParameters.Add(
new FixedParameter()
{
Attributes = GetTextWithWhitespace(tokenStream, fixedParameter?.attributes()),
ParameterModifier = fixedParameter?.parameter_modifier()?.GetText(),
Type = GetTextWithWhitespaceMinifiedLite(tokenStream, fixedParameter.type_()),
Identifier = fixedParameter.identifier().GetText(),
DefaultArgument = GetTextWithWhitespace(
tokenStream, fixedParameter?.default_argument()?.expression())
}
);
}
}
var parameterArray = formalParameterList?.parameter_array();
if (parameterArray != null)
{
formalParamList.ParameterArray = new ParameterArray()
{
Attributes = GetTextWithWhitespace(tokenStream, parameterArray?.attributes()),
Type = GetTextWithWhitespaceMinifiedLite(tokenStream, parameterArray.array_type()),
Identifier = parameterArray.identifier().GetText()
};
}
}
return(formalParamList);
}
public string GetTextWithWhitespace(BufferedTokenStream tokenStream, ParserRuleContext context)
{
if (context is null)
{
return(null);
}
var startToken = context.Start.TokenIndex;
var stopToken = context.Stop.TokenIndex;
var tokens = tokenStream.Get(startToken, stopToken);
var filteredText = new StringBuilder();
foreach (var t in tokens)
{
if (t.Channel == Lexer.DefaultTokenChannel || t.Channel == Lexer.Hidden)
{
filteredText.Append(t.Text);
}
}
return(filteredText.ToString());
}
private ParserRuleContext ParseTokens(
AntlrMemoryErrorListener errorListener, BufferedTokenStream codeTokenStream,
Func <ITokenStream, Parser> initParserFunc = null, Func <Parser, ParserRuleContext> parseFunc = null)
{
Parser parser = initParserFunc != null?initParserFunc(codeTokenStream) : InitParser(codeTokenStream);
parser.Interpreter = new ParserATNSimulator(parser, GetOrCreateAtn(ParserSerializedATN));
parser.RemoveErrorListeners();
ParserRuleContext syntaxTree;
if (UseFastParseStrategyAtFirst)
{
parser.Interpreter.PredictionMode = PredictionMode.Sll;
parser.ErrorHandler = new BailErrorStrategy();
parser.TrimParseTree = true;
try
{
syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser);
}
catch (ParseCanceledException)
{
parser.AddErrorListener(errorListener);
codeTokenStream.Reset();
parser.Reset();
parser.Interpreter.PredictionMode = PredictionMode.Ll;
parser.ErrorHandler = new DefaultErrorStrategy();
syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser);
}
}
else
{
parser.AddErrorListener(errorListener);
syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser);
}
return(syntaxTree);
}