/// <summary> /// Initializes a new instance of the <see cref="Parser"/> class. /// </summary> /// <param name="language">The language.</param> /// <param name="scanner">The scanner.</param> /// <param name="root">The root.</param> /// <exception cref="Exception"> /// </exception> public Parser(LanguageData language, Scanner scanner, NonTerminal root) { Language = language; Context = new ParsingContext(this); Scanner = scanner ?? language.CreateScanner(); if (Scanner != null) { Scanner.Initialize(this); } else { Language.Errors.Add(GrammarErrorLevel.Error, null, "Scanner is not initialized for this grammar"); } CoreParser = new CoreParser(this); Root = root; if (Root == null) { Root = Language.Grammar.Root; InitialState = Language.ParserData.InitialState; } else { if (Root != Language.Grammar.Root && !Language.Grammar.SnippetRoots.Contains(Root)) { throw new Exception(string.Format(Resources.ErrRootNotRegistered, root.Name)); } InitialState = Language.ParserData.InitialStates[Root]; } }
private Token CompleteMatch(ParsingContext context, ISourceStream source) { //Find end symbol while (!source.EOF()) { int firstCharPos; if (EndSymbols.Count == 1) firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition); else firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition); if (firstCharPos < 0) { source.PreviewPosition = source.Text.Length; return null; //indicating error } //We found a character that might start an end symbol; let's see if it is true. source.PreviewPosition = firstCharPos; foreach (string endSymbol in EndSymbols) { if (source.MatchSymbol(endSymbol)) { //We found end symbol; eat end symbol only if it is not line comment. // For line comment, leave LF symbol there, it might be important to have a separate LF token if (!_isLineComment) source.PreviewPosition += endSymbol.Length; return source.CreateToken(this.OutputTerminal); }//if }//foreach endSymbol source.PreviewPosition++; //move to the next char and try again }//while return null; //might happen if we found a start char of end symbol, but not the full endSymbol }//method
public void Init(ParsingContext context, ParseTreeNode parseNode) { ParamType type = ParamType.Value; Object param = null; if (HasChildNodes(parseNode)) { throw new NotImplementedQLException("TODO"); ///Das sollte hier jetzt viel einfacher sein, weil die definition einer edge maximal so aussehen kann User(Weighted) oder Set<User (Weighted)> // throw new NotImplementedQLException("TODO"); // if (GraphDBTypeMapper.IsBasicType(parseNode.ChildNodes[0].Token.ValueString)) // { // param = GraphDBTypeMapper.GetGraphObjectFromTypeName(parseNode.ChildNodes[0].Token.ValueString); // type = ParamType.Type; // } // else // { // param = parseNode.ChildNodes[0].Token.Value; // type = ParamType.Value; // } //} //EdgeTypeParamDefinition = new EdgeTypeParamDefinition(type, param); } }
public void DirectInit(ParsingContext context, ParseTreeNode parseNode) { var _elementsToBeAdded = (CollectionOfDBObjectsNode)parseNode.ChildNodes[3].AstNode; var _AttrName = parseNode.ChildNodes[2].FirstChild.FirstChild.Token.ValueString; AttributeUpdateList = new AttributeAssignOrUpdateList(_elementsToBeAdded.CollectionDefinition, ((IDNode)parseNode.ChildNodes[2].AstNode).IDChainDefinition, false); }
private Token CreateIncompleteToken(ParsingContext context, ISourceStream source) { source.PreviewPosition = source.Text.Length; Token result = source.CreateToken(this.OutputTerminal); result.Flags |= TokenFlags.IsIncomplete; context.VsLineScanState.TerminalIndex = this.MultilineIndex; return result; }
public override Token TryMatch(ParsingContext context, ISourceStream source) { Token result; if (context.VsLineScanState.Value != 0) { byte commentLevel = context.VsLineScanState.TokenSubType; result = CompleteMatch(context, source, commentLevel); } else { byte commentLevel = 0; if (!BeginMatch(context, source, ref commentLevel)) return null; result = CompleteMatch(context, source, commentLevel); } if (result != null) return result; if (context.Mode == ParseMode.VsLineScan) return CreateIncompleteToken(context, source); return context.CreateErrorToken("unclosed comment"); }
public void DirectInit(ParsingContext context, ParseTreeNode parseNode) { var idChain = ((IDNode)parseNode.ChildNodes[2].AstNode).IDChainDefinition; var tupleDefinition = ((TupleNode)parseNode.ChildNodes[3].AstNode).TupleDefinition; var AttrName = parseNode.ChildNodes[2].FirstChild.FirstChild.Token.ValueString; ToBeRemovedList = new AttributeRemoveList(idChain, AttrName, tupleDefinition); }
public void Init(ParsingContext context, ParseTreeNode parseNode) { var aSelectNode = (SelectNode)parseNode.ChildNodes[0].AstNode; SelectDefinition = new SelectDefinition(aSelectNode.TypeList, aSelectNode.SelectedElements, aSelectNode.WhereExpressionDefinition, aSelectNode.GroupByIDs, aSelectNode.Having, aSelectNode.Limit, aSelectNode.Offset, aSelectNode.OrderByDefinition, aSelectNode.ResolutionDepth); }
public void Init(ParsingContext context, ParseTreeNode parseNode) { if (HasChildNodes(parseNode)) { BinExprNode = (BinaryExpressionNode)parseNode.ChildNodes[1].AstNode; } }
public override void Init(ParsingContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); foreach (var child in treeNode.ChildNodes) { AddChild("parameter", child); } AsString = "Param list"; }
public override Token TryMatch(ParsingContext context, ISourceStream source) { Match m = _expression.Match(source.Text, source.PreviewPosition); if (!m.Success || m.Index != source.PreviewPosition) return null; source.PreviewPosition += m.Length; return source.CreateToken(this.OutputTerminal); }
//Completes reduce: pops child nodes from the stack and pushes result node into the stack protected void CompleteReduce(ParsingContext context) { var resultNode = context.CurrentParserInput; var childCount = Production.RValues.Count; //Pop stack context.ParserStack.Pop(childCount); //Copy comment block from first child; if comments precede child node, they precede the parent as well. if (resultNode.ChildNodes.Count > 0) resultNode.Comments = resultNode.ChildNodes[0].Comments; //Inherit precedence and associativity, to cover a standard case: BinOp->+|-|*|/; // BinOp node should inherit precedence from underlying operator symbol. //TODO: this special case will be handled differently. A ToTerm method should be expanded to allow "combined" terms like "NOT LIKE". // OLD COMMENT: A special case is SQL operator "NOT LIKE" which consists of 2 tokens. We therefore inherit "max" precedence from any children if (Production.LValue.Flags.IsSet(TermFlags.InheritPrecedence)) InheritPrecedence(resultNode); //Push new node into stack and move to new state //First read the state from top of the stack context.CurrentParserState = context.ParserStack.Top.State; if (context.TracingEnabled) context.AddTrace(Resources.MsgTracePoppedState, Production.LValue.Name); #region comments on special case //Special case: if a non-terminal is Transient (ex: BinOp), then result node is not this NonTerminal, but its its child (ex: symbol). // Shift action will invoke OnShifting on actual term being shifted (symbol); we need to invoke Shifting even on NonTerminal itself // - this would be more expected behavior in general. ImpliedPrecHint relies on this #endregion if (resultNode.Term != Production.LValue) //special case Production.LValue.OnShifting(context.SharedParsingEventArgs); // Shift to new state - execute shift over the non-terminal of the production. var shift = context.CurrentParserState.Actions[Production.LValue]; // Execute shift to new state shift.Execute(context); //Invoke Reduce event Production.LValue.OnReduced(context, Production, resultNode); }
public override void Init(ParsingContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); TargetRef = AddChild("Target", treeNode.ChildNodes[0]); _targetName = treeNode.ChildNodes[0].FindTokenAndGetText(); Arguments = AddChild("Args", treeNode.ChildNodes[1]); AsString = "Call " + _targetName; }
public void Init(ParsingContext context, ParseTreeNode parseNode) { if (HasChildNodes(parseNode)) { //get type if (parseNode.ChildNodes[1] != null && parseNode.ChildNodes[1].AstNode != null) { _Type = ((AstNode)(parseNode.ChildNodes[1].AstNode)).AsString; } else { throw new NotImplementedQLException(""); } if (parseNode.ChildNodes[3] != null && HasChildNodes(parseNode.ChildNodes[3])) { _AttributeAssignList = new List<AAttributeAssignOrUpdate>((parseNode.ChildNodes[3].AstNode as AttributeUpdateOrAssignListNode).ListOfUpdate.Select(e => e as AAttributeAssignOrUpdate)); } if (parseNode.ChildNodes[4] != null && ((WhereExpressionNode)parseNode.ChildNodes[4].AstNode).BinaryExpressionDefinition != null) { _WhereExpression = ((WhereExpressionNode)parseNode.ChildNodes[4].AstNode).BinaryExpressionDefinition; } } }
public override void Init(ParsingContext context, ParseTreeNode parseNode) { base.Init(context, parseNode); foreach (var node in parseNode.ChildNodes) { if (node.AstNode is Function) { AddFunction(node.AstNode as Function); } else if (node.AstNode is AuxiliaryNode) { var ids = (node.AstNode as AuxiliaryNode).ChildNodes.OfType<IdentifierNode>(); foreach (var id in ids) { ExternalFunction ef = new ExternalFunction(); ef.SetSpan(id.Span); ef.Name = id.Symbol; AddFunction(ef); } } } AsString = "Refal-5 program"; }
private bool BeginMatch(ParsingContext context, ISourceStream source) { //Check starting symbol if (!source.MatchSymbol(StartSymbol)) return false; source.PreviewPosition += StartSymbol.Length; return true; }
private static string ReadQuotedBody(ParsingContext context, ISourceStream source) { const char dQuoute = '"'; StringBuilder sb = null; var from = source.Location.Position + 1; //skip initial double quote while(true) { var until = source.Text.IndexOf(dQuoute, from); if (until < 0) throw new Exception(Resources.ErrDsvNoClosingQuote); // "Could not find a closing quote for quoted value." source.PreviewPosition = until; //now points at double-quote var piece = source.Text.Substring(from, until - from); source.PreviewPosition++; //move after double quote if (source.PreviewChar != dQuoute && sb == null) return piece; //quick path - if sb (string builder) was not created yet, we are looking at the very first segment; // and if we found a standalone dquote, then we are done - the "piece" is the result. if (sb == null) sb = new StringBuilder(100); sb.Append(piece); if (source.PreviewChar != dQuoute) return sb.ToString(); //we have doubled double-quote; add a single double-quoute char to the result and move over both symbols sb.Append(dQuoute); from = source.PreviewPosition + 1; } }
public override void Init(ParsingContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); Value = treeNode.Token.Value; AsString = Value == null ? "null" : Value.ToString(); if (Value is string) AsString = "\"" + AsString + "\""; }
public override void Init(ParsingContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); foreach (var node in treeNode.ChildNodes) { // linearize AuxiliaryNode children if (node.AstNode is AuxiliaryNode) { var auxNode = node.AstNode as AuxiliaryNode; foreach (var n in auxNode.ChildNodes) ChildNodes.Add(n); foreach (var n in auxNode.ChildParseNodes) ChildParseNodes.Add(n); continue; } // copy AstNode nodes if (node.AstNode is AstNode) { ChildNodes.Add(node.AstNode as AstNode); continue; } // otherwise, save parse nodes ChildParseNodes.Add(node); } }
public override void Init(ParsingContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); Test = AddChild("Test", treeNode.ChildNodes[0]); IfTrue = AddChild("IfTrue", treeNode.ChildNodes[1]); if (treeNode.ChildNodes.Count > 2) IfFalse = AddChild("IfFalse", treeNode.ChildNodes[2]); }
private Token CompleteMatch(ParsingContext context, ISourceStream source, byte level) { string text = source.Text.Substring(source.PreviewPosition); var matches = Regex.Matches(text, @"\](=*)\]"); foreach(Match match in matches) { if (match.Groups[1].Value.Length == (int)level) { source.PreviewPosition += match.Index + match.Length; if (context.VsLineScanState.Value != 0) { SourceLocation tokenStart = new SourceLocation(); tokenStart.Position = 0; string lexeme = source.Text.Substring(0, source.PreviewPosition); context.VsLineScanState.Value = 0; return new Token(this, tokenStart, lexeme, null); } else { return source.CreateToken(this.OutputTerminal); } } } context.VsLineScanState.TerminalIndex = this.MultilineIndex; context.VsLineScanState.TokenSubType = level; return null; }
public override void Init(ParsingContext context, ParseTreeNode treeNode) { var address = treeNode.ChildNodes[0]; base.Init(context, address); AsString = address.Token.Text; S7Address = (sBLOCK200_ADDRESS_STRUCT)address.Token.Value; }
public override IEnumerable<Token> BeginFiltering(ParsingContext context, IEnumerable<Token> tokens) { this.pcontext = context; foreach (Token t in tokens) { Console.WriteLine(" -> {0} ({1})", t, bracket_indent_level); currentLoc = t.Location; if (t.Terminal == grammar.Eof) { Console.WriteLine("CLI: {0}, CIL: {1}", current_line_indent, current_indent_level); current_line_indent = 0; while(current_indent_level > 0) { current_indent_level--; Console.WriteLine(" <- DEDENT"); yield return new Token(grammar.Dedent,currentLoc,string.Empty,null); } yield return t; break; } if (bracket_indent_level == 0) { while (ProcessToken(t)) { ;} while (OutputTokens.Count > 0) yield return OutputTokens.Pop(); continue; } if (t.Terminal == opening_bracket) { bracket_indent_level++; yield return new Token(grammar.Indent, currentLoc, string.Empty, null); continue; } else if (t.Terminal == closing_bracket) { Debug.Assert(bracket_indent_level > 0); bracket_indent_level--; yield return new Token(grammar.Dedent, currentLoc, string.Empty, null); continue; } yield return t; } }
public override void Init(ParsingContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); foreach (var child in treeNode.MappedChildNodes) AddChild(NodeUseType.Parameter, "param", child); AsString = "param_list[" + ChildNodes.Count + "]"; }
public void Init(ParsingContext context, ParseTreeNode parseNode) { if (HasChildNodes(parseNode)) { ParallelTasks = Convert.ToUInt32(parseNode.ChildNodes[1].Token.Value); } }
public override Token TryMatch(ParsingContext context, ISourceStream source) { string tokenText = string.Empty; while (true) { //Find next position var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition); if(newPos == -1) { if(IsSet(FreeTextOptions.AllowEof)) { source.PreviewPosition = source.Text.Length; return source.CreateToken(this.OutputTerminal); } else return null; } if (newPos == source.PreviewPosition) // DC { context.AddParserError("(DC) in TryMatch, newPos == source.PreviewPosition", new object[] {}); break; // DC } tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition); source.PreviewPosition = newPos; //if it is escape, add escaped text and continue search if (CheckEscape(source, ref tokenText)) continue; //check terminators if (CheckTerminators(source, ref tokenText)) break; //from while (true) } return source.CreateToken(this.OutputTerminal, tokenText); }
/// <summary> /// Converts identifiers to compound symbols (strings in double quotes), /// expands character strings (in single quotes) to arrays of characters /// </summary> public static void InitNode(ParsingContext context, ParseTreeNode parseNode) { foreach (var node in parseNode.ChildNodes) { if (node.AstNode is LiteralValueNode) { if (node.Term.Name == "Char") { var literal = node.AstNode as LiteralValueNode; literal.Value = literal.Value.ToString().ToCharArray(); } parseNode.AstNode = node.AstNode; } else { // identifiers in expressions are treated as strings (True is same as "True") parseNode.AstNode = new LiteralValueNode() { Value = node.FindTokenAndGetText(), Span = node.Span }; } } }
public void Init(ParsingContext context, ParseTreeNode parseNode) { if (HasChildNodes(parseNode)) { //get type if (parseNode.ChildNodes[1] != null && parseNode.ChildNodes[1].AstNode != null) { _type = ((AstNode)(parseNode.ChildNodes[1].AstNode)).AsString; } else { throw new NotImplementedQLException(""); } if (parseNode.ChildNodes[3] != null && HasChildNodes(parseNode.ChildNodes[3])) { _attributeAssignList = (parseNode.ChildNodes[3].AstNode as AttributeAssignListNode).AttributeAssigns; } if (parseNode.ChildNodes[4] != null && ((WhereExpressionNode)parseNode.ChildNodes[4].AstNode).BinaryExpressionDefinition != null) { _whereExpression = ((WhereExpressionNode)parseNode.ChildNodes[4].AstNode).BinaryExpressionDefinition; } } }
public void Init(ParsingContext context, ParseTreeNode parseNode) { var multiplyString = parseNode.ChildNodes[0].Token.Text.ToUpper(); CollectionType collectionType; switch (multiplyString) { case SonesGQLConstants.LISTOF: collectionType = CollectionType.List; break; case SonesGQLConstants.SETOF: collectionType = CollectionType.Set; break; case SonesGQLConstants.SETOFUUIDS: default: throw new InvalidOperationException( String.Format("The specified command [{0}] is not valid.", multiplyString)); } if (parseNode.ChildNodes[1].AstNode is TupleNode) { CollectionDefinition = new CollectionDefinition(collectionType, ((TupleNode)parseNode.ChildNodes[1].AstNode).TupleDefinition); } else { throw new NotImplementedException("The following node cannot be created."); } }
public void Init(ParsingContext context, ParseTreeNode parseNode) { #region get Name _TypeName = parseNode.ChildNodes[0].Token.ValueString; #endregion #region get Extends if (HasChildNodes(parseNode.ChildNodes[1])) _Extends = parseNode.ChildNodes[1].ChildNodes[1].Token.ValueString; #endregion #region get myAttributes if (HasChildNodes(parseNode.ChildNodes[2])) _Attributes = GetAttributeList(parseNode.ChildNodes[2].ChildNodes[1]); #endregion #region get Comment if (HasChildNodes(parseNode.ChildNodes[3])) _Comment = parseNode.ChildNodes[3].ChildNodes[2].Token.ValueString; #endregion }
public override void Init(Irony.Parsing.ParsingContext context, Irony.Parsing.ParseTreeNode treeNode) { base.Init(context, treeNode); if (treeNode.Token.ValueString.Length > MaxLen) { context.AddParserMessage(ParserErrorLevel.Error, this.Span, "The maximum length is {0} character{1}.", MaxLen, MaxLen != 1 ? "s" : string.Empty); } }
public override void Init(Irony.Parsing.ParsingContext context, ParseTreeNode treeNode) { base.Init(context, treeNode); Expression = ExpressionNode.Reduce(treeNode.ChildNodes[1]); ChildNodes.Add(Expression); Expression.Parent = this; m_Sections = new List <SwitchSectionNode>(treeNode.LastChild.ChildNodes.Count); foreach (ParseTreeNode node in treeNode.LastChild.ChildNodes) { SwitchSectionNode section = (SwitchSectionNode)node.AstNode; m_Sections.Add(section); ChildNodes.Add(section); section.Parent = this; } }
public override void Init(Irony.Parsing.ParsingContext context, Irony.Parsing.ParseTreeNode treeNode) { base.Init(context, treeNode); DateTime date; var cleanDate = GetCleanDate(treeNode); if (!TryParseDate(cleanDate, out date)) { var message = string.Format("{0} is not a valid date", GetCleanDate(treeNode)); throw new AstException(this, message); } Value = date; AsString = cleanDate; }
protected override void InitDetails(ParsingContext context, CompoundTokenDetails details) { base.InitDetails(context, details); details.Flags = (short)Options; }
/// <summary> /// Override this method to help scanner select a terminal to create token when there are more than one candidates /// for an input char. context.CurrentTerminals contains candidate terminals; leave a single terminal in this list /// as the one to use. /// </summary> public virtual void OnScannerSelectTerminal(ParsingContext context) { }
//Most numbers in source programs are just one-digit instances of 0, 1, 2, and maybe others until 9 // so we try to do a quick parse for these, without starting the whole general process protected override Token QuickParse(ParsingContext context, ISourceStream source) { if (IsSet(NumberOptions.DisableQuickParse)) { return(null); } var current = source.PreviewChar; //it must be a digit followed by a whitespace or delimiter if (!char.IsDigit(current)) { return(null); } if (!Grammar.IsWhitespaceOrDelimiter(source.NextPreviewChar)) { return(null); } var iValue = current - '0'; object value; switch (DefaultIntTypes[0]) { case TypeCode.Int32: value = iValue; break; case TypeCode.UInt32: value = (uint)iValue; break; case TypeCode.Byte: value = (byte)iValue; break; case TypeCode.SByte: value = (sbyte)iValue; break; case TypeCode.Int16: value = (short)iValue; break; case TypeCode.UInt16: value = (ushort)iValue; break; case TypeCode.Int64: value = (long)iValue; break; case TypeCode.UInt64: value = (ulong)iValue; break; default: return(null); } source.PreviewPosition++; return(source.CreateToken(OutputTerminal, value)); }
//Constructs the error message in situation when parser has no available action for current input. // override this method if you want to change this message public virtual string ConstructParserErrorMessage(ParsingContext context, StringSet expectedTerms) { return string.Format(Resources.ErrParserUnexpInput, expectedTerms.ToString(" ")); }
//This method is called if Scanner fails to produce a token; it offers custom method a chance to produce the token public virtual Token TryMatch(ParsingContext context, ISourceStream source) { return(null); }
public abstract IEnumerable <Token> BeginFiltering(ParsingContext context, IEnumerable <Token> tokens);
//Gives a chance of custom AST node creation at Grammar level // by default calls Term's method public virtual void CreateAstNode(ParsingContext context, ParseTreeNode nodeInfo) { nodeInfo.Term.CreateAstNode(context, nodeInfo); }
public ParsingEventArgs(ParsingContext context) { Context = context; }
public override Token TryMatch(ParsingContext context, ISourceStream source) { return(source.CreateToken(this)); //Create an empty token representing an implied symbol. }
protected virtual Token QuickParse(ParsingContext context, ISourceStream source) { return(null); }
internal void OnReduced(ParsingContext context, Production reducedProduction, ParseTreeNode resultNode) { Reduced?.Invoke(this, new ReducedEventArgs(context, reducedProduction, resultNode)); }
public override Token TryMatch(ParsingContext context, ISourceStream source) { Token token; //Try quick parse first, but only if we're not continuing if (context.VsLineScanState.Value == 0) { token = QuickParse(context, source); if (token != null) { return(token); } source.PreviewPosition = source.Position; //revert the position } CompoundTokenDetails details = new CompoundTokenDetails(); InitDetails(context, details); if (context.VsLineScanState.Value == 0) { ReadPrefix(source, details); } if (!ReadBody(source, details)) { return(null); } if (details.Error != null) { return(context.CreateErrorToken(details.Error)); } if (details.IsPartial) { details.Value = details.Body; } else { ReadSuffix(source, details); if (!ConvertValue(details, context)) { if (string.IsNullOrEmpty(details.Error)) { details.Error = Resources.ErrInvNumber; } return(context.CreateErrorToken(details.Error)); // "Failed to convert the value: {0}" } } token = CreateToken(context, source, details); if (details.IsPartial) { //Save terminal state so we can continue context.VsLineScanState.TokenSubType = (byte)details.SubTypeIndex; context.VsLineScanState.TerminalFlags = (short)details.Flags; context.VsLineScanState.TerminalIndex = this.MultilineIndex; } else { context.VsLineScanState.Value = 0; } return(token); }
} //method protected virtual bool ConvertValue(CompoundTokenDetails details, ParsingContext context) { details.Value = details.Body; return(false); }
public virtual void Execute(ParsingContext context) { }
protected virtual void InitDetails(ParsingContext context, CompoundTokenDetails details) { details.PartialOk = (context.Mode == ParseMode.VsLineScan); details.PartialContinues = (context.VsLineScanState.Value != 0); }