/// <summary> /// Parses the input and returns the produced AST /// </summary> /// <returns>AST produced by the parser representing the input, or null if unrecoverable errors were encountered</returns> public override ParseResult Parse() { reductions = new Queue <Reduction>(); shifts = new Queue <Shift>(); int Ui = gss.CreateGeneration(); int v0 = gss.CreateNode(0); nextToken = lexer.GetNextToken(this); // bootstrap the shifts and reductions queues int count = parserAutomaton.GetActionsCount(0, nextToken.TerminalID); for (int i = 0; i != count; i++) { LRAction action = parserAutomaton.GetAction(0, nextToken.TerminalID, i); if (action.Code == LRActionCode.Shift) { shifts.Enqueue(new Shift(v0, action.Data)); } else if (action.Code == LRActionCode.Reduce) { reductions.Enqueue(new Reduction(v0, parserAutomaton.GetProduction(action.Data), SPPF.EPSILON)); } } while (nextToken.TerminalID != Symbol.SID_EPSILON) // Wait for ε token { // the stem length (initial number of nodes in the generation before reductions) int stem = gss.GetGeneration(Ui).Count; // apply all reduction actions Reducer(Ui); // no scheduled shift actions? if (shifts.Count == 0) { // the next token was not expected OnUnexpectedToken(stem); return(new ParseResult(new ROList <ParseError>(allErrors), lexer.Input)); } // look for the next next-token Lexer.TokenKernel oldtoken = nextToken; nextToken = lexer.GetNextToken(this); // apply the scheduled shift actions Ui = Shifter(oldtoken); } GSSGeneration genData = gss.GetGeneration(Ui); for (int i = genData.Start; i != genData.Start + genData.Count; i++) { int state = gss.GetRepresentedState(i); if (parserAutomaton.IsAcceptingState(state)) { // Has reduction _Axiom_ -> axiom $ . on ε GSSPath[] paths = gss.GetPaths(i, 2, out count); return(new ParseResult(new ROList <ParseError>(allErrors), lexer.Input, sppf.GetTree(paths[0][1]))); } } // At end of input but was still waiting for tokens return(new ParseResult(new ROList <ParseError>(allErrors), lexer.Input)); }
/// <summary> /// Initializes a new automaton from the given binary stream /// </summary> /// <param name="reader">The binary stream to load from</param> public LRkAutomaton(BinaryReader reader) { ncols = reader.ReadUInt16(); nstates = reader.ReadUInt16(); int nprod = reader.ReadUInt16(); columns = new ColumnMap(); for (int i = 0; i != ncols; i++) { columns.Add(reader.ReadUInt16(), i); } contexts = new LRContexts[nstates]; for (int i = 0; i != nstates; i++) { contexts[i] = new LRContexts(reader); } table = new LRAction[nstates * ncols]; for (int i = 0; i != nstates * ncols; i++) { table[i] = new LRAction(reader); } productions = new LRProduction[nprod]; for (int i = 0; i != nprod; i++) { productions[i] = new LRProduction(reader); } }
/// <summary> /// Executes a shift operation /// </summary> /// <param name="gen">The GSS generation to start from</param> /// <param name="label">The GSS label to use for the new GSS edges</param> /// <param name="shift">The shift operation</param> private void ExecuteShift(int gen, int label, Shift shift) { int w = gss.FindNode(gen, shift.to); if (w != -1) { // A node for the target state is already in the GSS gss.CreateEdge(w, shift.from, label); // Look for the new reductions at this state int count = parserAutomaton.GetActionsCount(shift.to, nextToken.TerminalID); for (int i = 0; i != count; i++) { LRAction action = parserAutomaton.GetAction(shift.to, nextToken.TerminalID, i); if (action.Code == LRActionCode.Reduce) { LRProduction prod = parserAutomaton.GetProduction(action.Data); // length 0 reduction are not considered here because they already exist at this point if (prod.ReductionLength != 0) { reductions.Enqueue(new Reduction(shift.from, prod, label)); } } } } else { // Create the new corresponding node in the GSS w = gss.CreateNode(shift.to); gss.CreateEdge(w, shift.from, label); // Look for all the reductions and shifts at this state int count = parserAutomaton.GetActionsCount(shift.to, nextToken.TerminalID); for (int i = 0; i != count; i++) { LRAction action = parserAutomaton.GetAction(shift.to, nextToken.TerminalID, i); if (action.Code == LRActionCode.Shift) { shifts.Enqueue(new Shift(w, action.Data)); } else if (action.Code == LRActionCode.Reduce) { LRProduction prod = parserAutomaton.GetProduction(action.Data); if (prod.ReductionLength == 0) // Length 0 => reduce from the head { reductions.Enqueue(new Reduction(w, prod, SPPF.EPSILON)); } else // reduce from the second node on the path { reductions.Enqueue(new Reduction(shift.from, prod, label)); } } } } }
/// <summary> /// Gets the next RNGLR state by a shift with the given variable ID /// </summary> /// <param name="state">A RNGLR state</param> /// <param name="var">A variable ID</param> /// <returns>The next RNGLR state, or 0xFFFF if no transition is found</returns> private int GetNextByVar(int state, int var) { int ac = parserAutomaton.GetActionsCount(state, var); for (int i = 0; i != ac; i++) { LRAction action = parserAutomaton.GetAction(state, var, i); if (action.Code == LRActionCode.Shift) { return(action.Data); } } return(0xFFFF); }
/// <summary> /// Gets the expected terminals for the specified state /// </summary> /// <param name="state">The DFA state</param> /// <param name="terminals">The possible terminals</param> /// <returns>The expected terminals</returns> public LRExpected GetExpected(int state, ROList <Symbol> terminals) { LRExpected result = new LRExpected(); int offset = ncols * state; for (int i = 0; i != terminals.Count; i++) { LRAction action = table[offset]; if (action.Code == LRActionCode.Shift) { result.Shifts.Add(terminals[i]); } else if (action.Code == LRActionCode.Reduce) { result.Reductions.Add(terminals[i]); } offset++; } return(result); }
/// <summary> /// Initializes a new automaton from the given binary stream /// </summary> /// <param name="reader">The binary stream to load from</param> public RNGLRAutomaton(BinaryReader reader) { axiom = reader.ReadUInt16(); ncols = reader.ReadUInt16(); nstates = reader.ReadUInt16(); int nactions = (int)reader.ReadUInt32(); int nprod = reader.ReadUInt16(); int nnprod = reader.ReadUInt16(); columns = new ColumnMap(); for (int i = 0; i != ncols; i++) { columns.Add(reader.ReadUInt16(), i); } contexts = new LRContexts[nstates]; for (int i = 0; i != nstates; i++) { contexts[i] = new LRContexts(reader); } table = new Cell[nstates * ncols]; for (int i = 0; i != table.Length; i++) { table[i] = new Cell(reader); } actions = new LRAction[nactions]; for (int i = 0; i != nactions; i++) { actions[i] = new LRAction(reader); } productions = new LRProduction[nprod]; for (int i = 0; i != nprod; i++) { productions[i] = new LRProduction(reader); } nullables = new ushort[nnprod]; for (int i = 0; i != nnprod; i++) { nullables[i] = reader.ReadUInt16(); } }
/// <summary> /// Gets the expected terminals for the specified state /// </summary> /// <param name="state">The DFA state</param> /// <param name="terminals">The possible terminals</param> /// <returns>The expected terminals</returns> public LRExpected GetExpected(int state, ROList <Symbol> terminals) { LRExpected result = new LRExpected(); for (int i = 0; i != terminals.Count; i++) { Cell cell = table[state * ncols + i]; for (int j = 0; j != cell.ActionsCount; j++) { LRAction action = actions[cell.ActionsIndex + j]; if (action.Code == LRActionCode.Shift) { result.AddUniqueShift(terminals[i]); } else if (action.Code == LRActionCode.Reduce) { result.AddUniqueReduction(terminals[i]); } } } return(result); }
/// <summary> /// Parses on the specified token kernel /// </summary> /// <param name="kernel">The token kernel to parse on</param> /// <returns>The LR action that was used</returns> private LRActionCode ParseOnToken(Lexer.TokenKernel kernel) { while (true) { LRAction action = automaton.GetAction(stack[head], kernel.TerminalID); if (action.Code == LRActionCode.Shift) { head++; if (head == stack.Length) { Array.Resize(ref stack, stack.Length + INIT_STACK_SIZE); Array.Resize(ref stackIDs, stackIDs.Length + INIT_STACK_SIZE); } stack[head] = action.Data; stackIDs[head] = kernel.TerminalID; builder.StackPushToken(kernel.Index); return(action.Code); } if (action.Code == LRActionCode.Reduce) { LRProduction production = automaton.GetProduction(action.Data); head -= production.ReductionLength; Reduce(production); action = automaton.GetAction(stack[head], symVariables[production.Head].ID); head++; if (head == stack.Length) { Array.Resize(ref stack, stack.Length + INIT_STACK_SIZE); Array.Resize(ref stackIDs, stackIDs.Length + INIT_STACK_SIZE); } stack[head] = action.Data; stackIDs[head] = symVariables[production.Head].ID; continue; } return(action.Code); } }
/// <summary> /// Checks whether the specified terminal is indeed expected for a reduction /// </summary> /// <param name="terminal">The terminal to check</param> /// <returns><code>true</code> if the terminal is really expected</returns> /// <remarks> /// This check is required because in the case of a base LALR graph, /// some terminals expected for reduction in the automaton are coming from other paths. /// </remarks> private bool CheckIsExpected(Symbol terminal) { // copy the stack to use for the simulation int[] myStack = new int[stack.Length]; Array.Copy(stack, myStack, head + 1); int myHead = head; // get the action for the stack's head LRAction action = automaton.GetAction(myStack[myHead], terminal.ID); while (action.Code != LRActionCode.None) { if (action.Code == LRActionCode.Shift) { // yep, the terminal was expected return(true); } if (action.Code == LRActionCode.Reduce) { // execute the reduction LRProduction production = automaton.GetProduction(action.Data); myHead -= production.ReductionLength; // this must be a shift action = automaton.GetAction(myStack[myHead], symVariables[production.Head].ID); myHead++; if (myHead == myStack.Length) { Array.Resize(ref myStack, myStack.Length + INIT_STACK_SIZE); } myStack[myHead] = action.Data; // now, get the new action for the terminal action = automaton.GetAction(action.Data, terminal.ID); } } // nope, that was a pathological case in a LALR graph return(false); }
/// <summary> /// Gets the priority of the specified context required by the specified terminal /// The priority is a positive integer. The lesser the value the higher the priority. /// A value of -1 represents the unavailability of the required context. /// </summary> /// <param name="context">A context</param> /// <param name="onTerminalID">The identifier of the terminal requiring the context</param> /// <returns>The context priority, or -1 if the context is unavailable</returns> public int GetContextPriority(int context, int onTerminalID) { // the default context is always active if (context == Automaton.DEFAULT_CONTEXT) { return(int.MaxValue); } if (lexer.tokens.Size == 0) { // this is the first token, does it open the context? return(automaton.GetContexts(0).Opens(onTerminalID, context) ? 0 : -1); } // retrieve the action for this terminal LRAction action = automaton.GetAction(stack[head], onTerminalID); // if the terminal is unexpected, do not validate if (action.Code == LRActionCode.None) { return(-1); } // does the context opens with the terminal? if (action.Code == LRActionCode.Shift && automaton.GetContexts(stack[head]).Opens(onTerminalID, context)) { return(0); } LRProduction production = (action.Code == LRActionCode.Reduce) ? automaton.GetProduction(action.Data) : null; // look into the stack for the opening of the context for (int i = head - 1; i != -1; i--) { if (automaton.GetContexts(stack[i]).Opens(stackIDs[i + 1], context)) { // the context opens here // but is it closed by the reduction (if any)? if (production == null || i < head - production.ReductionLength) { // no, we are still in the context return(head - i); } } } // at this point, the requested context is not yet open or is closed by a reduction // now, if the action is something else than a reduction (shift, accept or error), the context can never be produced // for the context to open, a new state must be pushed onto the stack // this means that the provided terminal must trigger a chain of at least one reduction if (action.Code != LRActionCode.Reduce) { return(-1); } // there is at least one reduction, simulate int[] myStack = new int[stack.Length]; Array.Copy(stack, myStack, head + 1); int myHead = head; while (action.Code == LRActionCode.Reduce) { // execute the reduction production = automaton.GetProduction(action.Data); myHead -= production.ReductionLength; // this must be a shift action = automaton.GetAction(myStack[myHead], symVariables[production.Head].ID); myHead++; if (myHead == myStack.Length) { Array.Resize(ref myStack, myStack.Length + INIT_STACK_SIZE); } myStack[myHead] = action.Data; // now, get the new action for the terminal action = automaton.GetAction(action.Data, onTerminalID); } // is this a shift action that opens the context? return((action.Code == LRActionCode.Shift && automaton.GetContexts(myStack[myHead]).Opens(onTerminalID, context)) ? 0 : -1); }
/// <summary> /// Executes a reduction operation for a given path /// </summary> /// <param name="generation">The current GSS generation</param> /// <param name="reduction">The reduction operation</param> /// <param name="path">The GSS path to use for the reduction</param> private void ExecuteReduction(int generation, Reduction reduction, GSSPath path) { // Get the rule's head Symbol head = symVariables[reduction.prod.Head]; // Resolve the sub-root int label = sppf.GetLabelFor(path.Generation, new TableElemRef(TableType.Variable, reduction.prod.Head)); if (label == SPPF.EPSILON) { // not in history, build the SPPF here label = BuildSPPF(generation, reduction.prod, reduction.first, path); } // Get the target state by transition on the rule's head int to = GetNextByVar(gss.GetRepresentedState(path.Last), head.ID); // Find a node for the target state in the GSS int w = gss.FindNode(generation, to); if (w != -1) { // A node for the target state is already in the GSS if (!gss.HasEdge(generation, w, path.Last)) { // But the new edge does not exist gss.CreateEdge(w, path.Last, label); // Look for the new reductions at this state if (reduction.prod.ReductionLength != 0) { int count = parserAutomaton.GetActionsCount(to, nextToken.TerminalID); for (int i = 0; i != count; i++) { LRAction action = parserAutomaton.GetAction(to, nextToken.TerminalID, i); if (action.Code == LRActionCode.Reduce) { LRProduction prod = parserAutomaton.GetProduction(action.Data); // length 0 reduction are not considered here because they already exist at this point if (prod.ReductionLength != 0) { reductions.Enqueue(new Reduction(path.Last, prod, label)); } } } } } } else { // Create the new corresponding node in the GSS w = gss.CreateNode(to); gss.CreateEdge(w, path.Last, label); // Look for all the reductions and shifts at this state int count = parserAutomaton.GetActionsCount(to, nextToken.TerminalID); for (int i = 0; i != count; i++) { LRAction action = parserAutomaton.GetAction(to, nextToken.TerminalID, i); if (action.Code == LRActionCode.Shift) { shifts.Enqueue(new Shift(w, action.Data)); } else if (action.Code == LRActionCode.Reduce) { LRProduction prod = parserAutomaton.GetProduction(action.Data); if (prod.ReductionLength == 0) { reductions.Enqueue(new Reduction(w, prod, SPPF.EPSILON)); } else if (reduction.prod.ReductionLength != 0) { reductions.Enqueue(new Reduction(path.Last, prod, label)); } } } } }
/// <summary> /// Checks whether the specified terminal is indeed expected for a reduction /// </summary> /// <param name="gssNode">The GSS node from which to reduce</param> /// <param name="terminal">The terminal to check</param> /// <returns><code>true</code> if the terminal is really expected</returns> /// <remarks> /// This check is required because in the case of a base LALR graph, /// some terminals expected for reduction in the automaton are coming from other paths. /// </remarks> private bool CheckIsExpected(int gssNode, Symbol terminal) { // queue of GLR states to inspect: List <int> queueGSSHead = new List <int>(); // the related GSS head List <int[]> queueVStack = new List <int[]>(); // the virtual stack // first reduction { int count = parserAutomaton.GetActionsCount(gss.GetRepresentedState(gssNode), terminal.ID); for (int j = 0; j != count; j++) { LRAction action = parserAutomaton.GetAction(gss.GetRepresentedState(gssNode), terminal.ID, j); if (action.Code == LRActionCode.Reduce) { // execute the reduction LRProduction production = parserAutomaton.GetProduction(action.Data); int nbPaths; GSSPath[] paths = gss.GetPaths(gssNode, production.ReductionLength, out nbPaths); for (int k = 0; k != nbPaths; k++) { GSSPath path = paths[k]; // get the target GLR state int next = GetNextByVar(gss.GetRepresentedState(path.Last), symVariables[production.Head].ID); // enqueue the info, top GSS stack node and target GLR state queueGSSHead.Add(path.Last); queueVStack.Add(new[] { next }); } } } } // now, close the queue for (int i = 0; i != queueGSSHead.Count; i++) { int head = queueVStack[i][queueVStack[i].Length - 1]; int count = parserAutomaton.GetActionsCount(head, terminal.ID); if (count == 0) { continue; } for (int j = 0; j != count; j++) { LRAction action = parserAutomaton.GetAction(head, terminal.ID, j); if (action.Code == LRActionCode.Shift) { // yep, the terminal was expected return(true); } if (action.Code == LRActionCode.Reduce) { // execute the reduction LRProduction production = parserAutomaton.GetProduction(action.Data); if (production.ReductionLength == 0) { // 0-length reduction => start from the current head int[] virtualStack = new int[queueVStack[i].Length + 1]; Array.Copy(queueVStack[i], virtualStack, queueVStack[i].Length); virtualStack[virtualStack.Length - 1] = GetNextByVar(head, symVariables[production.Head].ID); // enqueue queueGSSHead.Add(queueGSSHead[i]); queueVStack.Add(virtualStack); } else if (production.ReductionLength < queueVStack[i].Length) { // we are still the virtual stack int[] virtualStack = new int[queueVStack[i].Length - production.ReductionLength + 1]; Array.Copy(queueVStack[i], virtualStack, virtualStack.Length - 1); virtualStack[virtualStack.Length - 1] = GetNextByVar(virtualStack[virtualStack.Length - 2], symVariables[production.Head].ID); // enqueue queueGSSHead.Add(queueGSSHead[i]); queueVStack.Add(virtualStack); } else { // we reach the GSS int nbPaths; GSSPath[] paths = gss.GetPaths(queueGSSHead[i], production.ReductionLength - queueVStack[i].Length, out nbPaths); for (int k = 0; k != nbPaths; k++) { GSSPath path = paths[k]; // get the target GLR state int next = GetNextByVar(gss.GetRepresentedState(path.Last), symVariables[production.Head].ID); // enqueue the info, top GSS stack node and target GLR state queueGSSHead.Add(path.Last); queueVStack.Add(new[] { next }); } } } } } // nope, that was a pathological case in a LALR graph return(false); }
/// <summary> /// Gets the priority of the specified context required by the specified terminal /// The priority is a positive integer. The lesser the value the higher the priority. /// A value of -1 represents the unavailability of the required context. /// </summary> /// <param name="context">A context</param> /// <param name="onTerminalID">The identifier of the terminal requiring the context</param> /// <returns>The context priority, or -1 if the context is unavailable</returns> public int GetContextPriority(int context, int onTerminalID) { // the default context is always active if (context == Automaton.DEFAULT_CONTEXT) { return(int.MaxValue); } if (lexer.tokens.Size == 0) { // this is the first token, does it open the context? return(parserAutomaton.GetContexts(0).Opens(onTerminalID, context) ? 0 : -1); } // try to only look at stack heads that expect the terminal List <int> queue = new List <int>(); List <LRProduction> productions = new List <LRProduction>(); List <int> distances = new List <int>(); bool foundOnPreviousShift = false; foreach (Shift shift in shifts) { int count = parserAutomaton.GetActionsCount(shift.to, onTerminalID); if (count == 0) { continue; } for (int i = 0; i != count; i++) { LRAction action = parserAutomaton.GetAction(shift.to, onTerminalID, i); if (action.Code == LRActionCode.Shift) { // does the context opens with the terminal? if (parserAutomaton.GetContexts(shift.to).Opens(onTerminalID, context)) { return(0); } // looking at the immediate history, does the context opens from the shift just before? if (parserAutomaton.GetContexts(gss.GetRepresentedState(shift.from)).Opens(nextToken.TerminalID, context)) { foundOnPreviousShift = true; break; } // no, enqueue if (!queue.Contains(shift.from)) { queue.Add(shift.from); productions.Add(null); distances.Add(2); } } else { // this is reduction LRProduction production = parserAutomaton.GetProduction(action.Data); // looking at the immediate history, does the context opens from the shift just before? if (parserAutomaton.GetContexts(gss.GetRepresentedState(shift.from)).Opens(nextToken.TerminalID, context)) { if (production.ReductionLength < 1) { // the reduction does not close the context foundOnPreviousShift = true; break; } } // no, enqueue if (!queue.Contains(shift.from)) { queue.Add(shift.from); productions.Add(production); distances.Add(2); } } } } if (foundOnPreviousShift) { // found the context opening on the previous shift (and was not immediately closed by a reduction) return(1); } if (queue.Count == 0) { // the track is empty, the terminal is unexpected return(-1); } // explore the current GSS to find the specified context for (int i = 0; i != queue.Count; i++) { int count; GSSPath[] paths = gss.GetPaths(queue[i], 1, out count); for (int p = 0; p != count; p++) { int from = paths[p].Last; int symbolID = sppf.GetSymbolOn(paths[p][0]).ID; int distance = distances[i]; LRProduction production = productions[i]; // was the context opened on this transition? if (parserAutomaton.GetContexts(gss.GetRepresentedState(from)).Opens(symbolID, context)) { if (production == null || production.ReductionLength < distance) { return(distance); } } // no, enqueue if (!queue.Contains(from)) { queue.Add(from); productions.Add(production); distances.Add(distance + 1); } } } // at this point, the requested context is not yet open // can it be open by a token with the specified terminal ID? // queue of GLR states to inspect: List <int> queueGSSHead = new List <int>(); // the related GSS head List <int[]> queueVStack = new List <int[]>(); // the virtual stack // first reduction foreach (Shift shift in shifts) { int count = parserAutomaton.GetActionsCount(shift.to, onTerminalID); if (count > 0) { // enqueue the info, top GSS stack node and target GLR state queueGSSHead.Add(shift.from); queueVStack.Add(new[] { shift.to }); } } // now, close the queue for (int i = 0; i != queueGSSHead.Count; i++) { int head = queueVStack[i][queueVStack[i].Length - 1]; int count = parserAutomaton.GetActionsCount(head, onTerminalID); if (count == 0) { continue; } for (int j = 0; j != count; j++) { LRAction action = parserAutomaton.GetAction(head, onTerminalID, j); if (action.Code != LRActionCode.Reduce) { continue; } // execute the reduction LRProduction production = parserAutomaton.GetProduction(action.Data); if (production.ReductionLength == 0) { // 0-length reduction => start from the current head int[] virtualStack = new int[queueVStack[i].Length + 1]; Array.Copy(queueVStack[i], virtualStack, queueVStack[i].Length); int next = GetNextByVar(head, symVariables[production.Head].ID); virtualStack[virtualStack.Length - 1] = next; // enqueue queueGSSHead.Add(queueGSSHead[i]); queueVStack.Add(virtualStack); } else if (production.ReductionLength < queueVStack[i].Length) { // we are still the virtual stack int[] virtualStack = new int[queueVStack[i].Length - production.ReductionLength + 1]; Array.Copy(queueVStack[i], virtualStack, virtualStack.Length - 1); int next = GetNextByVar(virtualStack[virtualStack.Length - 2], symVariables[production.Head].ID); virtualStack[virtualStack.Length - 1] = next; // enqueue queueGSSHead.Add(queueGSSHead[i]); queueVStack.Add(virtualStack); } else { // we reach the GSS int nbPaths; GSSPath[] paths = gss.GetPaths(queueGSSHead[i], production.ReductionLength - queueVStack[i].Length, out nbPaths); for (int k = 0; k != nbPaths; k++) { GSSPath path = paths[k]; // get the target GLR state int next = GetNextByVar(gss.GetRepresentedState(path.Last), symVariables[production.Head].ID); // enqueue the info, top GSS stack node and target GLR state queueGSSHead.Add(path.Last); queueVStack.Add(new[] { next }); } } } } foreach (int[] vstack in queueVStack) { int state = vstack[vstack.Length - 1]; int count = parserAutomaton.GetActionsCount(state, onTerminalID); for (int i = 0; i != count; i++) { LRAction action = parserAutomaton.GetAction(state, onTerminalID, i); if (action.Code == LRActionCode.Shift && parserAutomaton.GetContexts(state).Opens(onTerminalID, context)) { // the context opens here return(0); } } } // the context is still unavailable return(-1); }