/// <summary> /// Read DFA state information. /// </summary> private void ReadDfaStates(BinaryReader reader) { Int32 index = ReadInt16Entry(reader); Boolean acceptState = ReadBoolEntry(reader); Int16 acceptIndex = ReadInt16Entry(reader); ReadEmptyEntry(reader); // Read DFA edges var edges = new DfaEdge[m_EntryCount / 3]; for (Int32 i = 0; i < edges.Length; i++) { edges[i] = new DfaEdge(ReadInt16Entry(reader), ReadInt16Entry(reader)); ReadEmptyEntry(reader); } // Create DFA state and store it in DFA state table Symbol acceptSymbol = acceptState ? m_SymbolTable[acceptIndex] : null; ObjectMap transitionVector = CreateDfaTransitionVector(edges); var dfaState = new DfaState(index, acceptSymbol, transitionVector); m_DfaStateTable[index] = dfaState; }
/// <summary> /// Reads table record counts and initializes tables. /// </summary> private void ReadTableCounts(BinaryReader reader) { // Initialize tables m_SymbolTable = new Symbol[ReadInt16Entry(reader)]; m_CharSetTable = new String[ReadInt16Entry(reader)]; m_RuleTable = new Rule[ReadInt16Entry(reader)]; m_DfaStateTable = new DfaState[ReadInt16Entry(reader)]; m_LRStateTable = new LRState[ReadInt16Entry(reader)]; }
/// <summary> /// Loads grammar from the binary reader. /// </summary> private void Load() { if (FileHeader != ReadString()) { throw new FileLoadException(SR.GetString(SR.Grammar_WrongFileHeader)); } while (m_reader.PeekChar() != -1) { RecordType recordType = ReadNextRecord(); switch (recordType) { case RecordType.Parameters: ReadHeader(); break; case RecordType.TableCounts: ReadTableCounts(); break; case RecordType.Initial: ReadInitialStates(); break; case RecordType.Symbols: ReadSymbols(); break; case RecordType.CharSets: ReadCharSets(); break; case RecordType.Rules: ReadRules(); break; case RecordType.DfaStates: ReadDfaStates(); break; case RecordType.LRStates: ReadLRStates(); break; default: throw new FileLoadException(SR.GetString(SR.Grammar_InvalidRecordType)); } } m_dfaInitialState = m_dfaStateTable[m_dfaInitialStateIndex]; OptimizeDfaTransitionVectors(); }
/// <summary> /// Read DFA state information. /// </summary> private void ReadDfaStates() { int index = ReadInt16Entry(); Symbol acceptSymbol = null; bool acceptState = ReadBoolEntry(); if (acceptState) { acceptSymbol = m_symbolTable[ReadInt16Entry()]; } else { ReadInt16Entry(); // Skip the entry. } ReadEmptyEntry(); // Read DFA edges DfaEdge[] edges = new DfaEdge[m_entryCount/3]; for (int i = 0; i < edges.Length; i++) { edges[i].CharSetIndex = ReadInt16Entry(); edges[i].TargetIndex = ReadInt16Entry(); ReadEmptyEntry(); } // Create DFA state and store it in DFA state table ObjectMap transitionVector = CreateDfaTransitionVector(edges); DfaState dfaState = new DfaState(index, acceptSymbol, transitionVector); m_dfaStateTable[index] = dfaState; }
/// <summary> /// Reads next token from the input stream. /// </summary> /// <returns>Token symbol which was read.</returns> public Symbol ReadToken() { m_token.m_text = null; m_token.m_start = m_charIndex; m_token.m_lineNumber = m_lineNumber; m_token.m_linePosition = m_charIndex - m_lineStart + 1; int lookahead = m_charIndex; // Next look ahead char in the input int tokenLength = 0; Symbol tokenSymbol = null; DfaState[] dfaStateTable = m_grammar.m_dfaStateTable; // End of buffer if (m_buffer.Length == lookahead) { m_token.m_symbol = m_grammar.m_endSymbol; m_token.m_length = 0; return(m_token.m_symbol); } char ch = m_buffer[lookahead]; DfaState dfaState = m_grammar.m_dfaInitialState; while (true) { dfaState = dfaState.m_transitionVector[ch] as DfaState; // This block-if statement checks whether an edge was found from the current state. // If so, the state and current position advance. Otherwise it is time to exit the main loop // and report the token found (if there was it fact one). If the LastAcceptState is -1, // then we never found a match and the Error Token is created. Otherwise, a new token // is created using the Symbol in the Accept State and all the characters that // comprise it. if (dfaState != null) { // This code checks whether the target state accepts a token. If so, it sets the // appropiate variables so when the algorithm in done, it can return the proper // token and number of characters. lookahead++; if (dfaState.m_acceptSymbol != null) { tokenSymbol = dfaState.m_acceptSymbol; tokenLength = lookahead - m_charIndex; } if (m_buffer.Length == lookahead) { ch = EndOfString; m_preserveChars = lookahead - m_charIndex; // Found end of of stream lookahead = m_charIndex + m_preserveChars; m_preserveChars = 0; } else { ch = m_buffer[lookahead]; } } else { if (tokenSymbol != null) { m_token.m_symbol = tokenSymbol; m_token.m_length = tokenLength; MoveBy(tokenLength); } else { //Tokenizer cannot recognize symbol m_token.m_symbol = m_grammar.m_errorSymbol; m_token.m_length = 1; MoveBy(1); } break; } } return(m_token.m_symbol); }