public CSharpOutput(DFA dfa) { var map = new Dictionary<State, int>(); int counter = 0; foreach (var st in dfa.States) { map[st] = counter++; } sb.AppendLine("namespace TestNS { static class Test { public static bool Match(System.IO.TextReader tr) {"); sb.AppendLine("int ch;"); sb.AppendFormat("goto state{0};", map[dfa.StartState]).AppendLine(); foreach (var st in dfa.States) { sb.AppendFormat("state{0}: {{", map[st]).AppendLine(); sb.AppendLine("\tch = tr.Read();"); foreach (var t in dfa.Transitions.Where(t => t.Item1 == st)) { sb.AppendFormat("\tif (ch == '{0}')", t.Item2.Value).AppendLine(); sb.AppendFormat("\t\tgoto state{0};", map[t.Item3]).AppendLine(); } sb.AppendFormat("\treturn {0};", st.IsFinal ? "ch == -1" : "false").AppendLine(); sb.AppendLine("}"); } sb.AppendLine("}}}"); }
static void Main(string[] args) { DFA M = new DFA();//initialize the DFA //prompt for filename Console.WriteLine("Input filename"); string filename = Console.ReadLine(); M.ReadDFA(filename);//converts from file to the DFA states //prompt for input Console.WriteLine("Input test string, or quit"); string input = Console.ReadLine(); while (input.ToLower() != "quit") { //M.ReadString returns true if after reading //the string, M is in a final state if (M.ReadString(input.ToLower())) { Console.WriteLine("String Accepted"); } else { Console.WriteLine("String Rejected"); } Console.WriteLine("Input test string, or quit"); input = Console.ReadLine(); } }
public MachineProbe(DFA dfa) { if (dfa == null) throw new ArgumentNullException("dfa"); this._dfa = dfa; }
public RegExpEvaluator(string regexp) { infix2postfix = new Infix2PostfixHelper(); automatonConstructor = new ThompsonHelper(); DfaConstructor = new SubsetConstructor(); String expression = infix2postfix.converter(regexp); Graph final; automaton = automatonConstructor.getAutomaton(expression, out final); DfaConstructor = new SubsetConstructor(); DfaAutomaton = DfaConstructor.BuildDfa(automaton, final); }
public override void ReportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, int prediction, SimulatorState acceptState) { }
//DFA runs within the scope of this method. According to states of DFA is filled CsvParserResult with parsed data. public CsvParserResult GetData() { DFA dfa = new DFA(this.recordSeparator, this.fieldSeparator, this.textDelimiter); bool firstLine = true; string field = ""; int columns = 0; // počet sloupců (od 1) int originalColumnNames = 0; //počet jmen sloupců nalezených v datech int currentColumn = 0; // index aktuální buňky (od 0) int rows = 0; // počet řádků (od 1) List<string> columnNames = new List<string>(); // jména sloupců (od 0) List<List<string>> cells = new List<List<string>>(); // data (od 0,0) List<int> recordsLength = new List<int>(); // délky záznamů for (int characterIndex = 0; characterIndex < data.Length; ++characterIndex) { char c = data[characterIndex]; if (firstLine) { DFA.State s = dfa.Next(c); if (s == DFA.State.Final) { if (this.hasHeader) { string header = this.Sanitize(field); if (string.IsNullOrEmpty(header)) { throw new CsvParserException(Language.GetString("parser.emptyHeaderError")); } if (columnNames.IndexOf(header) != -1) { throw new CsvParserException(Language.GetString("parser.columnNameError")); } columnNames.Add(header); cells.Add(new List<string>()); ++originalColumnNames; } else { columnNames.Add(Language.GetString("parser.column") + " " + (currentColumn + 1).ToString()); List<string> l = new List<string>(); l.Add(this.Sanitize(field)); cells.Add(l); } ++currentColumn; field = ""; if (c == this.recordSeparator) { firstLine = false; if (!this.hasHeader) { recordsLength.Add(currentColumn); ++rows; } columns = currentColumn; currentColumn = 0; } } else if (s == DFA.State.Fail) { throw new CsvParserException(Language.GetString("parser.dataFormatError")); } else { field += c; } } else { DFA.State s = dfa.Next(c); if (s == DFA.State.Final) { if (currentColumn < columns) { cells[currentColumn].Add(this.Sanitize(field)); } else { ++columns; if (columnNames.IndexOf(Language.GetString("parser.column") + " " + columns.ToString()) != -1) { throw new CsvParserException(Language.GetString("parser.columnNameError")); } columnNames.Add(Language.GetString("parser.column") + " " + columns.ToString()); List<string> l = new List<string>(new string[rows]); l.Add(this.Sanitize(field)); cells.Add(l); } ++currentColumn; if (c == this.recordSeparator) { if (currentColumn < columns) { for (int i = currentColumn; i < columns; ++i) { cells[i].Add(null); } } recordsLength.Add(currentColumn); currentColumn = 0; ++rows; } field = ""; } else if (s == DFA.State.Fail) { throw new CsvParserException(Language.GetString("parser.dataFormatError")); } else { field += c; } } } return new CsvParserResult(false, originalColumnNames, columns, rows, new ReadOnlyCollection<string>(columnNames), new ReadOnlyCollection<List<string>>(cells), new ReadOnlyCollection<int>(recordsLength)); }
private static void printGraph(DFA theAutomaton, System.IO.StreamWriter writer) { foreach (KeyValuePair<KeyValuePair<dfaState, input>, dfaState> pair in theAutomaton.transitionTable) { writer.Write(pair.Key.Key.ToString() + " -> " + pair.Value.ToString() + " [label=" + pair.Key.Value + "];"); if (theAutomaton.final.Contains(pair.Value)) writer.Write(pair.Value.ToString() + " [ style=filled color=\"dodgerblue\" fillcolor=\"lightyellow\" ]"); } }
public static DFA BuildLr0(List<GrammarRule> AugmentedGrammar, out HashSet<Tuple<dfaState, Symbol, Int32>> ReduceStates, out HashSet<input> Tokens) { Tokens = new HashSet<input>(); ReduceStates = new HashSet<Tuple<dfaState, Symbol, Int32>>(); DFA E = new DFA(); E.start = 0; List<Lr0State> T = new List<Lr0State>(); GrammarRule FirstProduction = AugmentedGrammar[0]; Lr0State InitialState = new Lr0State(); InitialState.Add(new Item(FirstProduction)); InitialState = Closure(InitialState, AugmentedGrammar); T.Add(InitialState); bool hasChanged = false; do { hasChanged = false; List<Lr0State> tempState = new List<Lr0State>(); foreach (Lr0State s in T) { tempState.Add(s); } foreach (Lr0State I in tempState) { foreach (Item item in I.itemSet) { if (!item.isFinal()) { Symbol X = item.getCurrentSymbol(); if (X.id != "úEndSymbol") { Lr0State J = Goto(I, X, AugmentedGrammar); int JIndex = -1; if (T.Contains(J)) { JIndex = T.IndexOf(J); } else { hasChanged = true; T.Add(J); JIndex = T.IndexOf(J); } if (item.getNextSymbol() != null) if (item.getNextSymbol().id == "úEndSymbol") E.final.Add(JIndex); int Iindex = T.IndexOf(I); string TransitionSymbol = X.id; Tokens.Add(TransitionSymbol); if (!E.transitionTable.ContainsKey(new KeyValuePair<int, string>(Iindex, TransitionSymbol))) E.transitionTable.Add(new KeyValuePair<int, string>(Iindex, TransitionSymbol), JIndex); } } else { HashSet<Symbol> followSet = Follow(item.Rule.RuleSymbol, AugmentedGrammar); foreach (Symbol x in followSet) { ReduceStates.Add(new Tuple<dfaState, Symbol, Int32>(T.IndexOf(I), x, AugmentedGrammar.IndexOf(item.Rule))); } } } } } while (hasChanged); return E; }
/// <summary> /// Получить грамматику из файла /// </summary> /// <param name="path">Полный путь к файлу</param> /// <returns>Объект Грамматики</returns> public static Grammar FromFile(string path) { // определяем кодировку m_Encoding = new UnicodeEncoding(false, true); m_Reader = new BinaryReader(new FileStream(path, FileMode.Open)); // Header == Заголовок if(ReadString() != "GOLD Parser Tables/v5.0") { m_Reader.Close(); throw new Exception("Invalid file format."); } // инициализируем объект грамматики Grammar grammar = new Grammar(); // начало чтения записей файла грамматики #region РАЗБОР ФАЙЛА ГРАММАТИКИ EGT-формат try { byte read_byte = m_Reader.ReadByte(); char rbch = (char)read_byte; while(read_byte == (byte)'M') { Int16 entries = m_Reader.ReadInt16(); // // Read entry // read_byte = m_Reader.ReadByte(); rbch = (char)read_byte; read_byte = m_Reader.ReadByte(); rbch = (char)read_byte; if (read_byte == (byte)'p') { // name m_Reader.ReadBytes(3); // index m_Reader.ReadByte(); // string prefix (83) ReadString(); // property name m_Reader.ReadByte(); // string prefix (83) grammar.Name = ReadString(); // property value // version m_Reader.ReadByte(); // new record ('M') m_Reader.ReadBytes(2); // number of entities m_Reader.ReadByte(); // byte separator m_Reader.ReadByte(); // property tag ('p') m_Reader.ReadBytes(3); // index m_Reader.ReadByte(); // string prefix (83) ReadString(); // property name m_Reader.ReadByte(); // string prefix (83) grammar.Version = ReadString(); // property value // author m_Reader.ReadByte(); // new record ('M') m_Reader.ReadBytes(2); // number of entities m_Reader.ReadByte(); // byte separator m_Reader.ReadByte(); // property tag ('p') m_Reader.ReadBytes(3); // index m_Reader.ReadByte(); // string prefix (83) ReadString(); // property name m_Reader.ReadByte(); // string prefix (83) grammar.Author = ReadString(); // property value // about m_Reader.ReadByte(); // new record ('M') m_Reader.ReadBytes(2); // number of entities m_Reader.ReadByte(); // byte separator m_Reader.ReadByte(); // property tag ('p') m_Reader.ReadBytes(3); // index m_Reader.ReadByte(); // string prefix (83) ReadString(); // property name m_Reader.ReadByte(); // string prefix (83) grammar.About = ReadString(); // property value // character set m_Reader.ReadByte(); // new record ('M') m_Reader.ReadBytes(2); // number of entities m_Reader.ReadByte(); // byte separator m_Reader.ReadByte(); // property tag ('p') m_Reader.ReadBytes(3); // index m_Reader.ReadByte(); // string prefix (83) ReadString(); // property name m_Reader.ReadByte(); // string prefix (83) //grammar. = ReadString(); // property value string character_set = ReadString(); // character mapping m_Reader.ReadByte(); // new record ('M') m_Reader.ReadBytes(2); // number of entities m_Reader.ReadByte(); // byte separator m_Reader.ReadByte(); // property tag ('p') m_Reader.ReadBytes(3); // index m_Reader.ReadByte(); // string prefix (83) ReadString(); // property name m_Reader.ReadByte(); // string prefix (83) //grammar. = ReadString(); // property value string character_mapping = ReadString(); // generated by m_Reader.ReadByte(); // new record ('M') m_Reader.ReadBytes(2); // number of entities m_Reader.ReadByte(); // byte separator m_Reader.ReadByte(); // property tag ('p') m_Reader.ReadBytes(3); // index m_Reader.ReadByte(); // string prefix (83) ReadString(); // property name m_Reader.ReadByte(); // string prefix (83) //grammar. = ReadString(); // property value string generated_by = ReadString(); // generation date m_Reader.ReadByte(); // new record ('M') m_Reader.ReadBytes(2); // number of entities m_Reader.ReadByte(); // byte separator m_Reader.ReadByte(); // property tag ('p') m_Reader.ReadBytes(3); // index m_Reader.ReadByte(); // string prefix (83) ReadString(); // property name m_Reader.ReadByte(); // string prefix (83) string generated_date = ReadString(); grammar.IsCaseSensitive = true; read_byte = m_Reader.ReadByte(); rbch = (char)read_byte; } else if (read_byte == (byte)'t') { /// /// Инициализация рабочих таблиц /// // таблица символов m_Reader.ReadByte(); grammar.SymbolTable = new Symbol[m_Reader.ReadInt16()]; // таблица групп сиволов m_Reader.ReadByte(); grammar.CharacterSetTable = new CharacterSet[m_Reader.ReadInt16()]; // таблица правил m_Reader.ReadByte(); grammar.RuleTable = new Rule[m_Reader.ReadInt16()]; // таблица состояний КА m_Reader.ReadByte(); grammar.DFATable = new DFA[m_Reader.ReadInt16()]; // таблица состояний LALR анализатора m_Reader.ReadByte(); grammar.LALRTable = new LALR[m_Reader.ReadInt16()]; // таблица ролей символов (групп по ролям) m_Reader.ReadByte(); int[] group_table = new int[m_Reader.ReadInt16()]; read_byte = m_Reader.ReadByte(); rbch = (char)read_byte; } else if (read_byte == (byte)'c') { /// /// Определение таблицы наборов символов /// CharacterSet item = new CharacterSet(); int temp = -1; rbch = (char)m_Reader.ReadByte(); item.Index = m_Reader.ReadInt16(); rbch = (char)m_Reader.ReadByte(); temp = m_Reader.ReadInt16(); // Codepage m_Reader.ReadByte(); // Range int total = m_Reader.ReadInt16(); rbch = (char)m_Reader.ReadByte(); // Reserved string char_str = string.Empty; while (true) { read_byte = m_Reader.ReadByte(); if (read_byte == 73) { int start_index = m_Reader.ReadInt16(); m_Reader.ReadByte(); int end_index = m_Reader.ReadInt16(); for (int ci = Math.Min(start_index, end_index); ci <= Math.Max(start_index, end_index); ci++) { char_str += Convert.ToChar(ci); } } else { rbch = (char)read_byte; break; } } item.Characters = char_str; grammar.CharacterSetTable[item.Index] = item; } else if (read_byte == (byte)'g') { /// /// Определение таблицы групп символов /// m_Reader.ReadByte(); int index = m_Reader.ReadInt16(); m_Reader.ReadByte(); string name = ReadString(); m_Reader.ReadByte(); int container_index = m_Reader.ReadInt16(); m_Reader.ReadByte(); int start_index = m_Reader.ReadInt16(); m_Reader.ReadByte(); int end_index = m_Reader.ReadInt16(); m_Reader.ReadByte(); int advanced_mode = m_Reader.ReadInt16(); m_Reader.ReadByte(); int ending_mode = m_Reader.ReadInt16(); m_Reader.ReadByte(); while (true) { read_byte = m_Reader.ReadByte(); if (read_byte == 73) { int group_index = m_Reader.ReadInt16(); } else { rbch = (char)read_byte; break; } } } else if (read_byte == (byte)'R') { /// /// Определение таблицы правил /// Rule item = new Rule(); m_Reader.ReadByte(); item.Index = m_Reader.ReadInt16(); m_Reader.ReadByte(); item.Nonterminal = m_Reader.ReadInt16(); m_Reader.ReadByte(); // читаем индексы символов item.Symbols = new short[entries - 4]; for (int x = 0; x < item.Symbols.Length; x++) { m_Reader.ReadByte(); item.Symbols[x] = m_Reader.ReadInt16(); } grammar.RuleTable[item.Index] = item; read_byte = m_Reader.ReadByte(); rbch = (char)read_byte; } else if (read_byte == (byte)'S') { /// /// Определение таблицы символов /// Symbol item = new Symbol(); m_Reader.ReadByte(); item.Index = m_Reader.ReadInt16(); m_Reader.ReadByte(); item.Name = ReadString(); m_Reader.ReadByte(); item.Kind = m_Reader.ReadInt16(); grammar.SymbolTable[item.Index] = item; read_byte = m_Reader.ReadByte(); rbch = (char)read_byte; } else if (read_byte == (byte)'D') { /// /// Определение таблицы состояний КА /// DFA item = new DFA(); m_Reader.ReadByte(); item.Index = m_Reader.ReadInt16(); m_Reader.ReadByte(); item.IsAccepting = !(m_Reader.ReadByte() == 0); m_Reader.ReadByte(); item.AcceptIndex = m_Reader.ReadInt16(); m_Reader.ReadByte(); // читаем набор ребер item.Edges = new Edge[(entries - 5) / 3]; for (int x = 0; x < item.Edges.Length; x++) { Edge edge = new Edge(); m_Reader.ReadByte(); edge.CharacterSetIndex = m_Reader.ReadInt16(); m_Reader.ReadByte(); edge.TargetIndex = m_Reader.ReadInt16(); m_Reader.ReadByte(); item.Edges[x] = edge; } grammar.DFATable[item.Index] = item; read_byte = m_Reader.ReadByte(); rbch = (char)read_byte; } else if (read_byte == (byte)'L') { /// /// Определение состояний LALR анализатора /// LALR item = new LALR(); m_Reader.ReadByte(); item.Index = m_Reader.ReadInt16(); m_Reader.ReadByte(); // Читаем действия item.Actions = new Action[(entries - 3) / 4]; for (int x = 0; x < item.Actions.Length; x++) { Action action = new Action(); m_Reader.ReadByte(); action.SymbolIndex = m_Reader.ReadInt16(); m_Reader.ReadByte(); action.ActionType = m_Reader.ReadInt16(); m_Reader.ReadByte(); action.Target = m_Reader.ReadInt16(); m_Reader.ReadByte(); item.Actions[x] = action; } grammar.LALRTable[item.Index] = item; read_byte = NextByte(); } else if (read_byte == (byte)'I') { /// /// Определение начальных состояний /// m_Reader.ReadByte(); grammar.InitialDFAState = m_Reader.ReadInt16(); m_Reader.ReadByte(); grammar.InitialLALRState = m_Reader.ReadInt16(); read_byte = NextByte(); } } } catch(Exception ) { return null; } #endregion return grammar; }
public AnalysisTimeoutException(DFA abortedDFA) { _abortedDFA = abortedDFA; }
public void ReportContextSensitivity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, int prediction, SimulatorState acceptState) { throw new Exception("Context sensitivity error "); }
protected void Bt_excute_Click(object sender, EventArgs e) { System.IO.DirectoryInfo DirInfo = new System.IO.DirectoryInfo(Server.MapPath("../SingleFile/")); if (!DirInfo.Exists) { return; } System.IO.DirectoryInfo[] Dirs = DirInfo.GetDirectories(); //获取对应语言文件路径集合 FileInfo[] Paths = DirInfo.GetFiles("*." + Request.Form["lang"].ToString()); List <string> files = new List <string>(); //循环Paths 将每个文件信息放入List里 foreach (FileInfo filepath in Paths) { files.Add(filepath.FullName); } if (Paths.Count() <= 1) { Response.Write("<script>alert('对应类型的文件数少于两个')</script>"); } else { TOKEN GenerateToken = new TOKEN(); Sim CalculateSimScore = new Sim(); DFA CalculateDFAScore = new DFA(); List <VN> MarkFile1 = new List <VN>(); List <VN> MarkFile2 = new List <VN>(); Winnowing Win = new Winnowing(); WinText GenerateWinText = new WinText(); CalculateDFAScore.Get_VN(files[0], MarkFile1); CalculateDFAScore.Get_VN(files[1], MarkFile2); //System.Diagnostics.Debug.WriteLine(Markfile1.Count().ToString()); string WinText1 = GenerateWinText.FileFilter(files[0]); string WinText2 = GenerateWinText.FileFilter(files[1]); double WinScore = Win.TextSimilarity(WinText1, WinText2); double SimScore = CalculateSimScore.Sim_Run(GenerateToken.Read_file(files[0]), GenerateToken.Read_file(files[1])); double DFAScore = CalculateDFAScore.GetVarSim(MarkFile1, MarkFile2); double TotalScore = DFAScore * 0.3 + SimScore * 0.5 + WinScore * 0.2; string FileName1 = files[0].Substring(files[0].LastIndexOf('\\') + 1); string FileName2 = files[1].Substring(files[1].LastIndexOf('\\') + 1); Excute_info.Text = FileName1 + " " + FileName2 + " 代码相似度:" + TotalScore.ToString() + "%"; Label3.Text = FileName1; Label2.Text = FileName2; Literal1.Text = "<textarea name=\"code\" class=\"" + GetHighlightLang(Request.Form["lang"].ToString()) + "\" rows=\"15\" cols=\"100\"> " + CommonFunction.GetFileContent(files[0]) + "</textarea>"; Literal2.Text = "<textarea name=\"code\" class=\"" + GetHighlightLang(Request.Form["lang"].ToString()) + "\" rows=\"15\" cols=\"100\"> " + CommonFunction.GetFileContent(files[1]) + "</textarea>"; Upload_info.Text = ""; for (int i = 0; i < files.Count; ++i) { if (System.IO.File.Exists(files[i])) { System.IO.File.Delete(files[i]); } } } }
/// <summary> /// Gets the total number of DFA states stored in the DFA cache for a /// particular decision. /// </summary> public virtual int GetDFASize(int decision) { DFA decisionToDFA = atnSimulator.atn.decisionToDFA[decision]; return(decisionToDFA.states.Count); }
public void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { }
protected internal override Tuple <DFAState, ParserRuleContext> ComputeTargetState(DFA dfa, DFAState s, ParserRuleContext remainingGlobalContext, int t, bool useContext, PredictionContextCache contextCache) { Tuple <DFAState, ParserRuleContext> targetState = base.ComputeTargetState(dfa, s, remainingGlobalContext, t, useContext, contextCache); if (useContext) { decisions[currentDecision].LL_ATNTransitions++; } else { decisions[currentDecision].SLL_ATNTransitions++; } return(targetState); }
/// <summary> /// Creates a new tokenizer. /// </summary> /// <param name="dfa">A Deterministic Finite Automata</param> public StringTokenizer(DFA dfa) { this.dfa = dfa; }
public void ReportAmbiguity(Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { throw new Exception("Ambiguity error "); }
public void ReportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet conflictingAlts, SimulatorState conflictState) { throw new Exception("Full context error "); }
public TransitionTable(DFA dfa, IList <NFA> nfas, IList <Tuple <string, Func <string, T> > > tokens) { // Get a list of all valid input ranges that are distinct. // This will fill up the entire spectrum from 0 to max char // Sort these ranges so that they start with the lowest to highest start var allValidRanges = nfas.Select( f => f.Transitions.Aggregate(Enumerable.Empty <CharRange>(), (acc, a) => acc.Union(a.ValidInput.Ranges))) .Aggregate((acc, a) => acc.Union(a)) .OrderBy(f => f.From) .ToList(); // This list might not be properly terminated at both ends. This happens if there // never is anything that accepts any character. char start = allValidRanges.First().From; if (start != '\0') { // Add a range that goes from \0 to the character before start allValidRanges.Insert(0, new CharRange { From = '\0', To = (char)(start - 1) }); } char end = allValidRanges.Last().To; if (end != char.MaxValue) { allValidRanges.Add(new CharRange { From = (char)(end + 1), To = char.MaxValue }); } // Create a 2D table // First dimension is the number of states found in the DFA // Second dimension is number of distinct character ranges var uncompressed = new short[dfa.States.Count(), allValidRanges.Count()]; // Fill table with -1 for (int i = 0; i < dfa.States.Count(); ++i) { for (int j = 0; j < allValidRanges.Count(); ++j) { uncompressed[i, j] = -1; } } // Save the ends of the input ranges into an array inputRangeEnds = allValidRanges.Select(f => f.To).ToArray(); actions = new Tuple <int, Func <string, T> > [dfa.States.Count]; foreach (var state in dfa.States) { // Store to avoid problems with modified closure DFA.State state1 = state; foreach (var transition in dfa.Transitions.Where(f => f.From == state1)) { // Set the table entry foreach (var range in transition.ValidInput.Ranges) { int ix = allValidRanges.BinarySearch(range); uncompressed[state.StateNumber, ix] = (short)transition.To.StateNumber; } } // If this is an accepting state, set the action function to be // the FIRST defined action function if multiple ones match if (state.NfaStates.Any(f => f.AcceptState)) { // Find the lowest ranking NFA which has the accepting state in it for (int tokenNumber = 0; tokenNumber < nfas.Count(); ++tokenNumber) { NFA nfa = nfas[tokenNumber]; if (nfa.States.Intersect(state.NfaStates.Where(f => f.AcceptState)).Any()) { // Match // This might be a token that we ignore. This is if the tokenNumber >= number of tokens // since the ignored tokens are AFTER the normal tokens. If this is so, set the action func to // int.MinValue, NULL to signal that the parsing should restart without reporting errors if (tokenNumber >= tokens.Count()) { actions[state.StateNumber] = new Tuple <int, Func <string, T> >(int.MinValue, null); } else { actions[state.StateNumber] = new Tuple <int, Func <string, T> >( tokenNumber, tokens[tokenNumber].Item2); } break; } } } } table = new CompressedTable(uncompressed); asciiIndices = new int[256]; for (int i = 0; i < asciiIndices.Length; ++i) { asciiIndices[i] = FindTableIndexFromRanges((char)i); } }
static DFA9() { DFA9_transitionS = new string[76] { "\u0002\v\u0002\uffff\u0001\v\u0012\uffff\u0001\v\u0001\u000f\u0001\f\u0001\u0014\u0001\uffff\u0001\u001d\u0001\u0011\u0001\r\u0001\u0013\u0001\u0012\u0001\u001c\u0001\u001a\u0001\u000e\u0001\u001b\u0001\u0001\u0001\n\u0001\b\t\t\u0001\uffff\u0001\u0015\u0001 \u0001\u001e\u0001\u001f\u0002\uffff\u0005\a\u0001\u0002\a\a\u0001\u0003\u0005\a\u0001\u0004\u0006\a\u0001\u0017\u0001\uffff\u0001\u0016\u0001\uffff\u0001\a\u0001\uffff\u0004\a\u0001\u0005\u0003\a\u0001\u0006\u0011\a\u0001\u0018\u0001\u0010\u0001\u0019", string.Empty, "\u0001!", "\u0001\"", "\u0001#", "\u0001$", "\u0001%\a\uffff\u0001&", string.Empty, "\u0001'", string.Empty, "\u0001(\u0004\uffff\u0001(", string.Empty, string.Empty, string.Empty, string.Empty, "\u0001*", "\u0001,", "\u0001.", string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, "\u00010", "\u00012\u00013", "\u00016\u00015", "\u00018", "\u00019", "\u0001:", "\u0001;", "\n\a\a\uffff\u001a\a\u0004\uffff\u0001\a\u0001\uffff\u001a\a", "\u0001=", string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, string.Empty, "\u0001>", "\u0001?", "\u0001@", "\u0001A", string.Empty, "\u0001B", "\u0001C", "\n\a\a\uffff\u001a\a\u0004\uffff\u0001\a\u0001\uffff\u001a\a", "\n\a\a\uffff\u001a\a\u0004\uffff\u0001\a\u0001\uffff\u001a\a", "\n\a\a\uffff\u001a\a\u0004\uffff\u0001\a\u0001\uffff\u001a\a", "\u0001G", "\n\a\a\uffff\u001a\a\u0004\uffff\u0001\a\u0001\uffff\u001a\a", string.Empty, string.Empty, string.Empty, "\u0001I", string.Empty, "\u0001J", "\n\a\a\uffff\u001a\a\u0004\uffff\u0001\a\u0001\uffff\u001a\a", string.Empty }; DFA9_eot = DFA.UnpackEncodedString("\u0002\uffff\u0005\a\u0001\uffff\u0001\t\u0001\uffff\u0001)\u0004\uffff\u0001+\u0001-\u0001/\f\uffff\u00011\u00014\u00017\u0004\a\u0001<\u0001\a\u0011\uffff\u0004\a\u0001\uffff\u0002\a\u0001D\u0001E\u0001F\u0001\a\u0001H\u0003\uffff\u0001\a\u0001\uffff\u0001\a\u0001K\u0001\uffff"); DFA9_eof = DFA.UnpackEncodedString("L\uffff"); DFA9_min = DFA.UnpackEncodedStringToUnsignedChars("\u0001\t\u0001\uffff\u0001A\u0001U\u0001R\u0001l\u0001f\u0001\uffff\u0001x\u0001\uffff\u0001*\u0004\uffff\u0001=\u0001|\u0001&\f\uffff\u0002=\u0001<\u0002L\u0001U\u0001s\u00010\u0001c\u0011\uffff\u0001S\u0001L\u0001E\u0001e\u0001\uffff\u0001l\u0001E\u00030\u0001u\u00010\u0003\uffff\u0001d\u0001\uffff\u0001e\u00010\u0001\uffff"); DFA9_max = DFA.UnpackEncodedStringToUnsignedChars("\u0001}\u0001\uffff\u0001A\u0001U\u0001R\u0001l\u0001n\u0001\uffff\u0001x\u0001\uffff\u0001/\u0004\uffff\u0001=\u0001|\u0001&\f\uffff\u0001=\u0001>\u0001=\u0002L\u0001U\u0001s\u0001z\u0001c\u0011\uffff\u0001S\u0001L\u0001E\u0001e\u0001\uffff\u0001l\u0001E\u0003z\u0001u\u0001z\u0003\uffff\u0001d\u0001\uffff\u0001e\u0001z\u0001\uffff"); DFA9_accept = DFA.UnpackEncodedString("\u0001\uffff\u0001\u0001\u0005\uffff\u0001\b\u0001\uffff\u0001\t\u0001\uffff\u0001\f\u0001\r\u0001\u000e\u0001\u000f\u0003\uffff\u0001\u0015\u0001\u0016\u0001\u0017\u0001\u0018\u0001\u0019\u0001\u001a\u0001\u001b\u0001\u001c\u0001\u001d\u0001\u001e\u0001\u001f\u0001!\t\uffff\u0001\n\u0001\v\u0001 \u0001$\u0001\u0010\u0001\u0013\u0001\u0011\u0001\u0014\u0001\u0012\u0001\"\u0001#\u0001(\u0001*\u0001%\u0001'\u0001)\u0001&\u0004\uffff\u0001\u0006\a\uffff\u0001\u0003\u0001\u0004\u0001\u0005\u0001\uffff\u0001\u0002\u0002\uffff\u0001\a"); DFA9_special = DFA.UnpackEncodedString("L\uffff}>"); int num = DFA9_transitionS.Length; DFA9_transition = new short[num][]; for (int i = 0; i < num; i++) { DFA9_transition[i] = DFA.UnpackEncodedString(DFA9_transitionS[i]); } }
/** * Test to see if the NFA was successfully converted into a valid DFA */ public static Boolean NFAClosureTest(NFA nfa, DFA dfa) { // 1 Output to 1 Input Test var index = nfa.nextStates.Keys.ToArray(); string[] input; int nextStateCount; int num = 0; int numErrors = 0; var node = dfa.nodes.ToArray(); // For each transition on every node in the DFA, confirm that there is a 1-to-1 Input/Output for each // transition for (int i = 0; i < node.Length; i++) { input = dfa.nextStates[node[i]].Keys.ToArray(); for (int j = 0; j < input.Length; j++) { nextStateCount = dfa.nextStates[node[i]][input[j]].Count(); if (nextStateCount > 1) { Console.WriteLine("1-to-1 Input/Output Test Failed: " + "\n" + "\t" + "Node Index: " + node[i] + " has " + nextStateCount + " transitions on input" + "\n" + "\t" + input[j]); return(false); } } } Console.WriteLine("1-to-1 Input/Output Test Passed"); // Proper States Test // Ensure that all the states in the DFA are comprised of states that existed in the NFA int[] stateArray; int maxIndex = ((nfa.GetTerm().Length + 1) * 10) + nfa.GetAllowableErrors(); int minIndex = 10; for (int i = 0; i < node.Length; i++) { stateArray = dfa.states[node[i]].ToArray(); for (int j = 0; j < stateArray.Length; j++) { if (stateArray[j] > maxIndex || stateArray[j] < minIndex) { Console.WriteLine("Proper States Test Failed: Index:" + node[i]); return(false); } } } Console.WriteLine("Proper States Test Passed"); // Any Transition Test // For the resulting DFA of a NFA under closure, confirm that all the information of the "any" // next states is incorporated in the new node /*HashSet<string> inputs = new HashSet<string>(); * int[] stateArray; * Dictionary<string, HashSet<int>> nextStates = new Dictionary<string, HashSet<int>>(); * * HashSet<string> cInputs; * HashSet<int> cStates; * Dictionary<int, Dictionary<string, HashSet<int>>> cNextStates = nfa.nextStates; * * var nodes = nfa.getNodes().ToArray(); * var cNodes = dfa.nodes.ToArray(); * * for (int i = 0; i < cNodes.Length; i++) * { * stateArray = nfa.states[cNodes[i]].ToArray(); * * // For each node in this closed index, * for (int j = 0; j < stateArray.Length; j++) * { * inputs.UnionWith(nfa.inputs[stateArray[j]]); * nfa.mergeNextStates(nextStates, nfa.nextStates[stateArray[j]]); * } * * // Now that the next states are filled, can test to see if they are identical to the closed node's * * // First check that the inputs are the same * if (inputs == nfa.inputs[cNodes[i]]) * { * // For each input, confirm that their next states are identical * input = inputs.ToArray(); * * for (int j = 0; j < input.Length; j++) * { * var indice = nfa.getCombinedIndex(nextStates[input[j]].ToArray()); * * if (indice != nfa.getCombinedIndex(cNextStates[cNodes[i]][input[j]].ToArray())) * { * // Test fails * return false; * } * } * } * else * { * // Test Fails * return false; * } * }*/ return(true); }
private List <CodeCheckResult> ExcuteCodeCheck(List <string> Files, double MinRange, ref double MaxScore, ref int MaxIndex1, ref int MaxIndex2) { List <CodeCheckResult> OutputExcel = new List <CodeCheckResult>(); TOKEN GenerateToken = new TOKEN(); Sim CalculateSimScore = new Sim(); DFA CalculateDFAScore = new DFA(); Winnowing CalculateWinScore = new Winnowing(); WinText GenerateWinText = new WinText(); BeginProgress(); int total = ((Files.Count() - 1) * Files.Count()) / 2; int Counter = 0, Bar = 0; List <string> WinFiles = new List <string>(); List <List <VN> > VNFiles = new List <List <VN> >(); List <List <string> > SimFiles = new List <List <string> >(); for (int i = 0; i < Files.Count(); ++i) { List <VN> MF = new List <VN>(); CalculateDFAScore.Get_VN(Files[i], MF); VNFiles.Add(MF); List <string> SF = GenerateToken.Read_file(Files[i]); SimFiles.Add(SF); WinFiles.Add(GenerateWinText.FileFilter(Files[i])); } for (int i = 0; i < Files.Count(); ++i) { for (int j = i + 1; j < Files.Count(); ++j) { double SimScore = CalculateSimScore.Sim_Run(SimFiles[i], SimFiles[j]); double DFAScore = CalculateDFAScore.GetVarSim(VNFiles[i], VNFiles[j]); double WinScore = CalculateWinScore.TextSimilarity(WinFiles[i], WinFiles[j]); double TotalScore = DFAScore * 0.3 + SimScore * 0.5 + WinScore * 0.2; //double TotalScore = SimScore * 1.0; if (TotalScore > MaxScore) { MaxIndex1 = i; MaxIndex2 = j; MaxScore = TotalScore; } // System.Diagnostics.Debug.WriteLine(TotalScore.ToString()); Bar = (Counter) * 100 / total; Counter++; SetProgress(Bar); System.Threading.Thread.Sleep(10); //System.Diagnostics.Debug.WriteLine(Bar.ToString()); if (TotalScore < MinRange) { continue; } OutputExcel.Add( new CodeCheckResult(Files[i].Substring(Files[i].LastIndexOf('\\') + 1) , Files[j].Substring(Files[j].LastIndexOf('\\') + 1) , TotalScore.ToString() + "%")); } } FinishProgress(); return(OutputExcel); }
/** Performs closure on all the nodes starting from the NFA's starting node and assigns the * new nodes and transitions to the DFA */ public void PerformClosure(DFA dfa) { // Save copies of the NFA's values as they will be modified for the closure var savedInputs = inputs; var savedStates = states; var savedNextStates = nextStates; var nodeSet = new HashSet <int> { startState }; // Declare the first node as the start state under closure int frontier = closure(startState, states[startState]); // List of nodes by their indexes that signifies that the node has already been added var seen = dfa.nodes; // Add the first node and set it to the start stae //seen.Add(frontier); dfa.setStartState(frontier); // Placeholders for output methods HashSet <int> nextStateSet; int[] nextStatesArray; string[] inputArray; // List of node indexes obtained by adding the next states of the current frontier HashSet <int> nodeStack = new HashSet <int> { frontier }; // Loop while frontier is not null. At the end of loop, set the frontier to next node on the stack while (frontier != null) { nodeStack.Remove(frontier); // Remove the node from the stack seen.Add(frontier); // Add the node to the DFA's node // Add the information (inputs, state, next states) for this node dfa.inputs.Add(frontier, inputs[frontier]); dfa.states.Add(frontier, states[frontier]); dfa.nextStates.Add(frontier, nextStates[frontier]); inputArray = inputs[frontier].ToArray(); // Array of inputs for this node // Loop through the inputs of this node and get all new/unseen nodes that are reachable for (int i = 0; i < inputArray.Length; i++) { nextStatesArray = nextStates[frontier][inputArray[i]].ToArray(); var except = nextStatesArray.Except(seen); // If there are new nodes, add them to the stack if (except != null) { nodeStack.UnionWith(except); } } frontier = nodeStack.FirstOrDefault(); // Set the frontier to the next node on the stack // If there are no nodes on the stack, exit the loop if (frontier == 0) { break; } } }
public IlOutput(DFA dfa) { this.dfa = dfa; }
private int SpecialStateTransition9(DFA dfa, int s, IIntStream _input) { IIntStream input = _input; int _s = s; switch (s) { case 0: int LA9_0 = input.LA(1); s = -1; if ((LA9_0 == 'a')) { s = 1; } else if ((LA9_0 == 'b')) { s = 2; } else if ((LA9_0 == 'd')) { s = 3; } else if ((LA9_0 == 'e')) { s = 4; } else if ((LA9_0 == 'f')) { s = 5; } else if ((LA9_0 == 'i')) { s = 6; } else if ((LA9_0 == 'l')) { s = 7; } else if ((LA9_0 == 'n')) { s = 8; } else if ((LA9_0 == 'o')) { s = 9; } else if ((LA9_0 == 't')) { s = 10; } else if ((LA9_0 == 'v')) { s = 11; } else if ((LA9_0 == 'w')) { s = 12; } else if ((LA9_0 == 's')) { s = 13; } else if (((LA9_0 >= 'A' && LA9_0 <= 'Z') || LA9_0 == 'c' || (LA9_0 >= 'g' && LA9_0 <= 'h') || (LA9_0 >= 'j' && LA9_0 <= 'k') || LA9_0 == 'm' || (LA9_0 >= 'p' && LA9_0 <= 'r') || LA9_0 == 'u' || (LA9_0 >= 'x' && LA9_0 <= 'z'))) { s = 14; } else if (((LA9_0 >= '0' && LA9_0 <= '9'))) { s = 15; } else if ((LA9_0 == '\"')) { s = 16; } else if (((LA9_0 >= '\t' && LA9_0 <= '\n') || LA9_0 == '\r' || LA9_0 == ' ')) { s = 17; } else if ((LA9_0 == '/')) { s = 18; } else if (((LA9_0 >= '\u0000' && LA9_0 <= '\b') || (LA9_0 >= '\u000B' && LA9_0 <= '\f') || (LA9_0 >= '\u000E' && LA9_0 <= '\u001F') || LA9_0 == '!' || (LA9_0 >= '#' && LA9_0 <= '.') || (LA9_0 >= ':' && LA9_0 <= '@') || (LA9_0 >= '[' && LA9_0 <= '`') || (LA9_0 >= '{' && LA9_0 <= '\uFFFF'))) { s = 19; } if (s >= 0) { return(s); } break; case 1: int LA9_16 = input.LA(1); s = -1; if (((LA9_16 >= '\u0000' && LA9_16 <= '\uFFFF'))) { s = 40; } else { s = 19; } if (s >= 0) { return(s); } break; } NoViableAltException nvae = new NoViableAltException(dfa.Description, 9, _s, input); dfa.Error(nvae); throw nvae; }
private static ParsingTable GetParsingTable(DFA theAutomaton, HashSet<input> theTokens, HashSet<input> NonTerminals, HashSet<Tuple<dfaState, Symbol, Int32>> ReduceStates, List<GrammarRule> theGrammar) { ParsingTable Table = new ParsingTable(); foreach (KeyValuePair<KeyValuePair<dfaState, input>, dfaState> item in theAutomaton.transitionTable) { if (NonTerminals.Contains(item.Key.Value)) { Table.transitionTable.Add(item.Key, new Action(item.Value, ActionType.Goto)); } else { Table.transitionTable.Add(item.Key, new Action(item.Value, ActionType.Shift)); } } foreach (dfaState theState in theAutomaton.final) { Table.transitionTable.Add( new KeyValuePair<dfaState, input>(theState, "úEndSymbol"), new Action(-1, ActionType.Accept)); } foreach (Tuple<dfaState, Symbol, Int32> item in ReduceStates) { if (!NonTerminals.Contains(item.Item2.id)) Table.transitionTable.Add( new KeyValuePair<dfaState, input>(item.Item1, item.Item2.id), new Action(item.Item3, ActionType.Reduce)); } return Table; }
public DFAPrinter(DFA graph) { dfa = graph; }
public override void ReportAmbiguity(Antlr4.Runtime.Parser recognizer, DFA dfa, int startIndex, int stopIndex, bool exact, BitSet ambigAlts, ATNConfigSet configs) { }
public MachineProbe(DFA dfa) { this.dfa = dfa; }
public void Init(string root) { _Root = root; _DFA = new DFEngine(); _DFA.Load(Path.Combine(root, "amon.sync")); }
public override void ReportAttemptingFullContext(Parser recognizer, DFA dfa, int startIndex, int stopIndex, BitSet conflictingAlts, SimulatorState conflictState) { }