public static Token MatchToken(string token_name, string haystack, int recurse_depth, Token as_prefix) // If you say you understand this you're a lier. { TokenHolder token = Tokens[token_name]; if (token.forceWhitespaceToggle) { whitespace_stack.Push(token.isForcedWhitespace); } else { whitespace_stack.Push(whitespace_stack.Peek()); } recurse_depth++; if (recurse_depth > 300) { throw new Exception("Stack overflow. Program is too deep, man!"); } foreach (List <TokenMatch> option in token.options) { string option_str = haystack; List <TokenItem> match = new List <TokenItem>(); bool needtousepref = as_prefix != null; bool success = true; foreach (TokenMatch to_match in option) { int max_matches = to_match.count[1]; List <Token> this_match = new List <Token>(); while (max_matches != 0) { if (whitespace_stack.Peek()) { option_str = Regex.Replace(option_str, "^(\\$\\*([^*]|\\*[^$])*\\*\\$|\\$[^*\\r\\n].*|\\s)*", ""); } if (to_match.isPrefix) { if (as_prefix != null) { this_match.Add(as_prefix); needtousepref = false; } } else if (to_match.type == TokenType.TOKEN) { Token subMatch = MatchToken(to_match.data, option_str, recurse_depth, null); if (subMatch != null) { option_str = subMatch.remainder; this_match.Add(subMatch); } else { break; } } else if (to_match.type == TokenType.REGEX) { Match str_match = to_match.regex.Match(option_str); if (str_match.Success) { option_str = to_match.regex.Replace(option_str, ""); Token string_token = new Token(); string_token.text = string_token.name = str_match.Groups[0].Value; string_token.remainder = option_str; string_token.isOnlyString = true; this_match.Add(string_token); } else { break; } } max_matches--; } if (this_match.Count < to_match.count[0] || (to_match.count[1] != -1 && this_match.Count > to_match.count[1])) { success = false; break; } TokenItem token_item = new TokenItem(); token_item.name = to_match.data; token_item.items = this_match; match.Add(token_item); } if (needtousepref) { success = false; } if (success) { Token this_tokn = new Token(); this_tokn.data = match; this_tokn.name = token_name; this_tokn.remainder = option_str; this_tokn.text = haystack.Substring(0, haystack.Length - option_str.Length); if (prefixes.ContainsKey(token_name)) { Dictionary <string, TokenHolder> prfxs = prefixes[token_name]; while (true) { Token shortest = this_tokn; int shortlen = option_str.Length; foreach (KeyValuePair <string, TokenHolder> kv in prfxs) { Token contestant = MatchToken(kv.Key, option_str, recurse_depth, this_tokn); if (contestant != null && contestant.remainder.Length < shortlen) { shortlen = contestant.remainder.Length; contestant.text = haystack.Substring(0, haystack.Length - contestant.remainder.Length); shortest = new Token(); shortest.name = token_name; shortest.remainder = contestant.remainder; shortest.text = contestant.text; shortest.data = new List <TokenItem>(); TokenItem sub_token_item = new TokenItem(); sub_token_item.name = kv.Key; sub_token_item.items = new List <Token>(); sub_token_item.items.Add(contestant); shortest.data.Add(sub_token_item); } } option_str = shortest.remainder; if (shortest == this_tokn) { whitespace_stack.Pop(); return(shortest); } this_tokn = shortest; } } whitespace_stack.Pop(); return(this_tokn); } } whitespace_stack.Pop(); return(null); }
public static void CompileTokens() { whitespace_stack = new Stack <bool>(); // Shutup, it's a relyable place whitespace_stack.Push(true); if (!RawTokens.ContainsKey("program")) { throw new Exception("Tokenizers must define 'program'"); } foreach (KeyValuePair <string, string> token_data in RawTokens) { string tkn = token_data.Value; var token_holder = new TokenHolder(); var options = token_holder.options = new List <List <TokenMatch> >(); var option = new List <TokenMatch>(); if (tkn[0] == '@') { token_holder.forceWhitespaceToggle = true; token_holder.isForcedWhitespace = tkn[1] == '1'; tkn = tkn.Substring(2); } while (tkn.Length > 0) { tkn = Regex.Replace(tkn, "^\\s+", ""); Match sub_token = TokenPatterns.MATCH_UPTO.Match(tkn); String spec = sub_token.Groups[6].Value; String sub_token_text = sub_token.Groups[1].Value; tkn = TokenPatterns.MATCH_UPTO.Replace(tkn, ""); if (sub_token_text.Length > 0) { TokenMatch compiled = new TokenMatch(sub_token_text); if (compiled.isPrefix) { if (!prefixes.ContainsKey(compiled.data)) { prefixes[compiled.data] = new Dictionary <string, TokenHolder>(); } prefixes[compiled.data][token_data.Key] = token_holder; } option.Add(compiled); } if (spec.Length > 0 && spec[0] == '|') { if (option.Count > 0) { options.Add(option); } option = new List <TokenMatch>(); } } if (option.Count > 0) { options.Add(option); } Tokens[token_data.Key] = token_holder; } }