示例#1
0
文件: Parser.cs 项目: emtees/old-code
		public InstructionList Parse (TokenList tokens) {
			TokenList partial = ParseForwards (tokens);
			partial = ParseInfix (partial);
			InstructionList tree = ParseBackwards (partial, false);

			if (to_parse != null) {
				// Null out the member before going re-entrant.
				// Otherwise we could go into an infinite loop.
				ArrayList parse_now = to_parse;
				to_parse = null;

				foreach (Function f in parse_now) {
					f.Parse (this);
				}
			}

			return tree;
		}
示例#2
0
        private void ParseForwards(TokenList partial, IEnumerator iterator, int parens)
        {
            while (iterator.MoveNext())
            {
                Token token = (Token)iterator.Current;
                switch (token.Type)
                {
                case TokenType.OpenBracket:
                    partial.Add(new Token(TokenType.PlaceholderElement, ParseList(iterator)));
                    break;

                case TokenType.OpenParens:
                    TokenList inner = new TokenList();
                    ParseForwards(inner, iterator, parens + 1);
                    partial.Add(new Token(TokenType.PlaceholderGroup, inner));
                    break;

                case TokenType.CloseParens:
                    return;

                case TokenType.Word:
                    if (String.Compare((string)token.Val, "to", true) == 0)
                    {
                        // FIXME: Usual case is in void context
                        // token = ParseTo (iterator);
                        ParseTo(iterator);
                    }
                    else
                    {
                        partial.Add(token);
                    }
                    break;

                default:
                    partial.Add(token);
                    break;
                }
            }
        }
示例#3
0
        public InstructionList Parse(TokenList tokens)
        {
            TokenList partial = ParseForwards(tokens);

            partial = ParseInfix(partial);
            InstructionList tree = ParseBackwards(partial, false);

            if (to_parse != null)
            {
                // Null out the member before going re-entrant.
                // Otherwise we could go into an infinite loop.
                ArrayList parse_now = to_parse;
                to_parse = null;

                foreach (Function f in parse_now)
                {
                    f.Parse(this);
                }
            }

            return(tree);
        }
示例#4
0
文件: Parser.cs 项目: emtees/old-code
		private void ExtendList (TokenList list, Stack stack) {
			Array tokens = stack.ToArray ();
			Array.Reverse (tokens);
			list.Extend (tokens);
		}
示例#5
0
文件: Parser.cs 项目: emtees/old-code
		private void ParseForwards (TokenList partial, IEnumerator iterator, int parens) {
			while (iterator.MoveNext ()) {
				Token token = (Token) iterator.Current;
				switch (token.Type) {
				case TokenType.OpenBracket:
					partial.Add (new Token (TokenType.PlaceholderElement, ParseList (iterator)));
					break;
				case TokenType.OpenParens:
					TokenList inner = new TokenList ();
					ParseForwards (inner, iterator, parens + 1);
					partial.Add (new Token (TokenType.PlaceholderGroup, inner));
					break;
				case TokenType.CloseParens:
					return;
				case TokenType.Word:
					if (String.Compare ((string) token.Val, "to", true) == 0) {
						// FIXME: Usual case is in void context
						// token = ParseTo (iterator);
						ParseTo (iterator);
					} else {
						partial.Add (token);
					}
					break;
				default:
					partial.Add (token);
					break;
				}
			}
		}
示例#6
0
文件: Parser.cs 项目: emtees/old-code
		private Token ParseTo (IEnumerator iterator) {
			ParseToState state = ParseToState.FunctionName;
			Token name = new Token (TokenType.Word, null);
			TokenList args = new TokenList (); 
			TokenList def = new TokenList ();
			
			bool reading = true;
			bool prev_newline = false;
			while (reading && iterator.MoveNext ()) {
				Token token = (Token) iterator.Current;

				if (state != ParseToState.Content && token.Type == TokenType.Newline) {
					state = ParseToState.Content;
					prev_newline = true;
					continue;
				}
				
				switch (state) {
				case ParseToState.FunctionName:
					name = token;
					state = AdvanceState (state);
					break;
				case ParseToState.Argument:
				case ParseToState.ArgumentWithValue:
				case ParseToState.ArgumentCollector:
					if (token.Type == TokenType.OpenBracket) {
						TokenList arg = ParseBrackets (iterator);
						if (state == ParseToState.Argument || (state == ParseToState.ArgumentWithValue && arg.Count == 1)) {
							state = AdvanceState (state);
						}
						args.Add (new Token (TokenType.PlaceholderElement, new Element (ElementType.List, arg)));
					} else {
						args.Add (token);
					}
					break;
				case ParseToState.Content:
					if (prev_newline && token.Type == TokenType.Word && String.Compare ((string) token.Val, "end", true) == 0) {
						reading = false;
					} else {
						def.Add (token);
						prev_newline = (token.Type == TokenType.Newline); 
					}
					break;
				}
			}

			TokenList func = new TokenList ();
			func.Add (name);
			func.Add (new Token (TokenType.PlaceholderGroup, args));
			func.Add (new Token (TokenType.PlaceholderGroup, def));

			Function func_obj = CreateFunction (func);
			funcs.AddMessage (func_obj);

			if (to_parse == null)
				to_parse = new ArrayList ();
			to_parse.Add (func_obj);

			return new Token (TokenType.PlaceholderElement, new Element (ElementType.Function, func_obj));
		}
示例#7
0
文件: Parser.cs 项目: emtees/old-code
		private Function CreateFunction (TokenList func) {
			string name = (string) func[0].Val;
			
			ArrayList args_list = new ArrayList ();
			foreach (Token token in (TokenList) func[1].Val) {
				ArgumentInfo info = new ArgumentInfo ();

				if (token.Type == TokenType.Variable)
					info = new ArgumentInfo ((string) token.Val, null, false);
				else if (token.Type == TokenType.PlaceholderElement) {
					TokenList arg = (TokenList) ((Element) token.Val).Val;
					// FIXME: What if it is longer than 2 elements?
					if (arg.Count > 1)
						info = new ArgumentInfo ((string) arg[0].Val, arg[1].Val, false);
					else
						info = new ArgumentInfo ((string) arg[0].Val, null, true);
				}
				
				args_list.Add (info);
			}
			ArgumentInfo[] args = (ArgumentInfo[]) args_list.ToArray (typeof (ArgumentInfo)); 
			TokenList tokens = (TokenList) func[2].Val;

			return new Function (name, tokens, args);
		}
示例#8
0
		private StringBuilder AddTokens (TokenList tokens, StringBuilder builder, int c_prev_int, int c_peek_int) {
			StringBuilder ret;
			string val = builder.ToString ();
			
			if (val.Length > 0) {
				ret = new StringBuilder ();
			} else {
				ret = builder;
			}

			double as_num;
			if (IsNumber (val, out as_num)) {
				Token token = new Token (TokenType.Number, as_num);
				if (tokens[tokens.Count - 1].Type == TokenType.Minus) {
					token.Val = -((double) token.Val);
					tokens[tokens.Count - 1] = token;
				} else {
					tokens.Add (token);
				}
			} else if (val.Length > 1) {
				Token token;
				
				string var_name, str_val;
				if (IsVariable (val, out var_name)) 
					token = new Token (TokenType.Variable, var_name);
				else if (IsString (val, out str_val)) 
					token = new Token (TokenType.String, str_val);
				else
					token = new Token (TokenType.Word, val);

				tokens.Add (token);
			} else if (val.Length == 1) {
				tokens.Add (TokenForChar (c_prev_int, c_peek_int, val[0]));
			}

			return ret;
		}
示例#9
0
文件: Parser.cs 项目: emtees/old-code
		private TokenList ParseBrackets (IEnumerator iterator) {
			TokenList list = new TokenList ();

			while (iterator.MoveNext ()) {
				Token token = (Token) iterator.Current;
				if (token.Type == TokenType.CloseBracket) 
					break;
				else 	
					list.Add (token);
			}
			
			return list;
		}
示例#10
0
        private TokenList ParseInfix(TokenList partial)
        {
            TokenList output           = new TokenList();
            Stack     stack            = new Stack();
            bool      last_was_operand = false;

            int length = partial.Count;

            for (int i = length - 1; i >= 0; i--)
            {
                Token token = partial[i];
                if (token.Type == TokenType.Word)
                {
                    ExtendList(output, stack);
                    output.Add(token);
                    stack.Clear();
                }
                else if (token.Type == TokenType.Infix)
                {
                    if (stack.Count == 0)
                    {
                        stack.Push(token);
                    }
                    else
                    {
                        Token prev = (Token)stack.Pop();
                        if (Weight(prev) <= Weight(token))
                        {
                            stack.Push(token);
                            stack.Push(prev);
                        }
                        else
                        {
                            output.Add(prev);
                            stack.Push(token);
                        }
                    }
                    last_was_operand = false;
                }
                else
                {
                    if (token.Type == TokenType.PlaceholderGroup)
                    {
                        token.Val = ParseInfix((TokenList)token.Val);
                    }

                    if (i > 0)
                    {
                        Token peek = partial[i - 1];
                        if (peek.Type == TokenType.Minus)
                        {
                            output.Add(token);
                            output.Add(peek);
                            last_was_operand = true;
                            i--;
                            continue;
                        }
                    }

                    if (last_was_operand)
                    {
                        ExtendList(output, stack);
                        stack.Clear();
                    }
                    output.Add(token);
                    last_was_operand = true;
                }
            }

            ExtendList(output, stack);
            output.Reverse();
            return(output);
        }
示例#11
0
        private Token ParseTo(IEnumerator iterator)
        {
            ParseToState state = ParseToState.FunctionName;
            Token        name  = new Token(TokenType.Word, null);
            TokenList    args  = new TokenList();
            TokenList    def   = new TokenList();

            bool reading      = true;
            bool prev_newline = false;

            while (reading && iterator.MoveNext())
            {
                Token token = (Token)iterator.Current;

                if (state != ParseToState.Content && token.Type == TokenType.Newline)
                {
                    state        = ParseToState.Content;
                    prev_newline = true;
                    continue;
                }

                switch (state)
                {
                case ParseToState.FunctionName:
                    name  = token;
                    state = AdvanceState(state);
                    break;

                case ParseToState.Argument:
                case ParseToState.ArgumentWithValue:
                case ParseToState.ArgumentCollector:
                    if (token.Type == TokenType.OpenBracket)
                    {
                        TokenList arg = ParseBrackets(iterator);
                        if (state == ParseToState.Argument || (state == ParseToState.ArgumentWithValue && arg.Count == 1))
                        {
                            state = AdvanceState(state);
                        }
                        args.Add(new Token(TokenType.PlaceholderElement, new Element(ElementType.List, arg)));
                    }
                    else
                    {
                        args.Add(token);
                    }
                    break;

                case ParseToState.Content:
                    if (prev_newline && token.Type == TokenType.Word && String.Compare((string)token.Val, "end", true) == 0)
                    {
                        reading = false;
                    }
                    else
                    {
                        def.Add(token);
                        prev_newline = (token.Type == TokenType.Newline);
                    }
                    break;
                }
            }

            TokenList func = new TokenList();

            func.Add(name);
            func.Add(new Token(TokenType.PlaceholderGroup, args));
            func.Add(new Token(TokenType.PlaceholderGroup, def));

            Function func_obj = CreateFunction(func);

            funcs.AddMessage(func_obj);

            if (to_parse == null)
            {
                to_parse = new ArrayList();
            }
            to_parse.Add(func_obj);

            return(new Token(TokenType.PlaceholderElement, new Element(ElementType.Function, func_obj)));
        }
示例#12
0
        private void PrintFunction(TokenList func)
        {
            Console.Write("function {0}", (string)func[0].Val);
            foreach (Token token in (TokenList)func[1].Val)
            {
                if (token.Type == TokenType.Variable)
                {
                    Console.Write(" :{0}", token.Val);
                }
                else if (token.Type == TokenType.PlaceholderElement)
                {
                    TokenList arg = (TokenList)((Element)token.Val).Val;
                    Console.Write(" [");
                    int j = 0;
                    foreach (Token t in arg)
                    {
                        if (j == 0)
                        {
                            Console.Write(":");
                        }
                        Console.Write(t.Val);
                        if (j < (arg.Count - 1))
                        {
                            Console.Write(" ");
                        }
                        j++;
                    }
                    Console.Write("]");
                }
            }
            Console.WriteLine();

            bool print_space = false;
            bool print_tab   = true;

            foreach (Token token in (TokenList)func[2].Val)
            {
                if (print_space && token.Type != TokenType.Newline)
                {
                    Console.Write(" ");
                }
                else
                {
                    print_space = true;
                }

                if (print_tab)
                {
                    Console.Write("\t");
                    print_tab = false;
                }

                if (token.Type == TokenType.Newline)
                {
                    print_tab   = true;
                    print_space = false;
                }

                Console.Write("{0}", token.Val);
            }

            Console.WriteLine();
        }
示例#13
0
		public TokenList Parse (TextReader reader) {
			TokenList tokens = new TokenList ();
			
			StringBuilder builder = new StringBuilder ();
			int c_int, c_prev_int = -1;
			bool inside_bars = false;
			bool ignore_bar = false;
			bool inside_comment = false; 
			for (c_int = reader.Read (); c_int != -1; c_prev_int = c_int, c_int = reader.Read ()) {
				char c = (char) c_int;

				if (inside_comment) {
					if (IsNewline (c)) {
						inside_comment = false;
					} else {
						continue;
					}
				}
				
				if (c == '|') {
					if (ignore_bar) {
						ignore_bar = false;
						continue;
					}
					
					if (inside_bars && reader.Peek () == (int) '|') {
						ignore_bar = true;
						builder.Append ('|');
						continue;
					}
					
					inside_bars = !inside_bars;
					continue;
				}

				if (inside_bars) {
					builder.Append (c);
					continue;
				}

				if (c == '_' && tokens.Count > 0 && tokens[tokens.Count - 1].Type == TokenType.Newline) {
					tokens.RemoveAt (tokens.Count - 1);
					continue;
				} else if (c == ';') {
					inside_comment = true;
					continue;
				} else if (!IsDelimiter (c) || c_prev_int == (int) '"') {
					builder.Append (c);
					continue;
				}

				builder = AddTokens (tokens, builder, c_prev_int, reader.Peek (), c);
			}
			
			builder = AddTokens (tokens, builder, c_prev_int, reader.Peek ()); 

			return tokens;
		}
示例#14
0
		private StringBuilder AddTokens (TokenList tokens, StringBuilder builder, int c_prev_int, int c_peek_int, char c) {
			StringBuilder ret = AddTokens (tokens, builder, c_prev_int, c_peek_int);

			if (!(IsWhitespace (c) || IsSpecial (c)))
				tokens.Add (TokenForChar (c_prev_int, c_peek_int, c));

			return ret;
		}
示例#15
0
文件: Parser.cs 项目: emtees/old-code
		private TokenList ParseInfix (TokenList partial) {
			TokenList output = new TokenList ();
			Stack stack = new Stack ();
			bool last_was_operand = false;

			int length = partial.Count;
			for (int i = length - 1; i >= 0; i--) {
				Token token = partial[i];
				if (token.Type == TokenType.Word) {
					ExtendList (output, stack);
					output.Add (token);
					stack.Clear ();
				} else if (token.Type == TokenType.Infix) {
					if (stack.Count == 0) {
						stack.Push (token);
					} else {
						Token prev = (Token) stack.Pop ();
						if (Weight (prev) <= Weight (token)) {
							stack.Push (token);
							stack.Push (prev);
						} else {
							output.Add (prev);
							stack.Push (token);
						}
					}
					last_was_operand = false;
				} else {
					if (token.Type == TokenType.PlaceholderGroup) {
						token.Val = ParseInfix ((TokenList) token.Val);
					}

					if (i > 0) {
						Token peek = partial[i - 1];
						if (peek.Type == TokenType.Minus) {
							output.Add (token);
							output.Add (peek);
							last_was_operand = true;
							i--;
							continue;
						}
					}
					
					if (last_was_operand) {
						ExtendList (output, stack);
						stack.Clear ();
					}
					output.Add (token);
					last_was_operand = true;
				}
			}

			ExtendList (output, stack);
			output.Reverse ();
			return output;
		}
示例#16
0
        private InstructionList ParseBackwards(TokenList tokens, bool grouped)
        {
            Stack stack = new Stack();

            for (int i = tokens.Count - 1; i >= 0; i--)
            {
                Token token = tokens[i];

                switch (token.Type)
                {
                case TokenType.Number:
                case TokenType.String:
                    stack.Push(new Element(ElementType.Literal, token.Val));
                    break;

                case TokenType.PlaceholderElement:
                    stack.Push(token.Val);
                    break;

                case TokenType.PlaceholderGroup:
                    InstructionList group = ParseBackwards((TokenList)token.Val, true);
                    if (group.Count == 1)
                    {
                        stack.Push(group[0]);
                    }
                    else if (group.Count == 0)
                    {
                    }
                    else
                    {
                        throw new Exception("Unexpected grouping");
                    }
                    break;

                case TokenType.Word:
                case TokenType.Infix:
                case TokenType.Minus:
                    InstructionList inner = new InstructionList();

                    int         count;
                    ElementType etype;
                    if (token.Type == TokenType.Word)
                    {
                        count = CountArgs((string)token.Val, grouped && i == 0);
                        if (count == -1)
                        {
                            count = stack.Count;
                        }
                        etype = ElementType.Statement;
                    }
                    else if (token.Type == TokenType.Minus)
                    {
                        count     = 1;
                        etype     = ElementType.Statement;
                        token.Val = "Minus";
                    }
                    else
                    {
                        count = 2;
                        etype = ElementType.Infix;
                    }

                    for (int j = 0; j < count; j++)
                    {
                        inner.Add((Element)stack.Pop());
                    }

                    stack.Push(new Element(etype, token.Val, inner));
                    break;

                case TokenType.Newline:
                    break;

                case TokenType.Variable:
                    stack.Push(new Element(ElementType.Variable, token.Val));
                    break;

                case TokenType.QuestionMark:
                    // Don't need to check AllowQuestionMark here as the
                    // tokenizer checks already,
                    stack.Push(new Element(ElementType.QuestionMark, token.Val));
                    break;

                case TokenType.PlaceholderFunction:
                default:
                    throw new Exception("Unexpected token: " + token.Type + "<" + token.Val + ">");
                }
            }

            InstructionList tree = new InstructionList();

            object[] toplevel = stack.ToArray();
            foreach (object o in toplevel)
            {
                tree.Add((Element)o);
            }

            return(tree);
        }
示例#17
0
文件: Parser.cs 项目: emtees/old-code
		private InstructionList ParseBackwards (TokenList tokens, bool grouped) {
			Stack stack = new Stack ();
			
			for (int i = tokens.Count - 1; i >= 0; i--) {
				Token token = tokens[i];
				
				switch (token.Type) {
				case TokenType.Number:
				case TokenType.String:
					stack.Push (new Element (ElementType.Literal, token.Val));
					break;
				case TokenType.PlaceholderElement:
					stack.Push (token.Val);
					break;
				case TokenType.PlaceholderGroup:
					InstructionList group = ParseBackwards ((TokenList) token.Val, true);
					if (group.Count == 1) {
						stack.Push (group[0]);
					} else if (group.Count == 0) {
					} else {
						throw new Exception ("Unexpected grouping");
					}
					break;
				case TokenType.Word:
				case TokenType.Infix:
				case TokenType.Minus:
					InstructionList inner = new InstructionList ();

					int count;
					ElementType etype;
					if (token.Type == TokenType.Word) {
						count = CountArgs ((string) token.Val, grouped && i == 0);
						if (count == -1)
							count = stack.Count;
						etype = ElementType.Statement;
					} else if (token.Type == TokenType.Minus) {
						count = 1;
						etype = ElementType.Statement;
						token.Val = "Minus";
					} else {
						count = 2;
						etype = ElementType.Infix;
					}
					
					for (int j = 0; j < count; j++) 
						inner.Add ((Element) stack.Pop ());

					stack.Push (new Element (etype, token.Val, inner));
					break;
				case TokenType.Newline:
					break;
				case TokenType.Variable:
					stack.Push (new Element (ElementType.Variable, token.Val));
					break;
				case TokenType.QuestionMark:
					// Don't need to check AllowQuestionMark here as the
					// tokenizer checks already,
					stack.Push (new Element (ElementType.QuestionMark, token.Val));
					break;
				case TokenType.PlaceholderFunction:
				default:
					throw new Exception ("Unexpected token: " + token.Type + "<" + token.Val + ">");
				}
			}

			InstructionList tree = new InstructionList ();
			object[] toplevel = stack.ToArray ();
			foreach (object o in toplevel) {
				tree.Add ((Element) o);
			}

			return tree;
		}
示例#18
0
        public TokenList Parse(TextReader reader)
        {
            TokenList tokens = new TokenList();

            StringBuilder builder = new StringBuilder();
            int           c_int, c_prev_int = -1;
            bool          inside_bars    = false;
            bool          ignore_bar     = false;
            bool          inside_comment = false;

            for (c_int = reader.Read(); c_int != -1; c_prev_int = c_int, c_int = reader.Read())
            {
                char c = (char)c_int;

                if (inside_comment)
                {
                    if (IsNewline(c))
                    {
                        inside_comment = false;
                    }
                    else
                    {
                        continue;
                    }
                }

                if (c == '|')
                {
                    if (ignore_bar)
                    {
                        ignore_bar = false;
                        continue;
                    }

                    if (inside_bars && reader.Peek() == (int)'|')
                    {
                        ignore_bar = true;
                        builder.Append('|');
                        continue;
                    }

                    inside_bars = !inside_bars;
                    continue;
                }

                if (inside_bars)
                {
                    builder.Append(c);
                    continue;
                }

                if (c == '_' && tokens.Count > 0 && tokens[tokens.Count - 1].Type == TokenType.Newline)
                {
                    tokens.RemoveAt(tokens.Count - 1);
                    continue;
                }
                else if (c == ';')
                {
                    inside_comment = true;
                    continue;
                }
                else if (!IsDelimiter(c) || c_prev_int == (int)'"')
                {
                    builder.Append(c);
                    continue;
                }

                builder = AddTokens(tokens, builder, c_prev_int, reader.Peek(), c);
            }

            builder = AddTokens(tokens, builder, c_prev_int, reader.Peek());

            return(tokens);
        }
示例#19
0
文件: Parser.cs 项目: emtees/old-code
		private TokenList ParseForwards (TokenList tokens) {
			TokenList partial = new TokenList ();
			ParseForwards (partial, tokens.GetEnumerator (), 0);
			return partial;
		}
示例#20
0
文件: Parser.cs 项目: emtees/old-code
		private void PrintFunction (TokenList func) {
			Console.Write ("function {0}", (string) func[0].Val);
			foreach (Token token in (TokenList) func[1].Val) {
				if (token.Type == TokenType.Variable)
					Console.Write (" :{0}", token.Val);
				else if (token.Type == TokenType.PlaceholderElement) {
					TokenList arg = (TokenList) ((Element) token.Val).Val;
					Console.Write (" [");
					int j = 0;
					foreach (Token t in arg) {
						if (j == 0)
							Console.Write (":");
						Console.Write (t.Val);
						if (j < (arg.Count - 1))
							Console.Write (" ");
						j++;
					}
					Console.Write ("]");
				}
			}
			Console.WriteLine ();

			bool print_space = false;
			bool print_tab = true;
			foreach (Token token in (TokenList) func[2].Val) {
				if (print_space && token.Type != TokenType.Newline)
					Console.Write (" ");
				else
					print_space = true;

				if (print_tab) {
					Console.Write ("\t");
					print_tab = false;
				}

				if (token.Type == TokenType.Newline) {
					print_tab = true;
					print_space = false;
				}
					
				Console.Write ("{0}", token.Val);
			}

			Console.WriteLine ();
		}
示例#21
0
		public Function (string name, TokenList tokens, ArgumentInfo[] args) {
			this.name = name;
			this.tokens = tokens;
			this.args = args;
		}
示例#22
0
 public Function(string name, TokenList tokens, ArgumentInfo[] args)
 {
     this.name   = name;
     this.tokens = tokens;
     this.args   = args;
 }