public void AddRange(ITokenCollection collection) { foreach (var token in collection) { tokenCollection.Add(token); } }
public void TypedAndLiteralAreMergedByTypedLiteral() { var target = new TokenCollection(); target.Add(TokenIdentity.Typed(-1, typeof(string))); Assert.AreEqual(1, target.Count); target.Add(TokenIdentity.Literal(-1, "foo")); Assert.AreEqual(2, target.Count); target.Add(TokenIdentity.TypedLiteral(-1, "foo", typeof(string))); Assert.AreEqual(1, target.Count); }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc.First.TokenKind == TokenKind.TextData && tc.Count > 2 && (tc[1].TokenKind == TokenKind.LeftParentheses) && tc.Last.TokenKind == TokenKind.RightParentheses) { FunctaionTag tag = new FunctaionTag(); tag.Name = tc.First.Text; Int32 pos = 0, start = 2, end; for (Int32 i = 2; i < tc.Count; i++) { end = i; switch (tc[i].TokenKind) { case TokenKind.Comma: if (pos == 0) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); if (coll.Count > 0) { tag.AddChild(parser.Read(coll)); } start = i + 1; } break; default: if (tc[i].TokenKind == TokenKind.LeftParentheses) { pos++; } else if (tc[i].TokenKind == TokenKind.RightParentheses) { pos--; } if (i == tc.Count - 1) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); if (coll.Count > 0) { tag.AddChild(parser.Read(coll)); } } break; } } return(tag); } return(null); }
public TokenCollection ExtractTokens(Response response) { TokenCollection tokens = new TokenCollection(); if (response.Headers.ContainsKey("Content-Type") && response.Headers["Content-Type"].Count >= 1 && response.Headers["Content-Type"][0].Contains("text/html")) { HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(response.Content); var nodes = doc.DocumentNode.SelectNodes("//input[@type=\"hidden\"]"); if (nodes != null) { foreach (HtmlNode node in nodes) { string value = node.GetAttributeValue("value", ""); if (value != "") { HtmlFormToken token = new HtmlFormToken(node.GetAttributeValue("name", "likely-csrf"), value, TypeGuesser.GuessTypes(value)); tokens.Add(token); } } } } return(tokens); }
private TokenCollection GetInBrackets() { TokenCollection result = new TokenCollection(); var gen = 0; do { var elem = tokens[currentposition]; switch (elem.Kind) { case SyntaxKind.OB: case SyntaxKind.OBA: gen++; break; case SyntaxKind.CB: case SyntaxKind.CBA: gen--; break; } result.Add(elem); currentposition++; }while (gen != 0); return(result); }
private void AppendToken(Token rlt) { if (rlt.TokenType != TokenType.Ignore) { Tokens.Add(rlt); } }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc != null && parser != null && tc.Count > 3 && Common.Utility.IsEqual(tc.First.Text, Field.KEY_ELSEIF)) { if (tc[1].TokenKind == TokenKind.LeftParentheses && tc.Last.TokenKind == TokenKind.RightParentheses) { ElseifTag tag = new ElseifTag(); TokenCollection coll = new TokenCollection(); coll.Add(tc, 2, tc.Count - 2); tag.Test = parser.Read(coll); return(tag); } else { throw new Exception.ParseException(String.Concat("syntax error near if:", tc), tc.First.BeginLine, tc.First.BeginColumn); } } return(null); }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc != null && parser != null && tc.Count > 3 && Common.Utility.IsEqual(tc.First.Text, Field.KEY_ELSEIF)) { if (tc[1].TokenKind == TokenKind.LeftParentheses && tc.Last.TokenKind == TokenKind.RightParentheses) { ElseifTag tag = new ElseifTag(); TokenCollection coll = new TokenCollection(); coll.Add(tc, 2, tc.Count - 2); tag.Test = parser.Read(coll); return tag; } else { throw new Exception.ParseException(String.Concat("syntax error near if:", tc), tc.First.BeginLine, tc.First.BeginColumn); } } return null; }
public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc.Count > 0 && Common.ParserHelpers.IsEqual(Field.KEY_FOREACH, tc.First.Text)) { if (tc.Count > 5 && tc[1].TokenKind == TokenKind.LeftParentheses && tc[2].TokenKind == TokenKind.TextData && Common.ParserHelpers.IsEqual(tc[3].Text, Field.KEY_IN) && tc.Last.TokenKind == TokenKind.RightParentheses) { ForeachTag tag = new ForeachTag(); tag.Name = tc[2].Text; TokenCollection coll = new TokenCollection(); coll.Add(tc, 4, tc.Count - 2); tag.Source = parser.Read(coll); while (parser.MoveNext()) { tag.Children.Add(parser.Current); if (parser.Current is EndTag) { return(tag); } } throw new Exception.ParseException(String.Concat("foreach is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn); } else { throw new Exception.ParseException(String.Concat("syntax error near foreach:", tc), tc.First.BeginLine, tc.First.BeginColumn); } } return(null); }
private Tag Read() { Tag t = null; if (IsTagStart()) { Token t1, t2; t1 = t2 = GetToken(); TokenCollection tc = new TokenCollection(); do { this.index++; t2.Next = GetToken(); t2 = t2.Next; tc.Add(t2); } while (!IsTagEnd()); tc.Remove(tc.Last); this.index++; try { t = Read(tc); } catch (Exception.TemplateException) { throw; } catch (System.Exception e) { throw new Exception.ParseException(String.Concat("Parse error:", tc, "\r\nError message:", e.Message), tc.First.BeginLine, tc.First.BeginColumn);//标签分析异常 } if (t != null) { t.FirstToken = t1; if (t.Children.Count == 0 || t.LastToken == null || t2.CompareTo(t.LastToken) > 0) { t.LastToken = t2; } } else { throw new Exception.ParseException(String.Concat("Unexpected tag:", tc), tc.First.BeginLine, tc.First.BeginColumn); //未知的标签 } } else { t = new TextTag(); t.FirstToken = GetToken(); t.LastToken = null; this.index++; } return(t); }
/// <summary> /// Interpreta las palabras /// </summary> internal TokenCollection Parse() { TokenCollection tokens = new TokenCollection(); // Interpreta la cadena while (!CharSeparator.IsEof) { bool found = false; int firstCharacter; // Salta los espacios CharSeparator.SkipSpaces(); // Guarda el índice del primer carácter firstCharacter = CharSeparator.IndexActualChar; // Si no ha terminado ya if (!CharSeparator.IsEof) { // Obtiene la regla que se corresponde con la siguiente cadena foreach (RuleBase ruleBase in Rules) { if (!found) { switch (ruleBase) { case RuleDelimited rule: CheckRuleDelimited(rule, tokens, ref found); break; case RulePattern rule: CheckRulePattern(rule, tokens, ref found); break; case RuleWord rule: CheckRuleWord(rule, tokens, ref found); break; case RuleWordFixed rule: CheckRuleWordFixed(rule, tokens, ref found); break; default: throw new NotImplementedException("Unknown lexical rule"); } } } // Si no se ha encontrado nada, obtiene una palabra hasta el siguiente espacio if (!found) { tokens.Add(ReadWordToSpaces()); } // Obtiene la indentación del primer carácter tokens[tokens.Count - 1].Indent = CharSeparator.GetIndentFrom(firstCharacter); } } // Devuelve la colección de palabras return(tokens); }
private TokenCollection JsonToTokens(string json) { TokenCollection tokens = new TokenCollection(); try { JsonTextReader reader = new JsonTextReader(new StringReader(json)); while (reader.Read()) { if (reader.Value != null && reader.TokenType == Newtonsoft.Json.JsonToken.PropertyName) { string name = reader.Value.ToString() ?? ""; string path = reader.Path; // Console.WriteLine(path); reader.Read(); if (reader.TokenType == Newtonsoft.Json.JsonToken.String) { string value = reader.Value.ToString() ?? ""; Types supported = Types.String | TypeGuesser.GuessTypes(value); tokens.Add(new Tokens.JsonToken(name, value, path, supported)); } else if (reader.TokenType == Newtonsoft.Json.JsonToken.Integer) { tokens.Add(new Tokens.JsonToken(name, reader.Value.ToString() ?? "", path, Types.Integer)); } else if (reader.TokenType == Newtonsoft.Json.JsonToken.Boolean) { tokens.Add(new Tokens.JsonToken(name, reader.Value.ToString() ?? "", path, Types.Boolean)); } else if (reader.TokenType == Newtonsoft.Json.JsonToken.Null) { // TODO: Re-Evaluate handling of null values. tokens.Add(new Tokens.JsonToken(name, "", path, Types.Boolean)); } } } } catch { Console.WriteLine("JSON parsing failure."); } return(tokens); }
public TokenCollection GetRequirements(List <IRequestTokenizer> tokenizers) { TokenCollection tokens = new TokenCollection(); foreach (IRequestTokenizer tokenizer in tokenizers) { tokens.Add(tokenizer.ExtractTokens(this)); } return(tokens); }
/// <summary> /// Comprueba una regla a partir de un patrón de caracteres /// </summary> private void CheckRulePattern(RulePattern rule, TokenCollection tokens, ref bool found) { if (CharSeparator.CheckPatternStart(rule.PatternStart)) { // Añade el token de un patrón tokens.Add(GetToken(rule, CharSeparator.GetCharsPattern(rule.PatternStart, rule.PatternContent), false)); // Indica que se ha grabado found = true; } }
public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc.Count > 2 && tc.First.TokenKind == TokenKind.TextData && HasDot(tc)) { ReferenceTag tag = new ReferenceTag(); Int32 start, end, pos; start = end = pos = 0; for (Int32 i = 0; i < tc.Count; i++) { end = i; switch (tc[i].TokenKind) { case TokenKind.Dot: if (pos == 0) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); tag.AddChild(parser.Read(coll)); start = i + 1; } break; default: if (tc[i].TokenKind == TokenKind.LeftParentheses) { pos++; } else if (tc[i].TokenKind == TokenKind.RightParentheses) { pos--; } if (i == tc.Count - 1) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end); tag.AddChild(parser.Read(coll)); } break; } } if (tag.Children.Count > 0) { if (tag.Children.Count == 1) { return(tag.Children[0]); } } return(tag); } return(null); }
// 将运算符送入m_OperatorArray // parameters: void PutOperatorToArray(string strToken, int nType) { Token token = null; token = new Token(); token.m_strToken = strToken; token.m_nType = nType; m_OperatorArray.Add(token); }
public TokenCollection ExtractTokens(Request request) { TokenCollection tokens = new TokenCollection(); if (request.Headers.ContainsKey("Authorization") && request.Headers["Authorization"].Count >= 1) { string bearerToken = request.Headers["Authorization"][0].Split(' ', 2)[1]; tokens.Add(new BearerToken(bearerToken)); } return(tokens); }
/// <summary> /// Comprueba una regla delimitada /// </summary> private void CheckRuleDelimited(RuleDelimited rule, TokenCollection tokens, ref bool found) { string startRule = GetStartRule(rule.Starts); if (!string.IsNullOrWhiteSpace(startRule)) { // Añade la palabra tokens.Add(ReadWord(startRule, rule)); // Indica que se ha encontrado found = true; } }
public void Test_Add() { TokenCollection a = Create(new[] { "mouse", "mouse", "mouse" }); TokenCollection b = Create(new[] { "mouse", "chicken", "chicken" }); TokenCollection add = TokenCollection.Add(a, b); Assert.AreEqual(4, add.Sum); Assert.AreEqual(4, add.get("mouse")); Assert.IsFalse(add.Contains("chicken")); Assert.AreEqual(0, add.get("house")); }
public TokenCollection ExtractTokens(Request request) { TokenCollection tokens = new TokenCollection(); NameValueCollection parameters = HttpUtility.ParseQueryString(request.Url.Query); foreach (string key in parameters.AllKeys) { tokens.Add(new QueryToken(key, parameters[key], Types.Integer | Types.String)); } return(tokens); }
public void Test_Subtract_Rule() { TokenCollection a = new TokenCollection(); a.Add("mouse"); a.Add("mouse"); a.Add("mouse"); TokenCollection b = new TokenCollection(); b.Add("house"); b.Add("house"); try { TokenCollection.Subtract(b, a); Assert.Fail(); } catch (ArgumentException) { } }
/// <summary> /// Obtiene un token a partir de una palabra clave de longitud fija /// </summary> private void CheckRuleWordFixed(RuleWordFixed rule, TokenCollection tokens, ref bool found) { foreach (string word in rule.Words) { if (!found && CharSeparator.LookAtChar(word.Length).EqualsIgnoreCase(word)) { // Añade el token tokens.Add(GetToken(rule, word, true)); // Indica que se ha encontrado found = true; } } }
protected TokenCollection LoadListBarCode(object ID) { TokenCollection listBarCode = new TokenCollection(); if (ID != null) { DataTable dt = data.GetListBarCode(ID); foreach (DataRow row in dt.Rows) { listBarCode.Add(row["Barcode"].ToString()); } } return(listBarCode); }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc != null && parser != null && tc.Count > 3 && Common.Utility.IsEqual(tc.First.Text, Field.KEY_IF)) { if (tc[1].TokenKind == TokenKind.LeftParentheses && tc.Last.TokenKind == TokenKind.RightParentheses) { IfTag tag = new IfTag(); ElseifTag t = new ElseifTag(); TokenCollection coll = new TokenCollection(); coll.Add(tc, 2, tc.Count - 2); t.Test = parser.Read(coll); t.FirstToken = coll.First; //t.LastToken = coll.Last; tag.AddChild(t); while (parser.MoveNext()) { if (parser.Current is EndTag) { tag.AddChild(parser.Current); return tag; } else if (parser.Current is ElseifTag || parser.Current is ElseTag) { tag.AddChild(parser.Current); } else { tag.Children[tag.Children.Count - 1].AddChild(parser.Current); } } throw new Exception.ParseException(String.Concat("if is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn); } else { throw new Exception.ParseException(String.Concat("syntax error near if:", tc), tc.First.BeginLine, tc.First.BeginColumn); } } return null; }
protected TokenCollection LoadListBarCode(int ID) { TokenCollection listBarCode = new TokenCollection(); if (ID != null) { List <hhBarcode> barcode = DBDataProvider.GetDanhSach(ID); foreach (var bc in barcode) { listBarCode.Add(bc.Barcode); } } return(listBarCode); }
public TokenCollection ParseTokensFromStream(Stream s) { //Read entire stream StreamReader read = new StreamReader(s); string value = read.ReadToEnd(); //Replace with string subs int i = value.Length - 1; int j = -1; bool open = false; int o = -1; List <string> org = new List <string>(); //Iterate backwards while (i > -1 && (j = value.LastIndexOf('"', i)) != -1) { //Decide if literal is valid if (j > 0 && value[j - 1] == '\\') { i = j - 1; continue; } //Open or close if (open) { org.Add(value.Substring(j, o - j + 1)); value = value.Remove(j, o - j + 1).Insert(j, $"%STR{org.Count - 1}%"); open = false; } else { o = j; open = true; } i = j - 1; } //Tokenize TokenCollection c = new TokenCollection(); while (value != "") { c.Add(ParseNext(ref value)); } return(c); }
public TokenCollection ExtractTokens(Response response) { TokenCollection tokens = new TokenCollection(); if (response.Headers.ContainsKey("Set-Cookie")) { foreach (string cookieHeader in response.Headers["Set-Cookie"]) { string[] vals = cookieHeader.Split(';', '='); tokens.Add(new CookieToken(vals[0], vals[1], TypeGuesser.GuessTypes(vals[1]))); } } return(tokens); }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc != null && parser != null && tc.Count > 3 && Common.Utility.IsEqual(tc.First.Text, Field.KEY_IF)) { if (tc[1].TokenKind == TokenKind.LeftParentheses && tc.Last.TokenKind == TokenKind.RightParentheses) { IfTag tag = new IfTag(); ElseifTag t = new ElseifTag(); TokenCollection coll = new TokenCollection(); coll.Add(tc, 2, tc.Count - 2); t.Test = parser.Read(coll); t.FirstToken = coll.First; //t.LastToken = coll.Last; tag.AddChild(t); while (parser.MoveNext()) { if (parser.Current is EndTag) { tag.AddChild(parser.Current); return(tag); } else if (parser.Current is ElseifTag || parser.Current is ElseTag) { tag.AddChild(parser.Current); } else { tag.Children[tag.Children.Count - 1].AddChild(parser.Current); } } throw new Exception.ParseException(String.Concat("if is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn); } else { throw new Exception.ParseException(String.Concat("syntax error near if:", tc), tc.First.BeginLine, tc.First.BeginColumn); } } return(null); }
/// <summary> /// Add a new token /// </summary> /// <param name="sender">The sender of the event</param> /// <param name="e">The event arguments</param> private void btnAdd_Click(object sender, EventArgs e) { Token t = new Token("New Token", "Token value"); TreeNode node = tvTokens.Nodes.Add(t.TokenName); node.Tag = t; tokens.Add(t); tvTokens.SelectedNode = node; if (tokens.Count == 1) { txtTokenID.Enabled = editor.Enabled = btnDelete.Enabled = true; } txtTokenID.Focus(); }
/// <summary> /// Comprueba una regla de palabra reservada /// </summary> private void CheckRuleWord(RuleWord rule, TokenCollection tokens, ref bool found) { foreach (string word in rule.Words) { if (!found && CharSeparator.LookAtChar(word.Length).Equals(word, StringComparison.CurrentCultureIgnoreCase)) { if (rule.ToFirstSpace && CharSeparator.CheckIsSpace(CharSeparator.LookAtChars(word.Length, 1))) { // Añade el token tokens.Add(GetToken(rule, word, true)); // Indica que se ha encontrado found = true; } } } }
//===================================================================== /// <summary> /// Add a new token to the collection /// </summary> /// <param name="sender">The sender of the event</param> /// <param name="e">The event arguments</param> private void cmdAddItem_Executed(object sender, ExecutedRoutedEventArgs e) { if (tokens != null) { Token t = new Token { TokenValue = "Add token content here" }; tokens.Add(t); t.IsSelected = true; lbTokens.ScrollIntoView(t); txtTokenName.SelectAll(); txtTokenValue.SelectAll(); txtTokenName.Focus(); } }
protected static TokenCollection Create(string[] pTokens) { TokenCollection a = new TokenCollection(); foreach (string token in pTokens) { a.Add(token); } Assert.AreEqual(pTokens.Length, a.Sum); Assert.AreEqual(pTokens.Distinct().Count(), a.Count); foreach (string token in pTokens) { Assert.AreNotEqual(0, a.get(token)); } return(a); }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc != null && parser != null && tc.Count > 0 && Common.ParserHelpers.IsEqual(Field.KEY_FOREACH, tc.First.Text)) { if (tc.Count > 5 && tc[1].TokenKind == TokenKind.LeftParentheses && tc[2].TokenKind == TokenKind.TextData && Common.ParserHelpers.IsEqual(tc[3].Text, Field.KEY_IN) && tc.Last.TokenKind == TokenKind.RightParentheses) { ForeachTag tag = new ForeachTag(); tag.Name = tc[2].Text; TokenCollection coll = new TokenCollection(); coll.Add(tc, 4, tc.Count - 2); tag.Source = parser.Read(coll); while (parser.MoveNext()) { tag.Children.Add(parser.Current); if (parser.Current is EndTag) { return tag; } } throw new Exception.ParseException(String.Concat("foreach is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn); } else { throw new Exception.ParseException(String.Concat("syntax error near foreach:", tc), tc.First.BeginLine, tc.First.BeginColumn); } } return null; }
public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc.Count > 2 && tc.First.TokenKind == TokenKind.TextData && HasDot(tc)) { ReferenceTag tag = new ReferenceTag(); Int32 start, end, pos; start = end = pos = 0; for (Int32 i = 0; i < tc.Count; i++) { end = i; switch (tc[i].TokenKind) { case TokenKind.Dot: if (pos == 0) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); tag.AddChild(parser.Read(coll)); start = i + 1; } break; default: if (tc[i].TokenKind == TokenKind.LeftParentheses) { pos++; } else if (tc[i].TokenKind == TokenKind.RightParentheses) { pos--; } if (i == tc.Count - 1) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end); tag.AddChild(parser.Read(coll)); } break; } } if (tag.Children.Count > 0) { if (tag.Children.Count == 1) { return tag.Children[0]; } } return tag; } return null; }
private TokenCollection BuildSentences(TokenCollection tokens) { var compressedTokens = new TokenCollection(); for (int i = 0; i < tokens.Count; i++) { var token = tokens[i]; if (token.Type == TokenType.Quote) { StringBuilder sentence = new StringBuilder(); for (int j = i + 1; j < tokens.Count; j++) { if (tokens[j].Type == TokenType.Quote) { i = j; // Skip to after the quote. break; } if (tokens[j].Type != TokenType.Word) throw new Exception(); if (j > i + 1) sentence.Append(" "); sentence.Append(tokens[j].Text); } compressedTokens.Add(new Token(sentence.ToString())); } else { compressedTokens.Add(token); } } return compressedTokens; }
/// <summary> /// Create tokens out of a raw search query. /// </summary> /// <param name="input"></param> /// <returns></returns> public TokenCollection Tokenize(string input) { var tokens = new TokenCollection(); if (!string.IsNullOrEmpty(input)) { StateFlags state = StateFlags.None; string[] words = input.Split(' ', '\t'); for (int i = 0; i < words.Length; i++) { string word = words[i]; int addClosingParen = 0; bool addClosingQuote = false; while (word[0] == '(') { tokens.Add(new Token(TokenType.ParenthesisOpen)); word = word.Substring(1); } if (word.Length == 0) continue; if (word[0] == '"') { if ((state & StateFlags.IsInsideQuotes) != 0) throw new Exception(); tokens.Add(new Token(TokenType.Quote)); state |= StateFlags.IsInsideQuotes; word = word.Substring(1); } if (word.Length == 0) continue; while (word[word.Length - 1] == ')') { ++addClosingParen; word = word.Substring(0, word.Length - 1); } if (word.Length == 0) { while (addClosingParen-- > 0) tokens.Add(new Token(TokenType.ParenthesisClose)); continue; } if (word[word.Length - 1] == '"') { addClosingQuote = true; word = word.Substring(0, word.Length - 1); } if (word.Length == 0) { if (addClosingQuote) tokens.Add(new Token(TokenType.Quote)); while (addClosingParen-- > 0) tokens.Add(new Token(TokenType.ParenthesisClose)); } int semiColonIndex = word.IndexOf(':'); if (semiColonIndex < 0 || (state & StateFlags.IsInsideQuotes) != 0) { if ((state & StateFlags.IsInsideQuotes) == 0 && word.Equals("AND", StringComparison.OrdinalIgnoreCase)) tokens.Add(new Token(TokenType.BooleanAnd)); else if ((state & StateFlags.IsInsideQuotes) == 0 && word.Equals("OR", StringComparison.OrdinalIgnoreCase)) tokens.Add(new Token(TokenType.BooleanOr)); else if ((state & StateFlags.IsInsideQuotes) == 0 && word.Equals("NOT", StringComparison.OrdinalIgnoreCase)) tokens.Add(new Token(TokenType.UnaryNot)); else tokens.Add(new Token(word)); } else { string operatorName = word.Substring(0, semiColonIndex); tokens.Add(new Token(TokenType.Operator, operatorName)); string operatorArgumentStart = word.Substring(semiColonIndex + 1); if (operatorArgumentStart[0] == '"') { tokens.Add(new Token(TokenType.Quote)); state |= StateFlags.IsInsideQuotes; operatorArgumentStart = operatorArgumentStart.Substring(1); } tokens.Add(new Token(operatorArgumentStart)); } if (addClosingQuote) { if ((state & StateFlags.IsInsideQuotes) == 0) throw new Exception(); tokens.Add(new Token(TokenType.Quote)); state &= ~StateFlags.IsInsideQuotes; } while (addClosingParen-- > 0) tokens.Add(new Token(TokenType.ParenthesisClose)); } } return BuildSentences(tokens); }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc == null || parser == null) { return null; } //支持写法:简写格式: //常规格式: if (tc.Count > 5 && Common.ParserHelpers.IsEqual(tc.First.Text, Field.KEY_SET) && tc[1].TokenKind == TokenKind.LeftParentheses && tc[3].Text == "=" && tc.Last.TokenKind == TokenKind.RightParentheses) { SetTag tag = new SetTag(); tag.Name = tc[2].Text; TokenCollection coll = new TokenCollection(); coll.Add(tc, 4, tc.Count - 2); tag.Value = parser.Read(coll); return tag; } if (tc.Count == 2 && tc.First.TokenKind == TokenKind.TextData && tc.Last.TokenKind == TokenKind.Operator && (tc.Last.Text == "++" || tc.Last.Text == "--")) { SetTag tag = new SetTag(); tag.Name = tc.First.Text; ExpressionTag c = new ExpressionTag(); c.AddChild(new VariableTag() { FirstToken = tc.First, Name = tc.First.Text }); c.AddChild(new TextTag() { FirstToken = new Token(TokenKind.Operator, tc.Last.Text[0].ToString()) }); c.AddChild(new NumberTag() { Value = 1, FirstToken = new Token(TokenKind.Number, "1") }); tag.Value = c; return tag; } if (tc.Count > 2 && tc.First.TokenKind == TokenKind.TextData && tc[1].Text == "=") { SetTag tag = new SetTag(); tag.Name = tc.First.Text; TokenCollection coll = new TokenCollection(); coll.Add(tc, 2, tc.Count - 1); tag.Value = parser.Read(coll); return tag; } return null; }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(JinianNet.JNTemplate.Parser.TemplateParser parser, TokenCollection tc) { if (tc != null && parser != null && tc.Count > 2) { Int32 start, end, pos; start = end = pos = 0; Boolean isFunc = false; List<Token> data = new List<Token>(); Queue<TokenCollection> queue = new Queue<TokenCollection>(); for (Int32 i = 0; i < tc.Count; i++) { end = i; if (tc[i].TokenKind == TokenKind.LeftParentheses) { if (pos == 0) { if (i > 0 && tc[i - 1].TokenKind == TokenKind.TextData) { isFunc = true; } } pos++; } else if (tc[i].TokenKind == TokenKind.RightParentheses) { if (pos > 0) { pos--; } else { throw new Exception.ParseException(String.Concat("syntax error near ):", tc), data[i].BeginLine, data[i].BeginColumn); } if (pos == 0) { TokenCollection coll = new TokenCollection(); if (!isFunc) { coll.Add(tc, start + 1, end - 1); } else { coll.Add(tc, start, end); } queue.Enqueue(coll); data.Add(null); start = i + 1; //tag.AddChild(parser.Read(coll)); } } else if (pos == 0 && (tc[i].TokenKind == TokenKind.Dot || tc[i].TokenKind == TokenKind.Operator)) { if (end > start) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); queue.Enqueue(coll); data.Add(null); } start = i + 1; data.Add(tc[i]); } if (i == tc.Count - 1 && end >= start) { if (start == 0 && end == i) { throw new Exception.ParseException(String.Concat("Unexpected tag:", tc), tc[0].BeginLine, tc[0].BeginColumn); } TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end); queue.Enqueue(coll); data.Add(null); start = i + 1; } } List<Tag> tags = new List<Tag>(); for (Int32 i = 0; i < data.Count; i++) { if (data[i] == null) { tags.Add(parser.Read(queue.Dequeue())); } else if (data[i].TokenKind == TokenKind.Dot) { if (tags.Count == 0 || i == data.Count - 1 || data[i + 1] != null) { throw new Exception.ParseException(String.Concat("syntax error near .:", tc), data[i].BeginLine, data[i].BeginColumn); } if (tags[tags.Count - 1] is ReferenceTag) { tags[tags.Count - 1].AddChild(parser.Read(queue.Dequeue())); } else { ReferenceTag t = new ReferenceTag(); t.AddChild(tags[tags.Count - 1]); t.AddChild(parser.Read(queue.Dequeue())); tags[tags.Count - 1] = t; } i++; } else if (data[i].TokenKind == TokenKind.Operator) { tags.Add(new TextTag()); tags[tags.Count - 1].FirstToken = data[i]; } } if (tags.Count == 1) { return tags[0]; } if (tags.Count > 1) { ExpressionTag t = new ExpressionTag(); for (Int32 i = 0; i < tags.Count; i++) { t.AddChild(tags[i]); } tags.Clear(); return t; } } return null; }
public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc.Count > 2 && HasOperator(tc)) { Int32 start, end, pos; start = end = pos = 0; #region 去括号 //(8+2) ==》 8+2 //(8+2) * (10-5) ==》(8+2) * (10-5) if (tc.First.TokenKind == TokenKind.LeftParentheses && tc.Last.TokenKind == TokenKind.RightParentheses) { for (Int32 i = 1; i < tc.Count - 1; i++) { switch (tc[i].TokenKind) { case TokenKind.LeftParentheses: pos++; break; case TokenKind.RightParentheses: if (pos > 0) { pos--; } break; } } if (pos == 0) { tc = new TokenCollection(tc, 1, tc.Count - 2); } else { pos = 0; } } #endregion ExpressionTag tag = new ExpressionTag(); #region 执行表达式折分 for (Int32 i = 0; i < tc.Count; i++) { end = i; switch (tc[i].TokenKind) { case TokenKind.Operator: if (pos == 0) { if (start != end) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); tag.AddChild(parser.Read(coll)); } tag.AddChild(new TextTag()); tag.Children[tag.Children.Count - 1].FirstToken = tc[i]; start = i + 1; } break; default: if (tc[i].TokenKind == TokenKind.LeftParentheses) { pos++; } else if (tc[i].TokenKind == TokenKind.RightParentheses) { pos--; } if (i == tc.Count - 1) { TokenCollection coll = new TokenCollection(); if (tc[start].TokenKind == TokenKind.RightParentheses) { coll.Add(tc, start + 1, end - 1); } else { coll.Add(tc, start, end); } start = i + 1; if (coll.Count > 0) { tag.AddChild(parser.Read(coll)); } } break; } } #endregion if (tag.Children.Count > 0) { if (tag.Children.Count == 1) { return tag.Children[0]; } return tag; } } return null; }
/// <summary> /// This converts the token entries to a token file and adds it to /// the project. /// </summary> private void CreateTokenFile() { XmlReader xr = converter.Reader; StreamWriter sw = null; string tokenFile = Path.Combine(converter.ProjectFolder, Path.GetFileNameWithoutExtension(converter.Project.Filename) + ".tokens"); // Create an empty token file try { sw = File.CreateText(tokenFile); sw.WriteLine("<content/>"); } finally { if(sw != null) sw.Close(); } FileItem fileItem = converter.Project.AddFileToProject(tokenFile, tokenFile); TokenCollection tokens = new TokenCollection(fileItem.FullPath); while(!xr.EOF && xr.NodeType != XmlNodeType.EndElement) { if(xr.NodeType == XmlNodeType.Element && xr.Name == "token") tokens.Add(new Token(xr.GetAttribute("name"), xr.GetAttribute("value"))); xr.Read(); } tokens.Save(); }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc != null && parser != null && tc.First.TokenKind == TokenKind.TextData && tc.Count > 2 && (tc[1].TokenKind == TokenKind.LeftParentheses) && tc.Last.TokenKind == TokenKind.RightParentheses) { FunctaionTag tag = new FunctaionTag(); tag.Name = tc.First.Text; Int32 pos = 0, start = 2, end; for (Int32 i = 2; i < tc.Count; i++) { end = i; switch (tc[i].TokenKind) { case TokenKind.Comma: if (pos == 0) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); if (coll.Count > 0) { tag.AddChild(parser.Read(coll)); } start = i + 1; } break; default: if (tc[i].TokenKind == TokenKind.LeftParentheses) { pos++; } else if (tc[i].TokenKind == TokenKind.RightParentheses) { pos--; } if (i == tc.Count - 1) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); if (coll.Count > 0) { tag.AddChild(parser.Read(coll)); } } break; } } return tag; } return null; }
/// <summary> /// 分析标签 /// </summary> /// <param name="parser">TemplateParser</param> /// <param name="tc">Token集合</param> /// <returns></returns> public Tag Parse(TemplateParser parser, TokenCollection tc) { if (tc != null && parser != null && tc.Count > 3 && Common.Utility.IsEqual(Field.KEY_FOR, tc.First.Text)) { if (tc[1].TokenKind == TokenKind.LeftParentheses && tc.Last.TokenKind == TokenKind.RightParentheses) { Int32 pos = 0, start = 2, end; List<Tag> ts = new List<Tag>(3); ForTag tag = new ForTag(); for (Int32 i = 2; i < tc.Count - 1; i++) { end = i; if (tc[i].TokenKind == TokenKind.Punctuation && tc[i].Text == ";") { if (pos == 0) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end - 1); if (coll.Count > 0) { ts.Add(parser.Read(coll)); } else { ts.Add(null); } start = i + 1; continue; } } if (tc[i].TokenKind == TokenKind.LeftParentheses) { pos++; } else if (tc[i].TokenKind == TokenKind.RightParentheses) { pos--; } if (i == tc.Count - 2) { TokenCollection coll = new TokenCollection(); coll.Add(tc, start, end); if (coll.Count > 0) { ts.Add(parser.Read(coll)); } else { ts.Add(null); } } } if (ts.Count != 3) { throw new Exception.ParseException(String.Concat("syntax error near for:", tc), tc.First.BeginLine, tc.First.BeginColumn); } tag.Initial = ts[0]; tag.Test = ts[1]; tag.Do = ts[2]; while (parser.MoveNext()) { tag.Children.Add(parser.Current); if (parser.Current is EndTag) { return tag; } } throw new Exception.ParseException(String.Concat("for is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn); } else { throw new Exception.ParseException(String.Concat("syntax error near for:", tc), tc.First.BeginLine, tc.First.BeginColumn); } } return null; }