public void AddRange(ITokenCollection collection)
{
foreach (var token in collection)
{
tokenCollection.Add(token);
}
}
public void TypedAndLiteralAreMergedByTypedLiteral()
{
var target = new TokenCollection();
target.Add(TokenIdentity.Typed(-1, typeof(string)));
Assert.AreEqual(1, target.Count);
target.Add(TokenIdentity.Literal(-1, "foo"));
Assert.AreEqual(2, target.Count);
target.Add(TokenIdentity.TypedLiteral(-1, "foo", typeof(string)));
Assert.AreEqual(1, target.Count);
}
public void TypedAndLiteralAreMergedByTypedLiteral()
{
var target = new TokenCollection();
target.Add(TokenIdentity.Typed(-1, typeof(string)));
Assert.AreEqual(1, target.Count);
target.Add(TokenIdentity.Literal(-1, "foo"));
Assert.AreEqual(2, target.Count);
target.Add(TokenIdentity.TypedLiteral(-1, "foo", typeof(string)));
Assert.AreEqual(1, target.Count);
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc.First.TokenKind == TokenKind.TextData &&
tc.Count > 2 &&
(tc[1].TokenKind == TokenKind.LeftParentheses) &&
tc.Last.TokenKind == TokenKind.RightParentheses)
{
FunctaionTag tag = new FunctaionTag();
tag.Name = tc.First.Text;
Int32 pos = 0,
start = 2,
end;
for (Int32 i = 2; i < tc.Count; i++)
{
end = i;
switch (tc[i].TokenKind)
{
case TokenKind.Comma:
if (pos == 0)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
if (coll.Count > 0)
{
tag.AddChild(parser.Read(coll));
}
start = i + 1;
}
break;
default:
if (tc[i].TokenKind == TokenKind.LeftParentheses)
{
pos++;
}
else if (tc[i].TokenKind == TokenKind.RightParentheses)
{
pos--;
}
if (i == tc.Count - 1)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
if (coll.Count > 0)
{
tag.AddChild(parser.Read(coll));
}
}
break;
}
}
return(tag);
}
return(null);
}
public TokenCollection ExtractTokens(Response response)
{
TokenCollection tokens = new TokenCollection();
if (response.Headers.ContainsKey("Content-Type") &&
response.Headers["Content-Type"].Count >= 1 &&
response.Headers["Content-Type"][0].Contains("text/html"))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(response.Content);
var nodes = doc.DocumentNode.SelectNodes("//input[@type=\"hidden\"]");
if (nodes != null)
{
foreach (HtmlNode node in nodes)
{
string value = node.GetAttributeValue("value", "");
if (value != "")
{
HtmlFormToken token = new HtmlFormToken(node.GetAttributeValue("name", "likely-csrf"), value, TypeGuesser.GuessTypes(value));
tokens.Add(token);
}
}
}
}
return(tokens);
}
private TokenCollection GetInBrackets()
{
TokenCollection result = new TokenCollection();
var gen = 0;
do
{
var elem = tokens[currentposition];
switch (elem.Kind)
{
case SyntaxKind.OB:
case SyntaxKind.OBA:
gen++;
break;
case SyntaxKind.CB:
case SyntaxKind.CBA:
gen--;
break;
}
result.Add(elem);
currentposition++;
}while (gen != 0);
return(result);
}
private void AppendToken(Token rlt)
{
if (rlt.TokenType != TokenType.Ignore)
{
Tokens.Add(rlt);
}
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc != null &&
parser != null &&
tc.Count > 3 &&
Common.Utility.IsEqual(tc.First.Text, Field.KEY_ELSEIF))
{
if (tc[1].TokenKind == TokenKind.LeftParentheses &&
tc.Last.TokenKind == TokenKind.RightParentheses)
{
ElseifTag tag = new ElseifTag();
TokenCollection coll = new TokenCollection();
coll.Add(tc, 2, tc.Count - 2);
tag.Test = parser.Read(coll);
return(tag);
}
else
{
throw new Exception.ParseException(String.Concat("syntax error near if:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
}
return(null);
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc != null
&& parser != null
&& tc.Count > 3
&& Common.Utility.IsEqual(tc.First.Text, Field.KEY_ELSEIF))
{
if (tc[1].TokenKind == TokenKind.LeftParentheses
&& tc.Last.TokenKind == TokenKind.RightParentheses)
{
ElseifTag tag = new ElseifTag();
TokenCollection coll = new TokenCollection();
coll.Add(tc, 2, tc.Count - 2);
tag.Test = parser.Read(coll);
return tag;
}
else
{
throw new Exception.ParseException(String.Concat("syntax error near if:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
}
return null;
}
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc.Count > 0 && Common.ParserHelpers.IsEqual(Field.KEY_FOREACH, tc.First.Text))
{
if (tc.Count > 5 &&
tc[1].TokenKind == TokenKind.LeftParentheses &&
tc[2].TokenKind == TokenKind.TextData &&
Common.ParserHelpers.IsEqual(tc[3].Text, Field.KEY_IN) &&
tc.Last.TokenKind == TokenKind.RightParentheses)
{
ForeachTag tag = new ForeachTag();
tag.Name = tc[2].Text;
TokenCollection coll = new TokenCollection();
coll.Add(tc, 4, tc.Count - 2);
tag.Source = parser.Read(coll);
while (parser.MoveNext())
{
tag.Children.Add(parser.Current);
if (parser.Current is EndTag)
{
return(tag);
}
}
throw new Exception.ParseException(String.Concat("foreach is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
else
{
throw new Exception.ParseException(String.Concat("syntax error near foreach:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
}
return(null);
}
private Tag Read()
{
Tag t = null;
if (IsTagStart())
{
Token t1, t2;
t1 = t2 = GetToken();
TokenCollection tc = new TokenCollection();
do
{
this.index++;
t2.Next = GetToken();
t2 = t2.Next;
tc.Add(t2);
} while (!IsTagEnd());
tc.Remove(tc.Last);
this.index++;
try
{
t = Read(tc);
}
catch (Exception.TemplateException)
{
throw;
}
catch (System.Exception e)
{
throw new Exception.ParseException(String.Concat("Parse error:", tc, "\r\nError message:", e.Message), tc.First.BeginLine, tc.First.BeginColumn);//标签分析异常
}
if (t != null)
{
t.FirstToken = t1;
if (t.Children.Count == 0 || t.LastToken == null || t2.CompareTo(t.LastToken) > 0)
{
t.LastToken = t2;
}
}
else
{
throw new Exception.ParseException(String.Concat("Unexpected tag:", tc), tc.First.BeginLine, tc.First.BeginColumn); //未知的标签
}
}
else
{
t = new TextTag();
t.FirstToken = GetToken();
t.LastToken = null;
this.index++;
}
return(t);
}
/// <summary>
/// Interpreta las palabras
/// </summary>
internal TokenCollection Parse()
{
TokenCollection tokens = new TokenCollection();
// Interpreta la cadena
while (!CharSeparator.IsEof)
{
bool found = false;
int firstCharacter;
// Salta los espacios
CharSeparator.SkipSpaces();
// Guarda el índice del primer carácter
firstCharacter = CharSeparator.IndexActualChar;
// Si no ha terminado ya
if (!CharSeparator.IsEof)
{
// Obtiene la regla que se corresponde con la siguiente cadena
foreach (RuleBase ruleBase in Rules)
{
if (!found)
{
switch (ruleBase)
{
case RuleDelimited rule:
CheckRuleDelimited(rule, tokens, ref found);
break;
case RulePattern rule:
CheckRulePattern(rule, tokens, ref found);
break;
case RuleWord rule:
CheckRuleWord(rule, tokens, ref found);
break;
case RuleWordFixed rule:
CheckRuleWordFixed(rule, tokens, ref found);
break;
default:
throw new NotImplementedException("Unknown lexical rule");
}
}
}
// Si no se ha encontrado nada, obtiene una palabra hasta el siguiente espacio
if (!found)
{
tokens.Add(ReadWordToSpaces());
}
// Obtiene la indentación del primer carácter
tokens[tokens.Count - 1].Indent = CharSeparator.GetIndentFrom(firstCharacter);
}
}
// Devuelve la colección de palabras
return(tokens);
}
private TokenCollection JsonToTokens(string json)
{
TokenCollection tokens = new TokenCollection();
try
{
JsonTextReader reader = new JsonTextReader(new StringReader(json));
while (reader.Read())
{
if (reader.Value != null && reader.TokenType == Newtonsoft.Json.JsonToken.PropertyName)
{
string name = reader.Value.ToString() ?? "";
string path = reader.Path;
// Console.WriteLine(path);
reader.Read();
if (reader.TokenType == Newtonsoft.Json.JsonToken.String)
{
string value = reader.Value.ToString() ?? "";
Types supported = Types.String | TypeGuesser.GuessTypes(value);
tokens.Add(new Tokens.JsonToken(name, value, path, supported));
}
else if (reader.TokenType == Newtonsoft.Json.JsonToken.Integer)
{
tokens.Add(new Tokens.JsonToken(name, reader.Value.ToString() ?? "", path, Types.Integer));
}
else if (reader.TokenType == Newtonsoft.Json.JsonToken.Boolean)
{
tokens.Add(new Tokens.JsonToken(name, reader.Value.ToString() ?? "", path, Types.Boolean));
}
else if (reader.TokenType == Newtonsoft.Json.JsonToken.Null)
{
// TODO: Re-Evaluate handling of null values.
tokens.Add(new Tokens.JsonToken(name, "", path, Types.Boolean));
}
}
}
}
catch
{
Console.WriteLine("JSON parsing failure.");
}
return(tokens);
}
public TokenCollection GetRequirements(List <IRequestTokenizer> tokenizers)
{
TokenCollection tokens = new TokenCollection();
foreach (IRequestTokenizer tokenizer in tokenizers)
{
tokens.Add(tokenizer.ExtractTokens(this));
}
return(tokens);
}
/// <summary>
/// Comprueba una regla a partir de un patrón de caracteres
/// </summary>
private void CheckRulePattern(RulePattern rule, TokenCollection tokens, ref bool found)
{
if (CharSeparator.CheckPatternStart(rule.PatternStart))
{
// Añade el token de un patrón
tokens.Add(GetToken(rule, CharSeparator.GetCharsPattern(rule.PatternStart, rule.PatternContent), false));
// Indica que se ha grabado
found = true;
}
}
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc.Count > 2 &&
tc.First.TokenKind == TokenKind.TextData &&
HasDot(tc))
{
ReferenceTag tag = new ReferenceTag();
Int32 start, end, pos;
start = end = pos = 0;
for (Int32 i = 0; i < tc.Count; i++)
{
end = i;
switch (tc[i].TokenKind)
{
case TokenKind.Dot:
if (pos == 0)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
tag.AddChild(parser.Read(coll));
start = i + 1;
}
break;
default:
if (tc[i].TokenKind == TokenKind.LeftParentheses)
{
pos++;
}
else if (tc[i].TokenKind == TokenKind.RightParentheses)
{
pos--;
}
if (i == tc.Count - 1)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end);
tag.AddChild(parser.Read(coll));
}
break;
}
}
if (tag.Children.Count > 0)
{
if (tag.Children.Count == 1)
{
return(tag.Children[0]);
}
}
return(tag);
}
return(null);
}
// 将运算符送入m_OperatorArray
// parameters:
void PutOperatorToArray(string strToken,
int nType)
{
Token token = null;
token = new Token();
token.m_strToken = strToken;
token.m_nType = nType;
m_OperatorArray.Add(token);
}
public TokenCollection ExtractTokens(Request request)
{
TokenCollection tokens = new TokenCollection();
if (request.Headers.ContainsKey("Authorization") && request.Headers["Authorization"].Count >= 1)
{
string bearerToken = request.Headers["Authorization"][0].Split(' ', 2)[1];
tokens.Add(new BearerToken(bearerToken));
}
return(tokens);
}
/// <summary>
/// Comprueba una regla delimitada
/// </summary>
private void CheckRuleDelimited(RuleDelimited rule, TokenCollection tokens, ref bool found)
{
string startRule = GetStartRule(rule.Starts);
if (!string.IsNullOrWhiteSpace(startRule))
{
// Añade la palabra
tokens.Add(ReadWord(startRule, rule));
// Indica que se ha encontrado
found = true;
}
}
public void Test_Add()
{
TokenCollection a = Create(new[] { "mouse", "mouse", "mouse" });
TokenCollection b = Create(new[] { "mouse", "chicken", "chicken" });
TokenCollection add = TokenCollection.Add(a, b);
Assert.AreEqual(4, add.Sum);
Assert.AreEqual(4, add.get("mouse"));
Assert.IsFalse(add.Contains("chicken"));
Assert.AreEqual(0, add.get("house"));
}
public TokenCollection ExtractTokens(Request request)
{
TokenCollection tokens = new TokenCollection();
NameValueCollection parameters = HttpUtility.ParseQueryString(request.Url.Query);
foreach (string key in parameters.AllKeys)
{
tokens.Add(new QueryToken(key, parameters[key], Types.Integer | Types.String));
}
return(tokens);
}
public void Test_Subtract_Rule()
{
TokenCollection a = new TokenCollection();
a.Add("mouse");
a.Add("mouse");
a.Add("mouse");
TokenCollection b = new TokenCollection();
b.Add("house");
b.Add("house");
try
{
TokenCollection.Subtract(b, a);
Assert.Fail();
}
catch (ArgumentException)
{
}
}
/// <summary>
/// Obtiene un token a partir de una palabra clave de longitud fija
/// </summary>
private void CheckRuleWordFixed(RuleWordFixed rule, TokenCollection tokens, ref bool found)
{
foreach (string word in rule.Words)
{
if (!found && CharSeparator.LookAtChar(word.Length).EqualsIgnoreCase(word))
{
// Añade el token
tokens.Add(GetToken(rule, word, true));
// Indica que se ha encontrado
found = true;
}
}
}
protected TokenCollection LoadListBarCode(object ID)
{
TokenCollection listBarCode = new TokenCollection();
if (ID != null)
{
DataTable dt = data.GetListBarCode(ID);
foreach (DataRow row in dt.Rows)
{
listBarCode.Add(row["Barcode"].ToString());
}
}
return(listBarCode);
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc != null
&& parser != null
&& tc.Count > 3
&& Common.Utility.IsEqual(tc.First.Text, Field.KEY_IF))
{
if (tc[1].TokenKind == TokenKind.LeftParentheses
&& tc.Last.TokenKind == TokenKind.RightParentheses)
{
IfTag tag = new IfTag();
ElseifTag t = new ElseifTag();
TokenCollection coll = new TokenCollection();
coll.Add(tc, 2, tc.Count - 2);
t.Test = parser.Read(coll);
t.FirstToken = coll.First;
//t.LastToken = coll.Last;
tag.AddChild(t);
while (parser.MoveNext())
{
if (parser.Current is EndTag)
{
tag.AddChild(parser.Current);
return tag;
}
else if (parser.Current is ElseifTag
|| parser.Current is ElseTag)
{
tag.AddChild(parser.Current);
}
else
{
tag.Children[tag.Children.Count - 1].AddChild(parser.Current);
}
}
throw new Exception.ParseException(String.Concat("if is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
else
{
throw new Exception.ParseException(String.Concat("syntax error near if:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
}
return null;
}
protected TokenCollection LoadListBarCode(int ID)
{
TokenCollection listBarCode = new TokenCollection();
if (ID != null)
{
List <hhBarcode> barcode = DBDataProvider.GetDanhSach(ID);
foreach (var bc in barcode)
{
listBarCode.Add(bc.Barcode);
}
}
return(listBarCode);
}
public TokenCollection ParseTokensFromStream(Stream s)
{
//Read entire stream
StreamReader read = new StreamReader(s);
string value = read.ReadToEnd();
//Replace with string subs
int i = value.Length - 1;
int j = -1;
bool open = false;
int o = -1;
List <string> org = new List <string>();
//Iterate backwards
while (i > -1 && (j = value.LastIndexOf('"', i)) != -1)
{
//Decide if literal is valid
if (j > 0 && value[j - 1] == '\\')
{
i = j - 1;
continue;
}
//Open or close
if (open)
{
org.Add(value.Substring(j, o - j + 1));
value = value.Remove(j, o - j + 1).Insert(j, $"%STR{org.Count - 1}%");
open = false;
}
else
{
o = j;
open = true;
}
i = j - 1;
}
//Tokenize
TokenCollection c = new TokenCollection();
while (value != "")
{
c.Add(ParseNext(ref value));
}
return(c);
}
public TokenCollection ExtractTokens(Response response)
{
TokenCollection tokens = new TokenCollection();
if (response.Headers.ContainsKey("Set-Cookie"))
{
foreach (string cookieHeader in response.Headers["Set-Cookie"])
{
string[] vals = cookieHeader.Split(';', '=');
tokens.Add(new CookieToken(vals[0], vals[1], TypeGuesser.GuessTypes(vals[1])));
}
}
return(tokens);
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc != null &&
parser != null &&
tc.Count > 3 &&
Common.Utility.IsEqual(tc.First.Text, Field.KEY_IF))
{
if (tc[1].TokenKind == TokenKind.LeftParentheses &&
tc.Last.TokenKind == TokenKind.RightParentheses)
{
IfTag tag = new IfTag();
ElseifTag t = new ElseifTag();
TokenCollection coll = new TokenCollection();
coll.Add(tc, 2, tc.Count - 2);
t.Test = parser.Read(coll);
t.FirstToken = coll.First;
//t.LastToken = coll.Last;
tag.AddChild(t);
while (parser.MoveNext())
{
if (parser.Current is EndTag)
{
tag.AddChild(parser.Current);
return(tag);
}
else if (parser.Current is ElseifTag ||
parser.Current is ElseTag)
{
tag.AddChild(parser.Current);
}
else
{
tag.Children[tag.Children.Count - 1].AddChild(parser.Current);
}
}
throw new Exception.ParseException(String.Concat("if is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
else
{
throw new Exception.ParseException(String.Concat("syntax error near if:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
}
return(null);
}
/// <summary>
/// Add a new token
/// </summary>
/// <param name="sender">The sender of the event</param>
/// <param name="e">The event arguments</param>
private void btnAdd_Click(object sender, EventArgs e)
{
Token t = new Token("New Token", "Token value");
TreeNode node = tvTokens.Nodes.Add(t.TokenName);
node.Tag = t;
tokens.Add(t);
tvTokens.SelectedNode = node;
if (tokens.Count == 1)
{
txtTokenID.Enabled = editor.Enabled = btnDelete.Enabled = true;
}
txtTokenID.Focus();
}
/// <summary>
/// Comprueba una regla de palabra reservada
/// </summary>
private void CheckRuleWord(RuleWord rule, TokenCollection tokens, ref bool found)
{
foreach (string word in rule.Words)
{
if (!found && CharSeparator.LookAtChar(word.Length).Equals(word, StringComparison.CurrentCultureIgnoreCase))
{
if (rule.ToFirstSpace && CharSeparator.CheckIsSpace(CharSeparator.LookAtChars(word.Length, 1)))
{
// Añade el token
tokens.Add(GetToken(rule, word, true));
// Indica que se ha encontrado
found = true;
}
}
}
}
//=====================================================================
/// <summary>
/// Add a new token to the collection
/// </summary>
/// <param name="sender">The sender of the event</param>
/// <param name="e">The event arguments</param>
private void cmdAddItem_Executed(object sender, ExecutedRoutedEventArgs e)
{
if (tokens != null)
{
Token t = new Token {
TokenValue = "Add token content here"
};
tokens.Add(t);
t.IsSelected = true;
lbTokens.ScrollIntoView(t);
txtTokenName.SelectAll();
txtTokenValue.SelectAll();
txtTokenName.Focus();
}
}
protected static TokenCollection Create(string[] pTokens)
{
TokenCollection a = new TokenCollection();
foreach (string token in pTokens)
{
a.Add(token);
}
Assert.AreEqual(pTokens.Length, a.Sum);
Assert.AreEqual(pTokens.Distinct().Count(), a.Count);
foreach (string token in pTokens)
{
Assert.AreNotEqual(0, a.get(token));
}
return(a);
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc != null
&& parser != null
&& tc.Count > 0
&& Common.ParserHelpers.IsEqual(Field.KEY_FOREACH, tc.First.Text))
{
if (tc.Count > 5
&& tc[1].TokenKind == TokenKind.LeftParentheses
&& tc[2].TokenKind == TokenKind.TextData
&& Common.ParserHelpers.IsEqual(tc[3].Text, Field.KEY_IN)
&& tc.Last.TokenKind == TokenKind.RightParentheses)
{
ForeachTag tag = new ForeachTag();
tag.Name = tc[2].Text;
TokenCollection coll = new TokenCollection();
coll.Add(tc, 4, tc.Count - 2);
tag.Source = parser.Read(coll);
while (parser.MoveNext())
{
tag.Children.Add(parser.Current);
if (parser.Current is EndTag)
{
return tag;
}
}
throw new Exception.ParseException(String.Concat("foreach is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
else
{
throw new Exception.ParseException(String.Concat("syntax error near foreach:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
}
return null;
}
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc.Count > 2
&& tc.First.TokenKind == TokenKind.TextData
&& HasDot(tc))
{
ReferenceTag tag = new ReferenceTag();
Int32 start, end, pos;
start = end = pos = 0;
for (Int32 i = 0; i < tc.Count; i++)
{
end = i;
switch (tc[i].TokenKind)
{
case TokenKind.Dot:
if (pos == 0)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
tag.AddChild(parser.Read(coll));
start = i + 1;
}
break;
default:
if (tc[i].TokenKind == TokenKind.LeftParentheses)
{
pos++;
}
else if (tc[i].TokenKind == TokenKind.RightParentheses)
{
pos--;
}
if (i == tc.Count - 1)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end);
tag.AddChild(parser.Read(coll));
}
break;
}
}
if (tag.Children.Count > 0)
{
if (tag.Children.Count == 1)
{
return tag.Children[0];
}
}
return tag;
}
return null;
}
private TokenCollection BuildSentences(TokenCollection tokens)
{
var compressedTokens = new TokenCollection();
for (int i = 0; i < tokens.Count; i++)
{
var token = tokens[i];
if (token.Type == TokenType.Quote)
{
StringBuilder sentence = new StringBuilder();
for (int j = i + 1; j < tokens.Count; j++)
{
if (tokens[j].Type == TokenType.Quote)
{
i = j; // Skip to after the quote.
break;
}
if (tokens[j].Type != TokenType.Word)
throw new Exception();
if (j > i + 1)
sentence.Append(" ");
sentence.Append(tokens[j].Text);
}
compressedTokens.Add(new Token(sentence.ToString()));
}
else
{
compressedTokens.Add(token);
}
}
return compressedTokens;
}
/// <summary>
/// Create tokens out of a raw search query.
/// </summary>
/// <param name="input"></param>
/// <returns></returns>
public TokenCollection Tokenize(string input)
{
var tokens = new TokenCollection();
if (!string.IsNullOrEmpty(input))
{
StateFlags state = StateFlags.None;
string[] words = input.Split(' ', '\t');
for (int i = 0; i < words.Length; i++)
{
string word = words[i];
int addClosingParen = 0;
bool addClosingQuote = false;
while (word[0] == '(')
{
tokens.Add(new Token(TokenType.ParenthesisOpen));
word = word.Substring(1);
}
if (word.Length == 0)
continue;
if (word[0] == '"')
{
if ((state & StateFlags.IsInsideQuotes) != 0)
throw new Exception();
tokens.Add(new Token(TokenType.Quote));
state |= StateFlags.IsInsideQuotes;
word = word.Substring(1);
}
if (word.Length == 0)
continue;
while (word[word.Length - 1] == ')')
{
++addClosingParen;
word = word.Substring(0, word.Length - 1);
}
if (word.Length == 0)
{
while (addClosingParen-- > 0)
tokens.Add(new Token(TokenType.ParenthesisClose));
continue;
}
if (word[word.Length - 1] == '"')
{
addClosingQuote = true;
word = word.Substring(0, word.Length - 1);
}
if (word.Length == 0)
{
if (addClosingQuote)
tokens.Add(new Token(TokenType.Quote));
while (addClosingParen-- > 0)
tokens.Add(new Token(TokenType.ParenthesisClose));
}
int semiColonIndex = word.IndexOf(':');
if (semiColonIndex < 0 || (state & StateFlags.IsInsideQuotes) != 0)
{
if ((state & StateFlags.IsInsideQuotes) == 0 && word.Equals("AND", StringComparison.OrdinalIgnoreCase))
tokens.Add(new Token(TokenType.BooleanAnd));
else if ((state & StateFlags.IsInsideQuotes) == 0 && word.Equals("OR", StringComparison.OrdinalIgnoreCase))
tokens.Add(new Token(TokenType.BooleanOr));
else if ((state & StateFlags.IsInsideQuotes) == 0 && word.Equals("NOT", StringComparison.OrdinalIgnoreCase))
tokens.Add(new Token(TokenType.UnaryNot));
else
tokens.Add(new Token(word));
}
else
{
string operatorName = word.Substring(0, semiColonIndex);
tokens.Add(new Token(TokenType.Operator, operatorName));
string operatorArgumentStart = word.Substring(semiColonIndex + 1);
if (operatorArgumentStart[0] == '"')
{
tokens.Add(new Token(TokenType.Quote));
state |= StateFlags.IsInsideQuotes;
operatorArgumentStart = operatorArgumentStart.Substring(1);
}
tokens.Add(new Token(operatorArgumentStart));
}
if (addClosingQuote)
{
if ((state & StateFlags.IsInsideQuotes) == 0)
throw new Exception();
tokens.Add(new Token(TokenType.Quote));
state &= ~StateFlags.IsInsideQuotes;
}
while (addClosingParen-- > 0)
tokens.Add(new Token(TokenType.ParenthesisClose));
}
}
return BuildSentences(tokens);
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc == null
|| parser == null)
{
return null;
}
//支持写法：简写格式：
//常规格式：
if (tc.Count > 5
&& Common.ParserHelpers.IsEqual(tc.First.Text, Field.KEY_SET)
&& tc[1].TokenKind == TokenKind.LeftParentheses
&& tc[3].Text == "="
&& tc.Last.TokenKind == TokenKind.RightParentheses)
{
SetTag tag = new SetTag();
tag.Name = tc[2].Text;
TokenCollection coll = new TokenCollection();
coll.Add(tc, 4, tc.Count - 2);
tag.Value = parser.Read(coll);
return tag;
}
if (tc.Count == 2
&& tc.First.TokenKind == TokenKind.TextData
&& tc.Last.TokenKind == TokenKind.Operator
&& (tc.Last.Text == "++" || tc.Last.Text == "--"))
{
SetTag tag = new SetTag();
tag.Name = tc.First.Text;
ExpressionTag c = new ExpressionTag();
c.AddChild(new VariableTag()
{
FirstToken = tc.First,
Name = tc.First.Text
});
c.AddChild(new TextTag()
{
FirstToken = new Token(TokenKind.Operator, tc.Last.Text[0].ToString())
});
c.AddChild(new NumberTag()
{
Value = 1,
FirstToken = new Token(TokenKind.Number, "1")
});
tag.Value = c;
return tag;
}
if (tc.Count > 2
&& tc.First.TokenKind == TokenKind.TextData
&& tc[1].Text == "=")
{
SetTag tag = new SetTag();
tag.Name = tc.First.Text;
TokenCollection coll = new TokenCollection();
coll.Add(tc, 2, tc.Count - 1);
tag.Value = parser.Read(coll);
return tag;
}
return null;
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(JinianNet.JNTemplate.Parser.TemplateParser parser, TokenCollection tc)
{
if (tc != null
&& parser != null
&& tc.Count > 2)
{
Int32 start, end, pos;
start = end = pos = 0;
Boolean isFunc = false;
List<Token> data = new List<Token>();
Queue<TokenCollection> queue = new Queue<TokenCollection>();
for (Int32 i = 0; i < tc.Count; i++)
{
end = i;
if (tc[i].TokenKind == TokenKind.LeftParentheses)
{
if (pos == 0)
{
if (i > 0 && tc[i - 1].TokenKind == TokenKind.TextData)
{
isFunc = true;
}
}
pos++;
}
else if (tc[i].TokenKind == TokenKind.RightParentheses)
{
if (pos > 0)
{
pos--;
}
else
{
throw new Exception.ParseException(String.Concat("syntax error near ):", tc), data[i].BeginLine, data[i].BeginColumn);
}
if (pos == 0)
{
TokenCollection coll = new TokenCollection();
if (!isFunc)
{
coll.Add(tc, start + 1, end - 1);
}
else
{
coll.Add(tc, start, end);
}
queue.Enqueue(coll);
data.Add(null);
start = i + 1;
//tag.AddChild(parser.Read(coll));
}
}
else if (pos == 0 && (tc[i].TokenKind == TokenKind.Dot || tc[i].TokenKind == TokenKind.Operator))
{
if (end > start)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
queue.Enqueue(coll);
data.Add(null);
}
start = i + 1;
data.Add(tc[i]);
}
if (i == tc.Count - 1 && end >= start)
{
if (start == 0 && end == i)
{
throw new Exception.ParseException(String.Concat("Unexpected tag:", tc), tc[0].BeginLine, tc[0].BeginColumn);
}
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end);
queue.Enqueue(coll);
data.Add(null);
start = i + 1;
}
}
List<Tag> tags = new List<Tag>();
for (Int32 i = 0; i < data.Count; i++)
{
if (data[i] == null)
{
tags.Add(parser.Read(queue.Dequeue()));
}
else if (data[i].TokenKind == TokenKind.Dot)
{
if (tags.Count == 0 || i == data.Count - 1 || data[i + 1] != null)
{
throw new Exception.ParseException(String.Concat("syntax error near .:", tc), data[i].BeginLine, data[i].BeginColumn);
}
if (tags[tags.Count - 1] is ReferenceTag)
{
tags[tags.Count - 1].AddChild(parser.Read(queue.Dequeue()));
}
else
{
ReferenceTag t = new ReferenceTag();
t.AddChild(tags[tags.Count - 1]);
t.AddChild(parser.Read(queue.Dequeue()));
tags[tags.Count - 1] = t;
}
i++;
}
else if (data[i].TokenKind == TokenKind.Operator)
{
tags.Add(new TextTag());
tags[tags.Count - 1].FirstToken = data[i];
}
}
if (tags.Count == 1)
{
return tags[0];
}
if (tags.Count > 1)
{
ExpressionTag t = new ExpressionTag();
for (Int32 i = 0; i < tags.Count; i++)
{
t.AddChild(tags[i]);
}
tags.Clear();
return t;
}
}
return null;
}
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc.Count > 2 && HasOperator(tc))
{
Int32 start, end, pos;
start = end = pos = 0;
#region 去括号
//(8+2) ==》 8+2
//(8+2) * (10-5) ==》(8+2) * (10-5)
if (tc.First.TokenKind == TokenKind.LeftParentheses && tc.Last.TokenKind == TokenKind.RightParentheses)
{
for (Int32 i = 1; i < tc.Count - 1; i++)
{
switch (tc[i].TokenKind)
{
case TokenKind.LeftParentheses:
pos++;
break;
case TokenKind.RightParentheses:
if (pos > 0)
{
pos--;
}
break;
}
}
if (pos == 0)
{
tc = new TokenCollection(tc, 1, tc.Count - 2);
}
else
{
pos = 0;
}
}
#endregion
ExpressionTag tag = new ExpressionTag();
#region 执行表达式折分
for (Int32 i = 0; i < tc.Count; i++)
{
end = i;
switch (tc[i].TokenKind)
{
case TokenKind.Operator:
if (pos == 0)
{
if (start != end)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
tag.AddChild(parser.Read(coll));
}
tag.AddChild(new TextTag());
tag.Children[tag.Children.Count - 1].FirstToken = tc[i];
start = i + 1;
}
break;
default:
if (tc[i].TokenKind == TokenKind.LeftParentheses)
{
pos++;
}
else if (tc[i].TokenKind == TokenKind.RightParentheses)
{
pos--;
}
if (i == tc.Count - 1)
{
TokenCollection coll = new TokenCollection();
if (tc[start].TokenKind == TokenKind.RightParentheses)
{
coll.Add(tc, start + 1, end - 1);
}
else
{
coll.Add(tc, start, end);
}
start = i + 1;
if (coll.Count > 0)
{
tag.AddChild(parser.Read(coll));
}
}
break;
}
}
#endregion
if (tag.Children.Count > 0)
{
if (tag.Children.Count == 1)
{
return tag.Children[0];
}
return tag;
}
}
return null;
}
/// <summary>
/// This converts the token entries to a token file and adds it to
/// the project.
/// </summary>
private void CreateTokenFile()
{
XmlReader xr = converter.Reader;
StreamWriter sw = null;
string tokenFile = Path.Combine(converter.ProjectFolder,
Path.GetFileNameWithoutExtension(converter.Project.Filename) + ".tokens");
// Create an empty token file
try
{
sw = File.CreateText(tokenFile);
sw.WriteLine("<content/>");
}
finally
{
if(sw != null)
sw.Close();
}
FileItem fileItem = converter.Project.AddFileToProject(tokenFile, tokenFile);
TokenCollection tokens = new TokenCollection(fileItem.FullPath);
while(!xr.EOF && xr.NodeType != XmlNodeType.EndElement)
{
if(xr.NodeType == XmlNodeType.Element && xr.Name == "token")
tokens.Add(new Token(xr.GetAttribute("name"), xr.GetAttribute("value")));
xr.Read();
}
tokens.Save();
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc != null
&& parser != null
&& tc.First.TokenKind == TokenKind.TextData
&& tc.Count > 2
&& (tc[1].TokenKind == TokenKind.LeftParentheses)
&& tc.Last.TokenKind == TokenKind.RightParentheses)
{
FunctaionTag tag = new FunctaionTag();
tag.Name = tc.First.Text;
Int32 pos = 0,
start = 2,
end;
for (Int32 i = 2; i < tc.Count; i++)
{
end = i;
switch (tc[i].TokenKind)
{
case TokenKind.Comma:
if (pos == 0)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
if (coll.Count > 0)
{
tag.AddChild(parser.Read(coll));
}
start = i + 1;
}
break;
default:
if (tc[i].TokenKind == TokenKind.LeftParentheses)
{
pos++;
}
else if (tc[i].TokenKind == TokenKind.RightParentheses)
{
pos--;
}
if (i == tc.Count - 1)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
if (coll.Count > 0)
{
tag.AddChild(parser.Read(coll));
}
}
break;
}
}
return tag;
}
return null;
}
/// <summary>
/// 分析标签
/// </summary>
/// <param name="parser">TemplateParser</param>
/// <param name="tc">Token集合</param>
/// <returns></returns>
public Tag Parse(TemplateParser parser, TokenCollection tc)
{
if (tc != null
&& parser != null
&& tc.Count > 3
&& Common.Utility.IsEqual(Field.KEY_FOR, tc.First.Text))
{
if (tc[1].TokenKind == TokenKind.LeftParentheses
&& tc.Last.TokenKind == TokenKind.RightParentheses)
{
Int32 pos = 0,
start = 2,
end;
List<Tag> ts = new List<Tag>(3);
ForTag tag = new ForTag();
for (Int32 i = 2; i < tc.Count - 1; i++)
{
end = i;
if (tc[i].TokenKind == TokenKind.Punctuation && tc[i].Text == ";")
{
if (pos == 0)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end - 1);
if (coll.Count > 0)
{
ts.Add(parser.Read(coll));
}
else
{
ts.Add(null);
}
start = i + 1;
continue;
}
}
if (tc[i].TokenKind == TokenKind.LeftParentheses)
{
pos++;
}
else if (tc[i].TokenKind == TokenKind.RightParentheses)
{
pos--;
}
if (i == tc.Count - 2)
{
TokenCollection coll = new TokenCollection();
coll.Add(tc, start, end);
if (coll.Count > 0)
{
ts.Add(parser.Read(coll));
}
else
{
ts.Add(null);
}
}
}
if (ts.Count != 3)
{
throw new Exception.ParseException(String.Concat("syntax error near for:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
tag.Initial = ts[0];
tag.Test = ts[1];
tag.Do = ts[2];
while (parser.MoveNext())
{
tag.Children.Add(parser.Current);
if (parser.Current is EndTag)
{
return tag;
}
}
throw new Exception.ParseException(String.Concat("for is not properly closed by a end tag:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
else
{
throw new Exception.ParseException(String.Concat("syntax error near for:", tc), tc.First.BeginLine, tc.First.BeginColumn);
}
}
return null;
}
