public void VariablesAssignment()
{
ScannerSpecification oSpec = new ScannerSpecification();
oSpec.AddLiteral("l", CharType.Letters, '_');
oSpec.AddLiteral("d", CharType.Numbers);
oSpec.AddLiteral("x", '=', ';');
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddTokenDeclaration("number", "d{d}");
oSpec.AddTokenDeclaration("sep", "x");
StateParser oStateParser = new StateParser(new StateScanner(oSpec, 4));
oStateParser.Spec.AddRule("Tag", "id sep number sep");
{
oStateParser.SetSource("sText=10;");
this.ReadAndAssertToken(oStateParser, "id", "sText");
this.ReadAndAssertToken(oStateParser, "sep", "=");
this.ReadAndAssertToken(oStateParser, "number", "10");
this.ReadAndAssertToken(oStateParser, "sep", ";");
var oToken = oStateParser.GetNextToken();
Assert.IsNull(oToken);
}
}
public void MultiLineTokens()
{
ScannerSpecification oSpec = new ScannerSpecification();
oSpec.AddLiteral("l", CharType.Letters, '_');
oSpec.AddLiteral("d", CharType.Numbers);
oSpec.AddLiteral("br1", '[');
oSpec.AddLiteral("br2", ']');
oSpec.AddTokenDeclaration("id", "l{l|d}");
oSpec.AddBoundedToken("id2", "br1", "br2", null);
StateParser oStateParser = new StateParser(new StateScanner(oSpec, 4));
oStateParser.Spec.AddRule("id", "id|id2");
{
oStateParser.SetSource(
@"select [
x int] from x
");
this.ReadAndAssertToken(oStateParser, "id", "select");
this.ReadAndAssertToken(oStateParser, "id2", "[");
this.ReadAndAssertToken(oStateParser, "id2", " x int]");
}
}
//=========================================================================================
void ReadAndAssertToken(StateParser parser, string tokenType, string text)
{
var oToken = parser.GetNextToken();
Assert.That(parser.Errors, Has.Count);
Assert.That(oToken.Text, Is.EqualTo(text));
Assert.That(oToken.TokenTypeName, Is.EqualTo(tokenType));
}
public JSONParser()
{
typeParser = TypeParser.Instance;
predicateParser = PredicateParser.Instance;
operatorParser = OperatorParser.Instance;
objectParser = ObjectParser.Instance;
initStateParser = StateParser.Instance;
goalStateParser = StateParser.Instance;
}
//=========================================================================================
void ReadTokenWithStyle(StateParser parser, string style, string text)
{
var oToken = parser.GetNextToken();
Assert.That(parser.Errors, Has.Count);
Assert.That(oToken.Text, Is.EqualTo(text));
Assert.AreEqual(oToken.Style.Name, style);
//Assert.That(oToken.TokenTypeName, Is.EqualTo(tokenType));
}
static List <List <string> > Parse(string text)
{
StateParser p1 = null; // 用于解析token的起始字符
StateParser p2 = null; // 用于解析作为分隔符的“-”的下一个字符
StateParser p3 = null; // 用于解析token中或结尾的单引号的下一个字符
StateParser p4 = null; // 用于解析单引号外的token字符
StateParser p5 = null; // 用于解析单引号内的token字符
var currentToken = new StringBuilder();
var currentTokenGroup = new List <string>();
var result = new List <List <string> >();
// 解析token的起始字符
p1 = ch =>
{
if (ch == '-')
{
// 如果token中需要包含单引号或“-”，
// 那么这个token在表示的时候一定需要用一对单引号包裹起来
throw new ArgumentException();
}
if (ch == '\'')
{
// 如果起始字符是单引号，
// 则开始解析单引号内的token字符
return(p5);
}
else
{
// 如果是普通字符，则作为当前token的字符，
// 并开始解析单引号外的token字符
currentToken.Append(ch);
return(p4);
}
};
// 解析作为分隔符的“-”的下一个字符
p2 = ch =>
{
if (ch == '-')
{
// 如果当前字符为“-”，说明一个token group结束了（因为前一个字符也是“-”），
// 则将当前的token group加入结果集，并且准备新的token group
result.Add(currentTokenGroup);
currentTokenGroup = new List <string>();
return(p1);
}
else if (ch == '\'')
{
// 如果当前字符为单引号，则说明新的token以单引号包裹
// 则开始解析单引号内的token字符
return(p5);
}
else
{
// 如果是普通字符，则算作当前token的字符，
// 并继续解析单引号外的token字符
currentToken.Append(ch);
return(p4);
}
};
// 解析token内部或结尾的单引号的下一个字符
p3 = ch =>
{
if (ch == '\'')
{
// 如果当前字符为单引号，则说明连续两个单引号，
// 所以表明token中出现了“单个”单引号，并且当前token一定包裹在单引号内，
// 因此继续解析单引号内的token字符
currentToken.Append('\'');
return(p5);
}
else if (ch == '-')
{
// 如果当前字符为一个分隔符，则说明上一个token已经结束了
// 于是将当前token加入当前token group，准备新的token，
// 并解析分隔符后的下一个字符
currentTokenGroup.Add(currentToken.ToString());
currentToken = new StringBuilder();
return(p2);
}
else
{
// 单引号后面只可能是另一个单引号或者一个分隔符，
// 否则说明输入错误，则抛出异常
throw new ArgumentException();
}
};
// 用于解析单引号外的token字符，
// 即一个没有特殊字符（分隔符或单引号）的token
p4 = ch =>
{
if (ch == '\'')
{
// 如果token中出现了单引号，则抛出异常
throw new ArgumentException();
}
if (ch == '-')
{
// 如果出现了分隔符，则表明当前token结束了，
// 于是将当前token加入当前token group，准备新的token，
// 并解析分隔符的下一个字符
currentTokenGroup.Add(currentToken.ToString());
currentToken = new StringBuilder();
return(p2);
}
else
{
// 对于其他字符，则当作token中的普通字符处理
// 继续解析单引号外的token字符
currentToken.Append(ch);
return(p4);
}
};
// 用于解析单引号内的token字符
p5 = ch =>
{
if (ch == '\'')
{
// 对于被单引号包裹的token内的第一个单引号，
// 需要解析其下一个字符，才能得到其真实含义
return(p3);
}
else
{
// 对于普通字符，则添加到当前token内
currentToken.Append(ch);
return(p5);
}
};
StateParser pn = p1;
foreach (char c in text)
{
pn = pn(c);
}
// text.Aggregate(p1, (sp, ch) => sp(ch));
currentTokenGroup.Add(currentToken.ToString());
result.Add(currentTokenGroup);
return(result);
}
/// <summary>
/// 解析函数
/// 例如:(T(表3.3).R(1,J1,J2).A*J1)+1/J2.Code
/// 产生结果:
/// List<stirng>列表
/// 成员:>表示执行查询操作 table|row|column,#表示需要访问对象属性，J表示取对象本身
/// </summary>
/// <param name="text"></param>
/// <returns></returns>
public List <string> Parser(string text)
{
StateParser p1 = null;
StateParser p2 = null;
StateParser p3 = null;
StateParser p4 = null;
StateParser p5 = null;
var result = new List <string>();
StringBuilder currentToken = new StringBuilder();
SplitElement se = (ch) =>
{
if (currentToken.Length > 0)
{
result.Add(currentToken.ToString());//之前项
}
currentToken = new StringBuilder();
currentToken.Append(ch); //运算符号
result.Add(currentToken.ToString());
currentToken = new StringBuilder(); //新项
};
FormatString fs = () =>
{
string tempAll = currentToken.ToString();
string[] strAlls = tempAll.Split(SPLIT_QUERY_TAG);
string tempR = strAlls[1];//R部分
string[] strRs = tempR.Split(SPLIT_CONDITION_TAG);
string strLast = strRs[strRs.Length - 1];
if (strLast[0] >= 65)
{
//J1==>J1={$J1}
// strLast = string.Format("{0}='{1}'",strLast.StartsWith(""));#【功能扩展】
strLast = strLast + "='{$" + strLast + "}'";
}
else
{
//1==> JN=J1
strLast = "JN='" + strLast + "'";
}
strRs[strRs.Length - 1] = strLast;
strAlls[1] = string.Join(" AND ", strRs);
currentToken = new StringBuilder(string.Join(SPLIT_QUERY_TAG.ToString(), strAlls));
};
//处理项的开头
p1 = ch =>
{
if (ch == POINT_TAG)
{
// new exption;
return(p1);
}
else if (ch == T)
{
currentToken.Append(QUERY_TAG);//查询
return(p2);
}
else if (ch == R)
{
currentToken.Append(SPLIT_QUERY_TAG);
return(p3);
}
else if (ch == J)
{
// currentToken=new StringBuilder();
currentToken.Append(ch);
return(p5);
}
else if (OPERATOR.IndexOf(ch) != -1)
{
se(ch);
return(p1);
}
else if (RIGHT_CLOSE_TAG == ch || LETF_CLOSE_TAG == ch)
{
se(ch);
return(p1);
}
else
{
currentToken.Append(ch);//数字直接插入
return(p1);
}
};
//表解析T(xxx)
p2 = ch =>
{
if (ch == LETF_CLOSE_TAG)
{
return(p2);
}
else
if (ch == RIGHT_CLOSE_TAG)
{
return(p1);
}
else
{
// app(char);
currentToken.Append(ch);
return(p2);
}
};
//R处理
p3 = ch =>
{
if (ch == LETF_CLOSE_TAG)
{
return(p3);
}
else
if (ch == RIGHT_CLOSE_TAG)
{
return(p3);
}
else if (ch == POINT_TAG)
{
fs(); //最后一项格式化
currentToken.Append(SPLIT_QUERY_TAG);
return(p4); //列处理,交给p1,
}
else if (ch == SPLIT_CONDITION_TAG)
{
fs();
currentToken.Append(ch);
return(p3);
}
else
{
currentToken.Append(ch);
return(p3);
}
};
//解析列
p4 = ch =>
{
if (OPERATOR.IndexOf(ch) != -1)
{
se(ch);
return(p1);
}
else if (RIGHT_CLOSE_TAG == ch || LETF_CLOSE_TAG == ch)
{
se(ch);
return(p1);
}
else
{
currentToken.Append(ch);
return(p4);
}
};
//解析变量
p5 = ch =>
{
if (OPERATOR.IndexOf(ch) != -1)
{
se(ch);
return(p1);
}
else if (RIGHT_CLOSE_TAG == ch || LETF_CLOSE_TAG == ch)
{
se(ch);
return(p1);
}
else if (ch == POINT_TAG)
{
currentToken.Insert(0, ATT_ACCESS_TAG);//#表示属性访问
currentToken.Append(ch);
return(p5);
}
else
{
currentToken.Append(ch);
}
return(p5);
};
StateParser pn = p1;
foreach (char ch in text)
{
pn = pn(ch);
}
result.Add(currentToken.ToString());
return(result);
}
//=========================================================================================
public XmlParsingTests()
{
this.Parser = new StateParser(new XmlSyntaxSettings(), 4);
}
//=========================================================================================
public MsSqlParsingTests()
{
this.Parser = new StateParser(new MsSqlSyntaxSettings(), 4);
}
public List <GMapPolygon> GetPolygons()
{
return(StateParser.GetPolygons(db.States));
}
public StateDao(string filepath)
{
db = Db.GetInstance(filepath);
db.Tweets = TweetParser.Parse(filepath, db.Sentiments);
StateParser.GroupTweetsByStates(db.States, db.Tweets);
}
public List <State> GetStates()
{
return(StateParser.ParseStates(fileName));
}
//=========================================================================================
public NonSyntaxSettingsTests()
{
this.Parser = new StateParser(new NonSyntaxSettings(), 4);
}
