public void ForumlaTest(string data, bool shouldPass)
{
var parser = new LexicalParser <TokenType>(_forumalRules);
ParserResult result = parser.Parse(data);
result.Should().NotBeNull();
result.IsSuccess.Should().Be(shouldPass);
}
public void RequiredSequenceTest(string data, int ruleNumber, bool shouldPass)
{
var parser = new LexicalParser <TokenType>(_Rules[ruleNumber]);
ParserResult result = parser.Parse(data);
result.Should().NotBeNull();
result.IsSuccess.Should().Be(shouldPass);
}
public void LetNodeSingleFailTests(string data)
{
var parser = new LexicalParser <TokenType>(_onlyPlusRules);
ParserResult result = parser.Parse(data);
result.Should().NotBeNull();
result.IsSuccess.Should().BeFalse();
}
public ExpressionLanguageAST Parser(string expression)
{
LexicalParser l = new LexicalParser();
l.SetParseContent(expression);
SemanticParser s = new SemanticParser();
s.SetParseContent(l.Parse());
return s.Parse();
}
/// <summary>
/// The most general way to parse a data structure using the math evaluation engine.
/// Can return an equation, a single expression, or a system of equations.
/// </summary>
/// <param name="str"></param>
/// <param name="evalData"></param>
/// <returns></returns>
private static List <EqSet> GenCreate(string str, EvalData evalData)
{
LexicalParser lexParser = new LexicalParser(evalData);
List <List <TypePair <LexemeType, string> > > lexemeTables;
List <string> pParseErrors = new List <string>();
// This will contain an array of the expressions entered by the user.
// This generic parse function works for equations and system of equations
// which is why a list of equation sides is returned.
List <EqSet> sides = lexParser.ParseInput(str, out lexemeTables, ref pParseErrors);
return(sides);
}
public void NoClassNameFailTest()
{
IEnumerable <string> data = new List <string>
{
"public class",
"{",
" public static string Cleanse(string name)",
"}",
};
var parser = new LexicalParser <TokenType>(_rules);
ParserResult result = parser.Parse(string.Join(" ", data));
result.Should().NotBeNull();
result.IsSuccess.Should().BeFalse();
}
public JoinPointDefination(Position position, string advisorName, string pattern)
{
_position = position;
_advisorName = advisorName;
_pattern = pattern;
LexicalParser lp = new LexicalParser();
lp.SetParseContent(_pattern);
var tokens = lp.Parse();
try
{
Init(tokens);
}
catch (Exception ex)
{
ExceptionHelper.ThrowWeaveSyntaxInitialize(_pattern, ex);
}
}
private static Dictionary <int, IEnumerable <string> > GetLexemes()
{
int lineNumber = 0;
var lexemes = new Dictionary <int, IEnumerable <string> >();
foreach (var line in Content)
{
var lineLexemes = LexicalParser.GetLexemes(line);
if (lineLexemes.Any())
{
lexemes.Add(++lineNumber, lineLexemes);
}
else
{
lineNumber++;
}
}
return(lexemes);
}
public void NoStaticTest()
{
IEnumerable <string> data = new List <string>
{
"public class TestClass",
"{",
" public static string Cleanse(string name)",
"}",
};
var parser = new LexicalParser <TokenType>(_rules);
ParserResult result = parser.Parse(string.Join(" ", data));
result.Should().NotBeNull();
result.IsSuccess.Should().BeTrue();
result.RootNode.OfType <Symbol <TokenType> >().Any(x => Language.SymStatic.Equals(x)).Should().BeFalse();
result.RootNode.OfType <Expression <TokenType> >().Single(x => Language.ClassName.Equals(x)).Value.Should().Be("TestClass");
result.RootNode.OfType <Expression <TokenType> >().Any(x => Language.SubclassName.Equals(x)).Should().BeFalse();
}
public void LetDifferentOpsValues(string data, string[] testValues)
{
var parser = new LexicalParser <TokenType>(_Rules);
ParserResult result = parser.Parse(data);
result.Should().NotBeNull();
result.IsSuccess.Should().BeTrue();
result.RootNode.OfType <Expression <TokenType> >().Single(x => Language.VariableName.Equals(x)).Value.Should().Be("MyVariable");
result.RootNode.OfType <Expression <TokenType> >().Single(x => Language.VariableType.Equals(x)).Value.Should().Be("double");
var values = result.RootNode.OfType <Expression <TokenType> >()
.Where(x => Language.Variable.Equals(x));
values.Count().Should().Be(testValues.Length);
values.Select(x => x.Value).OrderBy(x => x)
.Zip(testValues.OrderBy(x => x), (o, i) => new { o, i })
.All(x => x.o == x.i)
.Should().BeTrue();
}
static double ExecuteExpression(string expression, Dictionary <string, NumberType> variables, out IList <Token> lexics)
{
lexics = LexicalParser.Parse(expression).ToList();
SyntaxTree syntaxTree = SyntaxParser.Parse(lexics);
DynamicMethod method = ArithmeticTreeCompiler.Compile(syntaxTree);
var undefinedVariables = syntaxTree.VariableNames.Except(variables.Keys).ToList();
if (undefinedVariables.Count != 0)
{
throw new ArgumentException($"Undefined variables found: {string.Join(", ", undefinedVariables)}");
}
object[] args =
syntaxTree.VariableNames
.OrderBy(i => i)
.Select(i => variables[i])
.Cast <object>()
.ToArray();
return((double)method.Invoke(null, args));
}
private QueryNode ParseData(IEnumerable <string> data, bool shouldPass)
{
string rawData = string.Join(" ", data);
var parser = new LexicalParser <TokenType>(_rules);
ParserResult result = parser.Parse(rawData);
result.IsSuccess.Should().Be(shouldPass, result?.LastGood?.ToString() ?? "<no good last>");
if (!result.IsSuccess)
{
return(null);
}
// ====================================================================================
string rowsetName = result.RootNode
.OfType <Expression <TokenType> >()
.FirstOrDefault(x => Language.Rowset.Equals(x))
?.Value;
// ====================================================================================
IEnumerable <Rowset> rowsetList = result.RootNode
.OfType <Expression <TokenType> >()
.Where(x => Language.RefRowset.Equals(x) || Language.RefRowsetAlias.Equals(x))
.Partition((l, x) => !Language.RefRowsetAlias.Equals(x))
.Select(x => new Rowset(x));
// ====================================================================================
IEnumerable <ColumnName> columnList = result.RootNode
.Where(x => Language.ColumnReference.Equals(x) || Language.SymAs.Equals(x) || Language.ColumnName.Equals(x))
.Partition((l, x) => Language.ColumnName.Equals(l.LastOrDefault()))
.Select(x => new ColumnName(x));
IEnumerable <Join> joinList = result.RootNode
.Where(x => Language.JoinReference.Equals(x))
.Partition(2)
.Select(x => new Join(x));
return(new QueryNode(rowsetName, rowsetList, columnList, joinList));
}
public void LetNodeSingleVariableTest()
{
IEnumerable <string> data = new List <string>
{
"let EmptyLong long? = null;",
};
var parser = new LexicalParser <TokenType>(_onlyPlusRules);
ParserResult result = parser.Parse(string.Join(" ", data));
result.Should().NotBeNull();
result.IsSuccess.Should().BeTrue();
result.RootNode.OfType <Expression <TokenType> >().Single(x => Language.VariableName.Equals(x)).Value.Should().Be("EmptyLong");
result.RootNode.OfType <Expression <TokenType> >().Single(x => Language.VariableType.Equals(x)).Value.Should().Be("long?");
var values = result.RootNode.OfType <Expression <TokenType> >()
.Where(x => Language.Variable.Equals(x));
values.Count().Should().Be(1);
values.First().Value.Should().Be("null");
}
public void FullClassTest()
{
IEnumerable <string> data = new List <string>
{
"public static class NameCleaner",
"{",
" public static string Cleanse(string name)",
" {",
" Dictionary<string, string> changes = new Dictionary<string, string>();",
"",
" changes.Add(\" Twp\", \" Township\");",
" changes.Add(\", Town of\", \"\");",
" changes.Add(\" Town of\", \"\");",
" changes.Add(\", Village of\", \"\");",
"",
" foreach (KeyValuePair<string, string> kvp in changes)",
" {",
" if (true == name.EndsWith(kvp.Key))",
" {",
" name = name.Substring(0, name.Length - kvp.Key.Length) + kvp.Value;",
" }",
" }",
"",
" return name;",
" }",
"}",
};
var parser = new LexicalParser <TokenType>(_rules);
ParserResult result = parser.Parse(string.Join(" ", data));
result.Should().NotBeNull();
result.IsSuccess.Should().BeTrue();
result.RootNode.OfType <Symbol <TokenType> >().Any(x => Language.SymStatic.Equals(x)).Should().BeTrue();
result.RootNode.OfType <Expression <TokenType> >().Single(x => Language.ClassName.Equals(x)).Value.Should().Be("NameCleaner");
result.RootNode.OfType <Expression <TokenType> >().Any(x => Language.SubclassName.Equals(x)).Should().BeFalse();
}
public void AstSingleRuleTest()
{
var rowset = new Expression <TokenType>(TokenType.Rowset);
var variable = new Expression <TokenType>(TokenType.Variable);
var equals = new Symbol <TokenType>(TokenType.Equal, "=");
var sstream = new Symbol <TokenType>(TokenType.SStream, "SSTREAM");
var semiColon = new Symbol <TokenType>(TokenType.SemiColon, ";");
var rules = new ParserProductionRules <TokenType>()
{
new RootNode() + rowset + equals + sstream + variable + semiColon,
} +_space;
RootNode tree = new LexicalParser <TokenType>(rules)
.Parse("RdfCountryOrig = SSTREAM @RdfCountrySS;")
?.RootNode;
tree.Should().NotBeNull();
var check = new RootNode
{
new Expression <TokenType>(TokenType.Rowset, "RdfCountryOrig"),
equals,
sstream,
new Expression <TokenType>(TokenType.Variable, "@RdfCountrySS"),
semiColon,
};
tree.Count.Should().Be(check.Count);
var checkList = tree
.Zip(check, (o, i) => new { o, i })
.ToList();
checkList.All(x => x.o.Equals(x.i)).Should().BeTrue();
}
public TemplateAST Parse(string path)
{
string templateFile = Path.Combine(_engine.Path, path);
_log.Debug("Begin parse template file [{0}].", templateFile);
if (!File.Exists(templateFile))
ExceptionHelper.ThrowFileNotFound(templateFile);
using (StreamReader r = new StreamReader(templateFile))
{
string template = r.ReadToEnd();
LexicalParser lp = new LexicalParser();
lp.SetParseContent(template);
SemanticParser sp = new SemanticParser();
sp.SetParseContent(lp.Parse());
TemplateAST ast = sp.Parse();
_log.Debug("Parse template file [{0}] success.", templateFile);
ParseIncludeTemplate(ast);
return ast;
}
}
/// <summary>
/// The most general way to parse a data structure using the math evaluation engine.
/// Can return an equation, a single expression, or a system of equations.
/// </summary>
/// <param name="str"></param>
/// <param name="evalData"></param>
/// <returns></returns>
private static List<EqSet> GenCreate(string str, EvalData evalData)
{
LexicalParser lexParser = new LexicalParser(evalData);
List<List<TypePair<LexemeType, string>>> lexemeTables;
List<string> pParseErrors = new List<string>();
// This will contain an array of the expressions entered by the user.
// This generic parse function works for equations and system of equations
// which is why a list of equation sides is returned.
List<EqSet> sides = lexParser.ParseInput(str, out lexemeTables, ref pParseErrors);
return sides;
}
public void AstRepeatRuleTest()
{
var equal = new Symbol <TokenType>(TokenType.Equal, "=");
var semiColon = new Symbol <TokenType>(TokenType.SemiColon, ";");
var declare = new Symbol <TokenType>(TokenType.Declare, "#DECLARE");
var plus = new Symbol <TokenType>(TokenType.Plus, "+");
var variable = new Expression <TokenType>(TokenType.Variable);
var variableType = new Expression <TokenType>(TokenType.VariableType);
var value = new Expression <TokenType>(TokenType.Value);
var repeatPlusValue = new Optional() + (new Repeat() + plus + value);
var rules = new ParserProductionRules <TokenType>()
{
new RootNode() + declare + variable + variableType + equal + value + repeatPlusValue + semiColon,
} +_space;
var variations = new[]
{
new {
RawData = "#DECLARE EntityTSV string = @@OutputPath@@ + ;",
Result = (RootNode)null,
},
new {
RawData = "#DECLARE EntityTSV string = @@OutputPath@@;",
Result = new RootNode {
declare,
new Expression <TokenType>(TokenType.Variable, "EntityTSV"),
new Expression <TokenType>(TokenType.VariableType, "string"),
equal,
new Expression <TokenType>(TokenType.Value, "@@OutputPath@@"),
semiColon,
}
},
new {
RawData = "#DECLARE EntityTSV string = @@OutputPath@@ + \"Entity.tsv\";",
Result = new RootNode {
declare,
new Expression <TokenType>(TokenType.Variable, "EntityTSV"),
new Expression <TokenType>(TokenType.VariableType, "string"),
equal,
new Expression <TokenType>(TokenType.Value, "@@OutputPath@@"),
plus,
new Expression <TokenType>(TokenType.Value, "\"Entity.tsv\""),
semiColon,
}
},
new {
RawData = "#DECLARE EntityTSV string = @@OutputPath@@ + \"Entity.tsv\" +;",
Result = (RootNode)null,
},
new {
RawData = "#DECLARE EntityTSV string = @@OutputPath@@ + \"Entity.tsv\" + \"/testPath\";",
Result = new RootNode {
declare,
new Expression <TokenType>(TokenType.Variable, "EntityTSV"),
new Expression <TokenType>(TokenType.VariableType, "string"),
equal,
new Expression <TokenType>(TokenType.Value, "@@OutputPath@@"),
plus,
new Expression <TokenType>(TokenType.Value, "\"Entity.tsv\""),
plus,
new Expression <TokenType>(TokenType.Value, "\"/testPath\""),
semiColon,
}
},
};
foreach (var test in variations)
{
RootNode tree = new LexicalParser <TokenType>(rules)
.Parse(test.RawData)
?.RootNode;
if (test.Result == null)
{
tree.Should().BeNull();
continue;
}
tree.Should().NotBeNull(test.RawData);
tree.Count.Should().Be(test.Result.Count);
var checkList = tree
.Zip(test.Result, (o, i) => new { o, i })
.ToList();
checkList.All(x => x.o.Equals(x.i)).Should().BeTrue();
}
}
public void AstOptionalRuleTest()
{
var symEqual = new Symbol <TokenType>(TokenType.Equal, "=");
var symSemiColon = new Symbol <TokenType>(TokenType.SemiColon, ";");
var symSelect = new Symbol <TokenType>(TokenType.Select, "SELECT");
var symAs = new Symbol <TokenType>(TokenType.As, "AS");
var symComma = new Symbol <TokenType>(TokenType.Comma, ",");
var symFrom = new Symbol <TokenType>(TokenType.From, "FROM");
var symWhere = new Symbol <TokenType>(TokenType.Where, "WHERE");
var rowset = new Expression <TokenType>(TokenType.Rowset);
var variable = new Expression <TokenType>(TokenType.Variable);
var referenceColumnName = new Expression <TokenType>(TokenType.ReferenceColumnName);
var columnName = new Expression <TokenType>(TokenType.ColumnName);
var optionalAs = new Optional() + referenceColumnName + symAs;
var repeatColumns = new Repeat() + symComma + optionalAs + columnName;
var rules = new ParserProductionRules <TokenType>()
{
new RootNode() + rowset + symEqual + symSelect + optionalAs + columnName + repeatColumns + symFrom + rowset + symSemiColon,
} +_space;
var variations = new[]
{
new {
RawData = new List <string> {
"a = SELECT FROM;"
},
Result = (RootNode)null,
},
new {
RawData = new List <string> {
"RdfCartoLink =",
" SELECT carto_id.ToString() AS carto_id,",
" link_id,",
" long_haul,",
" coverage_indicator,",
" line_of_control,",
" claimed_by,",
" controlled_by,",
" expanded_inclusion",
" FROM RdfCartoLinkOrig;",
},
Result = new RootNode {
new Expression <TokenType>(TokenType.Rowset, "RdfCartoLink"),
symEqual,
symSelect,
new Expression <TokenType>(TokenType.ReferenceColumnName, "carto_id.ToString()"),
symAs,
new Expression <TokenType>(TokenType.ColumnName, "carto_id"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "link_id"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "long_haul"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "coverage_indicator"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "line_of_control"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "claimed_by"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "controlled_by"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "expanded_inclusion"),
symFrom,
new Expression <TokenType>(TokenType.Rowset, "RdfCartoLinkOrig"),
symSemiColon,
}
},
new {
RawData = new List <string> {
"RdfLink =",
" SELECT link_id,",
" ref_node_id,",
" nonref_node_id,",
" left_admin_place_id.ToString() AS left_admin_place_id,",
" right_admin_place_id.ToString() AS right_admin_place_id,",
" left_postal_area_id,",
" right_postal_area_id,",
" bridge,",
" tunnel,",
" map_edge_link",
" FROM RdfLinkOrig;",
},
Result = new RootNode {
new Expression <TokenType>(TokenType.Rowset, "RdfLink"),
symEqual,
symSelect,
new Expression <TokenType>(TokenType.ColumnName, "link_id"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "ref_node_id"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "nonref_node_id"),
symComma,
new Expression <TokenType>(TokenType.ReferenceColumnName, "left_admin_place_id.ToString()"),
symAs,
new Expression <TokenType>(TokenType.ColumnName, "left_admin_place_id"),
symComma,
new Expression <TokenType>(TokenType.ReferenceColumnName, "right_admin_place_id.ToString()"),
symAs,
new Expression <TokenType>(TokenType.ColumnName, "right_admin_place_id"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "left_postal_area_id"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "right_postal_area_id"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "bridge"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "tunnel"),
symComma,
new Expression <TokenType>(TokenType.ColumnName, "map_edge_link"),
symFrom,
new Expression <TokenType>(TokenType.Rowset, "RdfLinkOrig"),
symSemiColon,
}
},
};
foreach (var test in variations)
{
RootNode tree = new LexicalParser <TokenType>(rules)
.Parse(string.Join(" ", test.RawData))
?.RootNode;
if (test.Result == null)
{
tree.Should().BeNull();
continue;
}
tree.Count.Should().Be(test.Result.Count);
var checkList = tree
.Zip(test.Result, (o, i) => new { o, i })
.ToList();
checkList.All(x => x.o.Equals(x.i)).Should().BeTrue();
}
}
public void AstMultipleRuleTest()
{
var symEqual = new Symbol <TokenType>(TokenType.Equal, "=");
var symSstream = new Symbol <TokenType>(TokenType.SStream, "SSTREAM");
var symReference = new Symbol <TokenType>(TokenType.Reference, "REFERENCE");
var symDeclare = new Symbol <TokenType>(TokenType.Declare, "#DECLARE");
var symSemiColon = new Symbol <TokenType>(TokenType.SemiColon, ";");
var symPlus = new Symbol <TokenType>(TokenType.Plus, "+");
var symUsing = new Symbol <TokenType>(TokenType.Using, "USING");
var rowset = new Expression <TokenType>(TokenType.Rowset);
var variable = new Expression <TokenType>(TokenType.Variable);
var variableType = new Expression <TokenType>(TokenType.VariableType);
var value = new Expression <TokenType>(TokenType.Value);
var rules = new ParserProductionRules <TokenType>()
{
new RootNode()
+ (new Choice()
+ (new RootNode() + rowset + symEqual + symSstream + variable + symSemiColon) // {rowset} = SSTREAM {variable};
+ (new RootNode() + symDeclare + variable + variableType + symEqual + value + symSemiColon) // #DECLARE {variable} int = {value};
+ (new RootNode() + symReference + value + symSemiColon) // REFERENCE {referenceAssembly};
+ (new RootNode() + symUsing + value + symSemiColon) // USING {value};
),
_space,
};
var variations = new[]
{
new {
RawData = "# PoiNamedPlace int = 4444;",
Result = (RootNode)null,
},
new {
RawData = "#DECLARE PoiNamedPlace int = 4444;",
Result = new RootNode {
symDeclare,
new Expression <TokenType>(TokenType.Variable, "PoiNamedPlace"),
new Expression <TokenType>(TokenType.VariableType, "int"),
symEqual,
new Expression <TokenType>(TokenType.Value, "4444"),
symSemiColon,
}
},
new {
RawData = "RdfCountryOrig = SS @RdfCountrySS; ",
Result = (RootNode)null,
},
new {
RawData = "RdfCountryOrig = SSTREAM @RdfCountrySS; ",
Result = new RootNode {
new Expression <TokenType>(TokenType.Rowset, "RdfCountryOrig"),
symEqual,
symSstream,
new Expression <TokenType>(TokenType.Variable, "@RdfCountrySS"),
symSemiColon,
}
},
new {
RawData = "REFERENCE \"System.Data.dll\"",
Result = (RootNode)null,
},
new {
RawData = "REFERENCE \"System.Data.dll\";",
Result = new RootNode {
symReference,
new Expression <TokenType>(TokenType.Value, "\"System.Data.dll\""),
symSemiColon,
}
},
};
int testNumber = -1;
Func <string> why = () => $"TestNumber: {testNumber}";
foreach (var test in variations)
{
testNumber++;
RootNode tree = new LexicalParser <TokenType>(rules)
.Parse(test.RawData)
?.RootNode;
if (test.Result == null)
{
tree.Should().BeNull(why());
continue;
}
tree.Should().NotBeNull(why());
tree.Count.Should().Be(test.Result.Count, why());
var checkList = tree
.Zip(test.Result, (o, i) => new { o, i })
.ToList();
checkList.All(x => x.o.Equals(x.i)).Should().BeTrue(why());
}
}
public string RenderRaw(string textTemplate)
{
textTemplate.ThrowIfNullArgument(nameof(textTemplate));
try
{
LexicalParser lp = new LexicalParser();
lp.SetParseContent(textTemplate);
SemanticParser sp = new SemanticParser();
sp.SetParseContent(lp.Parse());
TemplateAST ast = sp.Parse();
return this.Render(ast);
}
catch (ELParseException)
{
throw;
}
catch (EvalException)
{
throw;
}
catch (TemplateParseException)
{
throw;
}
catch (TemplateRenderException)
{
throw;
}
catch (Exception ex)
{
ExceptionHelper.ThrowWrapped(ex);
return string.Empty;
}
}
