static bool TryExtractTimeFormatString(Expr timeExpr, out Expr fieldExpr, out string timeFmtStr)
{
fieldExpr = timeExpr;
timeFmtStr = null;
if (timeExpr.nodeType != ExprType.Call)
{
return(false);
}
Expr field = null;
ConstExpr timeFmt = ConstExpr.Null;
bool withToDate = timeExpr.Traverse(e =>
{
if (e.nodeType != ExprType.Call)
{
return(false);
}
var ce = e as CallExpr;
if (ce == null || ce.args.Count < 2 || string.Compare(ce.funcName, "TO_DATE", StringComparison.OrdinalIgnoreCase) != 0)
{
return(false);
}
field = ce.args[0];
timeFmt = (ConstExpr)ce.args[1];
return(true);
}).Any(f => f);
if (!withToDate)
{
return(false);
}
fieldExpr = field;
timeFmtStr = Convert.ToString(timeFmt.value);
return(true);
}
// Arthmentic Simplification:
/*
* +/- zero to a column, or multiply/divide a column by 1
*/
internal bool IsArithIdentity(ConstExpr lce, ConstExpr rce)
{
ConstExpr ve = lce != null ? lce : rce;
ColExpr ce = (children_[0] is ColExpr) ? (ColExpr)children_[0] : (children_[1] is ColExpr ? (ColExpr)children_[1] : null);
if (ce == null)
{
return(false);
}
if (!(TypeBase.IsNumberType(ve.type_) && TypeBase.IsNumberType(ce.type_)))
{
return(false);
}
if ((op_ == "+" || op_ == "-") && ve.IsZero())
{
return(true);
}
if ((op_ == "*" || op_ == "/") && ve.IsOne())
{
return(true);
}
return(false);
}
public void ElementCreate()
{
Element e = new Element(3.141592);
Assert.IsNotNull(e);
Element p = new Element(100);
Element q = new Element(p);
Element z = new Element(DateTime.Now);
ConstExpr j = new ConstExpr(z);
ConstExpr k = new ConstExpr(j.Value.BoolValue());
ConstExpr v = new ConstExpr(k);
ConstExpr u = new ConstExpr(v);
Element b = new Element(true);
Element i = new Element(100);
Element l = new Element(9812080980101);
Element d = new Element(3.1314);
Element s = new Element("Hello World!");
Element dt = new Element(DateTime.Now);
Element ts = new Element(TimeSpan.FromSeconds(100));
Element r = new Element(new Regex("abc[0-9]"));
Assert.IsTrue(b.DataType == ElementType.BOOL);
Assert.IsTrue(l.DataType == ElementType.LONG);
Assert.IsTrue(d.DataType == ElementType.DOUBLE);
Assert.IsTrue(s.DataType == ElementType.STRING);
Assert.IsTrue(dt.DataType == ElementType.DATETIME);
Assert.IsTrue(ts.DataType == ElementType.TIMESPAN);
Assert.IsTrue(r.DataType == ElementType.REGEX);
}
public override Expression Evaluate(EvalStore store)
{
ConstExpr e = (ConstExpr)this.Arguments[0].Evaluate(store);
if (e.Value.DataType == ElementType.LONG) {
if (e.Value.LongValue() < 0) e = new ConstExpr(-e.Value.LongValue());
}
else if (e.Value.DataType == ElementType.DOUBLE) {
if (e.Value.DoubleValue() < 0) e = new ConstExpr(-e.Value.DoubleValue());
}
return e;
}
public override Expr Normalize()
{
Expr x = base.Normalize();
if (x.AllArgsConst())
{
Value val = Exec(null, null);
return(ConstExpr.MakeConst(val, type_, outputName_));
}
// NOT NOT X => X
if (op_ == "!")
{
if (child_() is UnaryExpr ue && ue.op_ == "!")
{
return(ue.child_());
}
// NOT (X AND Y) => NOT A OR NOT B
// NOT (X OR Y) => NOT A AND NOT B
/*
* NOT OR
* | / \
* AND => NOT NOT
* / \ | |
* X Y X Y
*/
if (child_() is LogicAndOrExpr le)
{
// Make two Unary expressions for NOT X and NOT Y
// Cloning avoids messing with bounded_ flag
UnaryExpr nlu = (UnaryExpr)Clone();
UnaryExpr nru = (UnaryExpr)Clone();
// get the new logical operator, reverse of current one
string nop = le.op_ == " and " ? " or " : " and ";
// save X, Y
Expr ole = le.lchild_();
Expr ore = le.rchild_();
// New Unary nodes point to old left and right
// logical expressions.
nlu.children_[0] = ole;
nru.children_[0] = ore;
return(makeAnyLogicalExpr(nlu, nru, nop));
}
}
return(x);
}
static void AttrsToComments <T>(this IEnumerable <KeyValuePair <T, object> > attrs, TextWriter wr)
{
foreach (var p in attrs)
{
wr.Write("--");
wr.Write(p.Key);
wr.Write('=');
ConstExpr.ToText(wr, p.Value);
wr.WriteLine();
}
}
static void Main(string[] args)
{
Console.WriteLine("1_______________________________________________________________________________");
// (3*x^2 + 7) / (x + 5) 82 / 10 = 8.2
Expression exp = new DivExpr(
new AddExpr(
MonomialExpr.createObject(3, 2),
ConstExpr.createObject(7)),
new AddExpr(
MonomialExpr.createObject(1, 1),
ConstExpr.createObject(5)));
Console.WriteLine("f(x)=" + exp.ToString());
Console.WriteLine("f(5)=" + exp.Evaluate(5));
Console.WriteLine("f'(x)=" + exp.Derive().ToString());
Console.WriteLine("2_______________________________________________________________________________");
// (0 + 10) - 0 * (x^1 + 0)
Expression exp5 = new AddExpr(MonomialExpr.createObject(0, 2),
ConstExpr.createObject(10));
Expression exp6 = new SubExpr(exp5,
new MulExpr(ConstExpr.createObject(0),
new AddExpr(MonomialExpr.createObject(1, 1),
ConstExpr.createObject(0))));
Console.WriteLine("f(x)=" + exp6.ToString());
Console.WriteLine("f(2)=" + exp6.Evaluate(2));
Console.WriteLine("f'(x)=" + exp6.Derive().ToString());
Console.WriteLine("3_______________________________________________________________________________");
//(((x^2 + 2) / (x + 1)) + 10) / (x^2 + 5)
Expression exp3 = new DivExpr(
new AddExpr(
MonomialExpr.createObject(1, 2),
ConstExpr.createObject(2)),
new AddExpr(
MonomialExpr.createObject(1, 1),
ConstExpr.createObject(1)));
Expression exp4 = new DivExpr(
new AddExpr(
exp3,
ConstExpr.createObject(10)),
new AddExpr(
MonomialExpr.createObject(1, 2),
ConstExpr.createObject(5)));
Console.WriteLine("f(x)=" + exp4.ToString());
Console.WriteLine("f(1)=" + exp4.Evaluate(1));
Console.WriteLine("f'(x)=" + exp4.Derive().ToString());
}
public override Expr Normalize()
{
Expr x = base.Normalize();
if (!x.AllArgsConst() || ExternalFunctions.set_.ContainsKey(funcName_))
{
return(this);
}
if (x.AnyArgNull())
{
return(ConstExpr.MakeConst("null", new AnyType(), outputName_));
}
switch (funcName_)
{
case "min":
case "max":
case "avg":
return(child_());
case "sum":
case "count":
case "count(*)":
case "coalesce":
case "tumble":
case "tumble_start":
case "tumble_end":
case "hop":
case "session":
case "stddev_samp":
return(this);
default:
break;
}
Value val = Exec(null, null);
return(ConstExpr.MakeConst(val, type_, outputName_));
}
internal Expr SimplifyArithmetic(ConstExpr lve, ConstExpr rve)
{
// we know we have a BinExpr with numeric children
ConstExpr ve = lve != null ? lve : rve;
ColExpr ce = (children_[0] is ColExpr) ? (ColExpr)children_[0] : (children_[1] is ColExpr ? (ColExpr)children_[1] : null);
if (ce is null || ve is null)
{
return(this);
}
// Col + 0, or Col - 0 => return col
if ((op_ == "+" || op_ == "-") && ve.IsZero())
{
return(ce);
}
if ((op_ == "*" || op_ == "/") && ve.IsOne())
{
return(ce);
}
return(this);
}
IEnumerable <(CellRange cell, Expr expr)> DumpDepsImpl(string forCell)
{
if (already.ContainsKey(forCell))
{
yield break;
}
already.Add(forCell, true);
var rng = CellRange.TryFromName(forCell);
if (rng == null)
{
yield break;
}
Expr expr;
if (sheet.fmls.TryGetValue(rng, out expr))
{
// Make all cell references relative
expr = Expr.RecursiveModifier(XlsxR.RangeRefToRelativeModifier)(expr);
foreach (var s in expr.EnumerateReferences())
{
foreach (var dep in DumpDepsImpl(s))
{
yield return(dep);
}
}
}
else if (!sheet.vals.TryGetValue(rng, out expr))
{
expr = new ConstExpr("??? (value not found)");
}
if (forCell == null)
{
}
yield return(rng, expr);
}
//--------------------------------------------------------------------
#region 内部方法
ExprBase Parse(Queue <string> stringQueue)
{
string word = null;
ExprBase first = null, prev = null, last = null;
while (stringQueue.Count > 0)
{
word = stringQueue.Dequeue();
if (word == "(")
{
ExprBase subExpr = Parse(stringQueue);
if (subExpr == null)
{
return(null);
}
last = new SubExpr(subExpr);
}
else if (word[0] == '\'')
{
int endPos = word.LastIndexOf('\'');
if (endPos > 0)
{
last = new StrExpr(word.Substring(1, endPos - 1));
}
else
{
return(null);
}
}
else if (word[0] == '\"')
{
int endPos = word.LastIndexOf('\"');
if (endPos > 0)
{
last = new StrExpr(word.Substring(1, endPos - 1));
}
else
{
return(null);
}
}
else if (word == "true")
{
last = new ConstExpr(1);
}
else if (word == "false")
{
last = new ConstExpr(0);
}
else if (word[0] == '-' || Char.IsDigit(word[0]))
{
if (word.Contains("."))
{
last = new ConstExpr(float.Parse(word));
}
else
{
last = new ConstExpr(int.Parse(word));
}
}
else
{
last = ParseFunc(word, stringQueue);
}
if (last == null)
{
return(null);
}
if (first == null)
{
first = prev = last;
}
else
{
prev.next = last;
prev = last;
}
// 条件连接关系
if (stringQueue.Count > 0)
{
word = stringQueue.Dequeue();
if (word == ")")
{
break;
}
last.connector = ExprConnectors.Get(word);
if (last.connector == null)
{
return(null);
}
}
last = null;
}
return(first);
}
public Expression GetNode(ExpressionParser parser)
{
var collection = parser.Collection;
Lexem lex = parser.Collection.CurrentLexem();
bool uniar = parser.waitValue;
Expression res = null;
if (lex.LexemType == LexType.Arfimetic)
{
switch (lex.LexemText)
{
case "+":
if (uniar)
{
res = new UniarPlus_BoolExpr();
}
else
{
res = new Plus_Arifmetic();
}
break;
case "-":
if (uniar)
{
res = new UniarMinus_BoolExpr();
}
else
{
res = new Minus_Arifmetic();
}
break;
case "*":
if (uniar)
{
res = new AllColumnExpr();
}
else
{
res = new Multi_Arifmetic();
}
break;
case "/":
res = new Div_Arifmetic();
break;
case "<":
res = new Less_CompExpr();
break;
case "<=":
res = new LessOrEqual_CompExpr();
break;
case ">=":
res = new GreatOrEqual_CompExpr();
break;
case ">":
res = new Great_CompExpr();
break;
case "=":
res = new Equal_CompExpr();
break;
case "<>":
case "!=":
res = new NotEqual_CompExpr();
break;
}
}
if (lex.LexemType == LexType.Number)
{
res = new ConstExpr();
if (lex.LexemText.Contains('.'))
{
(res as ConstExpr).Init(lex.LexemText.ParseDouble(), SimpleTypes.Float);
}
else
{
(res as ConstExpr).Init(long.Parse(lex.LexemText), SimpleTypes.Integer);
}
}
if (lex.LexemType == LexType.Text)
{
if (lex.LexemText.StartsWith("'"))
{
res = new ConstExpr();
(res as ConstExpr).Init(ParserUtils.StandartDecodeEscape(lex.LexemText), SimpleTypes.String);
}
}
if (lex.LexemType == LexType.Command)
{
switch (lex.LexemText.ToLower())
{
case "not":
res = new Not_BoolExpr();
break;
case "and":
res = new And_BoolExpr();
break;
case "or":
res = new Or_BoolExpr();
break;
case "true":
res = new ConstExpr();
(res as ConstExpr).Init(true, SimpleTypes.Boolean);
break;
case "false":
res = new ConstExpr();
(res as ConstExpr).Init(false, SimpleTypes.Boolean);
break;
case "date":
if (collection.GetNext() != null)
{
var c = collection.GetNext();
if (c.LexemType == LexType.Text && c.LexemText.StartsWith("'"))
{
res = new ConstExpr();
string s = ParserUtils.StandartDecodeEscape(c.LexemText);
var dt = CommonUtils.ParseDateTime(s);
if (dt != null)
{
collection.GotoNext();
((ConstExpr)res).Init(dt.Value, SimpleTypes.Date);
}
else
{
collection.Error("Can not parse date lexem", lex);
}
}
}
break;
case "datetime":
case "timestamp":
if (collection.GetNext() != null)
{
var c = collection.GetNext();
if (c.LexemType == LexType.Text && c.LexemText.StartsWith("'"))
{
res = new ConstExpr();
string s = ParserUtils.StandartDecodeEscape(c.LexemText);
var dt = CommonUtils.ParseDateTime(s);
if (dt != null)
{
collection.GotoNext();
((ConstExpr)res).Init(dt.Value, SimpleTypes.DateTime);
}
else
{
collection.Error("Can not parse datetime lexem", lex);
}
}
}
break;
case "time":
if (collection.GetNext() != null)
{
var c = collection.GetNext();
if (c.LexemType == LexType.Text && c.LexemText.StartsWith("'"))
{
res = new ConstExpr();
string s = ParserUtils.StandartDecodeEscape(c.LexemText);
DateTime dt;
var r = CommonUtils.ParseDateTime(s, out dt);
if (r == ParserDateTimeStatus.Time)
{
collection.GotoNext();
((ConstExpr)res).Init(dt.TimeOfDay, SimpleTypes.Time);
}
else
{
collection.Error("Can not parse time lexem", lex);
}
}
}
break;
}
}
return(res);
}
protected Expression DoOptimize(out bool changed)
{
changed = false;
if (Childs != null)
{
for (int i = 0; i < Childs.Count; i++)
{
bool ch;
Childs[i] = Childs[i].DoOptimize(out ch);
if (ch)
{
changed = true;
}
}
}
if (OnlyOnline() && !(this is ConstExpr))
{
ConstExpr ce = CreateConst();
switch (GetResultType())
{
case SimpleTypes.Boolean:
ce.Init(GetBoolResultOut(null), SimpleTypes.Boolean);
break;
case SimpleTypes.Date:
ce.Init(GetDateTimeResultOut(null), SimpleTypes.Date);
break;
case SimpleTypes.DateTime:
ce.Init(GetDateTimeResultOut(null), SimpleTypes.DateTime);
break;
case SimpleTypes.Float:
ce.Init(GetFloatResultOut(null), SimpleTypes.Float);
break;
case SimpleTypes.String:
ce.Init(GetStrResultOut(null), SimpleTypes.String);
break;
case SimpleTypes.Time:
ce.Init(GetTimeResultOut(null), SimpleTypes.Time);
break;
case SimpleTypes.Integer:
ce.Init(GetIntResultOut(null), SimpleTypes.Integer);
break;
case SimpleTypes.Blob:
ce.Init(GetBlobResultOut(null), SimpleTypes.Blob);
break;
case SimpleTypes.Geometry:
ce.Init(GetGeomResultOut(null), SimpleTypes.Geometry, GetCoordinateSystem());
break;
}
changed = true;
return(ce);
}
return(this);
}
// D Xs (Filter(T)) => Filter(D Xs T)
LogicNode djoinOnRightFilter(LogicSingleJoin singleJoinNode, ScalarSubqueryExpr scalarExpr)
{
var nodeLeft = singleJoinNode.lchild_();
var nodeSubquery = singleJoinNode.rchild_();
var nodeSubqueryFilter = nodeSubquery.filter_;
Debug.Assert(scalarExpr.query_.selection_.Count == 1);
var singleValueExpr = scalarExpr.query_.selection_[0];
nodeSubquery.NullifyFilter();
// nullify nodeA's filter: the rest is push to top filter. However,
// if nodeA is a Filter|MarkJoin, keep its mark filter.
var trueCondition = ConstExpr.MakeConstBool(true);
var nodeLeftFilter = nodeLeft.filter_;
if (nodeLeftFilter != null)
{
// a1 > @1 and a2 > @2 and a3 > 2, scalarExpr = @1
// keeplist: a1 > @1 and a3 > 2
// andlist after removal: a2 > @2
// nodeAFilter = a1 > @1 and a3 > 2
//
var andlist = nodeLeftFilter.FilterToAndList();
var keeplist = andlist.Where(x => x.VisitEachExists(e => e.Equals(scalarExpr))).ToList();
andlist.RemoveAll(x => x.VisitEachExists(e => e.Equals(scalarExpr)));
if (andlist.Count == 0)
{
nodeLeft.NullifyFilter();
}
else
{
nodeLeft.filter_ = andlist.AndListToExpr();
if (keeplist.Count > 0)
{
nodeLeftFilter = keeplist.AndListToExpr();
}
else
{
nodeLeftFilter = trueCondition;
}
}
}
// make a non-movable filter on top of the single join to replace a1 > @1
LogicFilter nonMovableFilter = null;
if (nodeLeftFilter != null)
{
// a1 > @1 => a1 > c1
var decExpr = nodeLeftFilter.SearchAndReplace(scalarExpr, singleValueExpr);
nonMovableFilter = new LogicFilter(singleJoinNode, decExpr)
{
movable_ = false
};
}
// b1 = ?outsideRef => b1 = outsideRef
nodeSubqueryFilter.DeParameter(nodeLeft.InclusiveTableRefs());
// join filter within sbuquery
Expr pullupFilter = nodeSubqueryFilter;
if (nodeLeft.filter_ != null && !nodeLeft.filter_.HasSubQuery())
{
pullupFilter = pullupFilter.AddAndFilter(nodeLeft.filter_);
nodeLeft.NullifyFilter();
}
LogicFilter Filter = new LogicFilter(nonMovableFilter != null ?
(LogicNode)nonMovableFilter : singleJoinNode, pullupFilter);
return(Filter);
}
public Expression GetNode(ExpressionParser parser)
{
Lexem lex = parser.Collection.CurrentLexem();
Expression ex = null;
bool uniar = parser.waitValue;
if (lex.LexemType == LexType.Arfimetic)
{
switch (lex.LexemText)
{
case "+":
if (uniar)
{
ex = new UniarPlus_BoolExpr();
}
else
{
ex = new Plus_Arifmetic();
}
break;
case "-":
if (uniar)
{
ex = new UniarMinus_BoolExpr();
}
else
{
ex = new Minus_Arifmetic();
}
break;
case "*":
if (uniar)
{
ex = new AllColumnExpr();
}
else
{
ex = new Multi_Arifmetic();
}
break;
case "/":
ex = new Div_Arifmetic();
break;
case "<":
ex = new Less_CompExpr();
break;
case "<=":
ex = new LessOrEqual_CompExpr();
break;
case ">=":
ex = new GreatOrEqual_CompExpr();
break;
case ">":
ex = new Great_CompExpr();
break;
case "=":
ex = new Equal_CompExpr();
break;
case "<>":
case "!=":
ex = new NotEqual_CompExpr();
break;
}
}
if (lex.LexemType == LexType.Number)
{
ex = new ConstExpr();
if (lex.LexemText.Contains('.'))
{
(ex as ConstExpr).Init(lex.LexemText.ParseDouble(), SimpleTypes.Float);
}
else
{
(ex as ConstExpr).Init(long.Parse(lex.LexemText), SimpleTypes.Integer);
}
}
if (lex.LexemType == LexType.Text)
{
if (lex.LexemText.StartsWith("'"))
{
ex = new ConstExpr();
(ex as ConstExpr).Init(ParserUtils.StandartDecodeEscape(lex.LexemText), SimpleTypes.String);
}
}
Lexem n1;
if (lex.LexemType == LexType.Command)
{
switch (lex.LexemText.ToLower())
{
case "not":
n1 = parser.Collection.GetNext();
if (n1 != null && n1.LexemType == LexType.Command && n1.LexemText.ToLower() == "in")
{
ex = new NotInExpr();
parser.Collection.GotoNext();
break;
}
ex = new Not_BoolExpr();
break;
case "case":
ex = new CaseExpr();
break;
case "contains":
ex = new Contains();
break;
case "containsic": //ic = ignore case
case "containscase":
ex = new ContainsIgnoreCase();
break;
case "startwith":
case "startswith":
ex = new StartsWith();
break;
case "endwith":
case "endswith":
ex = new EndsWith();
break;
}
if (parser.Collection.GetNext() != null && parser.Collection.GetNext().IsSkobraOpen())
{
switch (lex.LexemText.ToLower())
{
case "cast":
ex = new Cast();
break;
case "in":
ex = new InExpr();
break;
case "abs":
ex = new Abs();
break;
case "substring":
ex = new SubString();
break;
case "position":
ex = new Position();
break;
case "ltrim":
ex = new LTrim();
break;
case "rtrim":
ex = new RTrim();
break;
case "trim":
ex = new Trim();
break;
case "length":
ex = new Length();
break;
case "upper":
ex = new Upper_operation();
break;
case "lower":
ex = new Lower_operation();
break;
case "left":
ex = new Left();
break;
case "right":
ex = new Right();
break;
case "replace":
ex = new Replace();
break;
case "now":
ex = new Now();
break;
case "tostr":
ex = new ToString();
break;
case "strtotime":
ex = new StrToTime();
break;
case "strtodatetime":
ex = new StrToDateTime();
break;
case "addseconds":
ex = new AddSeconds();
break;
case "addminutes":
ex = new AddMinutes();
break;
case "addhours":
ex = new AddHours();
break;
case "adddays":
ex = new AddDays();
break;
case "day":
ex = new Day();
break;
case "month":
ex = new Month();
break;
case "year":
ex = new Year();
break;
case "round":
ex = new ParserCore.Expr.Extend.Math.Round();
break;
case "ceiling":
ex = new ParserCore.Expr.Extend.Math.Ceiling();
break;
case "floor":
ex = new ParserCore.Expr.Extend.Math.Floor();
break;
case "pi":
ex = new ParserCore.Expr.Extend.Math.Pi();
break;
case "tan":
ex = new ParserCore.Expr.Extend.Math.Tan();
break;
case "log":
ex = new ParserCore.Expr.Extend.Math.Log();
break;
}
}
}
return(ex);
}
internal Expr SimplifyRelop()
{
Expr l = lchild_();
Expr r = rchild_();
ConstExpr lce = l is ConstExpr ? (ConstExpr)l : null;
ConstExpr rce = r is ConstExpr ? (ConstExpr)r : null;
if (op_ == "is" || op_ == "is not")
{
if ((lce == null && rce != null) || (lce != null && rce == null))
{
return(this);
}
if (((lce == null && rce == null)) || ((lce != null && lce.IsNull()) && (rce != null && rce.IsNull())))
{
if (op_ == "is" || op_ == "is not")
{
string val = op_ == "is" ? "true" : "false";
return(ConstExpr.MakeConst(val, new BoolType(), outputName_));
}
}
if (((lce is null && rce is null)) || ((lce != null && !lce.IsNull()) && (rce != null && !rce.IsNull())))
{
if (op_ == "is" || op_ == "is not")
{
string val = op_ == "is" ? "false" : "true";
return(ConstExpr.MakeConst(val, new BoolType(), outputName_));
}
}
if (lce != null && !lce.IsNull() && rce != null && rce.IsNull())
{
if (op_ == "is" || op_ == "is not")
{
string val = op_ == "is" ? "false" : "true";
return(ConstExpr.MakeConst(val, new BoolType(), outputName_));
}
}
if (((lce != null && lce.IsNull()) || (rce != null && rce.IsNull())))
{
if (op_ == "is" || op_ == "is not")
{
string val = op_ == "is" ? "true" : "false";
return(ConstExpr.MakeConst(val, new BoolType(), outputName_));
}
}
}
if (lce == null && rce == null)
{
return(this);
}
if (!(lce is null || rce is null))
{
if (lce.IsTrue() && rce.IsTrue())
{
return(lce);
}
else
{
return(lce.IsTrue() ? lce : rce);
}
}
// expands [NOT] EXISTS filter to mark join
//
// LogicNode_A
// Filter: @1 AND|OR <others1>
// <ExistSubqueryExpr> 1
// -> LogicNode_B
// Filter: b.b1[0]=?a.a1[0] AND|OR <others2>
// =>
// Filter
// Filter: #marker AND|OR <others1>
// MarkJoin
// Filter: (b.b1[0]=a.a1[0]) AND|OR <others2> as #marker
// LogicNode_A
// LogicNode_B
//
// further convert DJoin to semi-join here is by decorrelate process
//
LogicNode existsToMarkJoin(LogicNode nodeA, ExistSubqueryExpr existExpr)
{
var nodeAIsOnMarkJoin =
nodeA is LogicFilter && (nodeA.child_() is LogicMarkJoin || nodeA.child_() is LogicSingleJoin);
// nodeB contains the join filter
var nodeB = existExpr.query_.logicPlan_;
var nodeBFilter = nodeB.filter_;
nodeB.NullifyFilter();
// nullify nodeA's filter: the rest is push to top filter. However,
// if nodeA is a Filter|MarkJoin, keep its mark filter.
var markerFilter = new ExprRef(new MarkerExpr(), 0);
var nodeAFilter = nodeA.filter_;
if (nodeAIsOnMarkJoin)
{
nodeA.filter_ = markerFilter;
}
else
{
if (nodeAFilter != null)
{
// a1 > @1 and a2 > @2 and a3 > 2, scalarExpr = @1
// keeplist: a1 > @1 and a3 > 2
// andlist after removal: a2 > @2
// nodeAFilter = a1 > @1 and a3 > 2
//
var andlist = nodeAFilter.FilterToAndList();
var keeplist = andlist.Where(x => x.VisitEachExists(e => e.Equals(existExpr))).ToList();
andlist.RemoveAll(x => x.VisitEachExists(e => e.Equals(existExpr)));
if (andlist.Count == 0)
{
nodeA.NullifyFilter();
}
else
{
nodeA.filter_ = andlist.AndListToExpr();
if (keeplist.Count > 0)
{
nodeAFilter = keeplist.AndListToExpr();
}
else
{
nodeAFilter = markerFilter;
}
}
}
}
// make a mark join
LogicMarkJoin markjoin;
if (existExpr.hasNot_)
{
markjoin = new LogicMarkAntiSemiJoin(nodeA, nodeB);
}
else
{
markjoin = new LogicMarkSemiJoin(nodeA, nodeB);
}
// make a filter on top of the mark join collecting all filters
Expr topfilter;
if (nodeAIsOnMarkJoin)
{
topfilter = nodeAFilter.SearchAndReplace(existExpr, ConstExpr.MakeConstBool(true));
}
else
{
topfilter = nodeAFilter.SearchAndReplace(existExpr, markerFilter);
}
nodeBFilter.DeParameter(nodeA.InclusiveTableRefs());
topfilter = topfilter.AddAndFilter(nodeBFilter);
LogicFilter Filter = new LogicFilter(markjoin, topfilter);
return(Filter);
}
public ModifyHelper()
{
modify = Expr.RecursiveModifier(e =>
{
if (e.nodeType == ExprType.Reference || e.nodeType == ExprType.Constant)
{
return(e);
}
if (e.nodeType != ExprType.Call)
{
goto tryConst;
}
var ce = (CallExpr)e;
bool isLet = ce.funcName == "let";
bool isDef = isLet || ce.funcName == "SV" || ce.funcName == "SSV";
if (!isDef && ce.funcName != "GV" && ce.funcName != "GSV")
{
goto tryConst;
}
var arg0 = ce.args[0];
string name;
if (arg0.nodeType == ExprType.Constant)
{
name = Convert.ToString((arg0 as ConstExpr).value);
if (!isLet && ce.args.Count == 1)
{
e = new ReferenceExpr(name);
}
goto constResolved;
}
if (arg0.nodeType == ExprType.Reference)
{
name = (arg0 as ReferenceExpr).name;
goto nameResolved;
}
// try to resolve name
var nameObj = TryGen(ctx, arg0);
if (nameObj == Fail || OPs.KindOf(nameObj) != ValueKind.Const)
{
thereIsNonresolved = true;
return(e);
}
name = Convert.ToString(nameObj);
thereIsNameResolve = true;
constResolved:
var args = new Expr[ce.args.Count];
if (isLet)
{
args[0] = new ReferenceExpr(name);
}
else
{
if (ce.args.Count == 1 && ce.funcName != "GSV")
{
return(new ReferenceExpr(name));
}
args[0] = new ConstExpr(name);
}
for (int i = 1; i < args.Length; i++)
{
args[i] = modify(ce.args[i]);
}
e = new CallExpr(ce.funcName, args);
nameResolved:
if (isDef && ce.args.Count == 2)
{
lets.Add(name);
//var value = modify(ce.args[1]);
//if (value.nodeType == ExprType.Constant)
//{ }
}
return(e);
tryConst:
var val = TryGen(ctx, e);
if (val == Fail || OPs.KindOf(val) != ValueKind.Const)
{
return(e);
}
return(new ConstExpr(val));
});
}
Expr SpecFuncs(CallExpr ce)
{
Expr src, ndx;
bool forPrev = false, forCell = false;
int nMinArgs = 1;
switch (ce.funcName)
{
case "PREVCELL":
forCell = true; forPrev = true; break;
case "PREV":
nMinArgs = 0; forPrev = true; break;
case "INDIRECT":
forCell = true; break;
case nameof(FuncDefs_Xlsx.COLUMN):
// avoid COLUMN(args) fixing
return(new CallExpr(nameof(FuncDefs_Xlsx.COLUMN), ce.args));
default:
return(ce);
}
if (ce.args.Count < nMinArgs)
{
return(ce);
}
Expr arg;
string name;
if (ce.args.Count == 0)
{
arg = null;
name = FCurrName;
}
else
{
arg = ce.args[0];
if (arg.nodeType == ExprType.Reference)
{
name = ((ReferenceExpr)arg).name;
if (IsColumnName(name) || !forPrev && IsFullRelativeCell(CellRange.FromName(name)))
{
forCell = true;
}
else
{
name = OPs.TryAsString(arg, ctxRow) ?? name;
}
}
else
{
arg = FixRowExpr(arg);
name = OPs.TryAsName(arg, ctxRow);
}
}
if (name == null)
{
var undefs = string.Join(", ", ctxHdr.NamesOfUndefinedValues().Concat(ctxRow.NamesOfUndefinedValues()));
throw new Source.Exception($"Constant expected as value of {ce.args[0]}={arg} // ??? {undefs}");
}
if (forCell || IsColumnName(name) || !forPrev && IsFullRelativeCell(CellRange.FromName(name)))
{
if (forPrev)
{
src = new IndexExpr(PrevValsRef, new ConstExpr(ctxRow.IndexOf(CellValsRef.name)));
}
else
{
var re1 = new ReferenceExpr(name);
var re2 = FixHdrCellRef(re1);
if (re2 != re1)
{
// ref to header cell
return(re2);
}
else
{
// ref to row cell
src = CellValsRef;
}
}
ndx = new ConstExpr(name);
}
else
{
int i = ctxRow.IndexOf(name);
if (i < 0 || Generator.Ctx.ctxDepthStep <= i)
{
throw new Source.Exception($"Value named '{name}' not found in row context");
}
src = PrevValsRef;
ndx = new ConstExpr(i);
}
return(new IndexExpr(src, ndx));
}
public static Expr BytecodeToInfix(EzSembleContext context, byte[] Bytes)
{
var bigEndianReverseBytes = Bytes.Reverse().ToArray();
Stack <Expr> exprs = new Stack <Expr>();
List <Expr> popArgs(int amount)
{
List <Expr> args = new List <Expr>();
for (int i = 0; i < amount; i++)
{
args.Add(exprs.Pop());
}
args.Reverse();
return(args);
}
for (int i = 0; i < Bytes.Length; i++)
{
var b = Bytes[i];
if (b >= 0 && b <= 0x7F)
{
exprs.Push(new ConstExpr {
Value = (sbyte)(b - 64)
});
}
else if (b == 0xA5)
{
int j = 0;
while (Bytes[i + j + 1] != 0 || Bytes[i + j + 2] != 0)
{
j += 2;
}
string text = context.IsBigEndian ?
Encoding.BigEndianUnicode.GetString(Bytes, i + 1, j)
: Encoding.Unicode.GetString(Bytes, i + 1, j);
if (text.Contains('"') || text.Contains('\r') || text.Contains('\n'))
{
throw new Exception("Illegal character in string literal");
}
exprs.Push(new ConstExpr {
Value = text
});
i += j + 2;
}
else if (b == 0x80)
{
float val;
if (!context.IsBigEndian)
{
val = BitConverter.ToSingle(Bytes, i + 1);
}
else
{
val = BitConverter.ToSingle(bigEndianReverseBytes, (bigEndianReverseBytes.Length - 1) - (i + 1) - 4);
}
exprs.Push(new ConstExpr {
Value = val
});
i += 4;
}
else if (b == 0x81)
{
double val;
if (!context.IsBigEndian)
{
val = BitConverter.ToDouble(Bytes, i + 1);
}
else
{
val = BitConverter.ToDouble(bigEndianReverseBytes, (bigEndianReverseBytes.Length - 1) - (i + 1) - 8);
}
exprs.Push(new ConstExpr {
Value = val
});
i += 8;
}
else if (b == 0x82)
{
int val;
if (!context.IsBigEndian)
{
val = BitConverter.ToInt32(Bytes, i + 1);
}
else
{
val = BitConverter.ToInt32(bigEndianReverseBytes, (bigEndianReverseBytes.Length - 1) - (i + 1) - 4);
}
exprs.Push(new ConstExpr {
Value = val
});
i += 4;
}
else if (b == 0x84)
{
exprs.Push(new FunctionCall {
Args = popArgs(0), Name = context.GetFunctionInfo(exprs.Pop().AsInt()).Name
});
}
else if (b == 0x85)
{
exprs.Push(new FunctionCall {
Args = popArgs(1), Name = context.GetFunctionInfo(exprs.Pop().AsInt()).Name
});
}
else if (b == 0x86)
{
exprs.Push(new FunctionCall {
Args = popArgs(2), Name = context.GetFunctionInfo(exprs.Pop().AsInt()).Name
});
}
else if (b == 0x87)
{
exprs.Push(new FunctionCall {
Args = popArgs(3), Name = context.GetFunctionInfo(exprs.Pop().AsInt()).Name
});
}
else if (b == 0x88)
{
exprs.Push(new FunctionCall {
Args = popArgs(4), Name = context.GetFunctionInfo(exprs.Pop().AsInt()).Name
});
}
else if (b == 0x89)
{
exprs.Push(new FunctionCall {
Args = popArgs(5), Name = context.GetFunctionInfo(exprs.Pop().AsInt()).Name
});
}
else if (b == 0x8A)
{
exprs.Push(new FunctionCall {
Args = popArgs(6), Name = context.GetFunctionInfo(exprs.Pop().AsInt()).Name
});
}
else if (OperatorsByByte.ContainsKey(b))
{
if (OperatorsByByte[b] == "N")
{
exprs.Push(new UnaryExpr {
Op = "N", Arg = exprs.Pop()
});
}
else
{
exprs.Push(new BinaryExpr {
Op = OperatorsByByte[b], Rhs = exprs.Pop(), Lhs = exprs.Pop()
});
}
}
else if (b == 0xA6)
{
Expr top = exprs.Peek();
top.IfFalse = FalseCond.CONTINUE;
}
else if (b >= 0xA7 && b <= 0xAE)
{
byte regIndex = (byte)(b - 0xA7);
exprs.Push(new FunctionCall {
Args = popArgs(1), Name = $"SetREG{regIndex}"
});
}
else if (b >= 0xAF && b <= 0xB6)
{
byte regIndex = (byte)(b - 0xAF);
exprs.Push(new FunctionCall {
Args = popArgs(0), Name = $"GetREG{regIndex}"
});
}
else if (b == 0xB7)
{
Expr top = exprs.Peek();
top.IfFalse = FalseCond.ABORT;
}
else if (b == 0xB8)
{
// exprs.Push(new FunctionCall { Args = popArgs(1), Name = "StateGroupArg" });
FunctionCall func = new FunctionCall {
Args = popArgs(1), Name = "StateGroupArg"
};
ConstExpr ce = func.Args[0] as ConstExpr;
// Console.WriteLine($"{ce} {ce.Value.GetType()}");
exprs.Push(func);
}
else if (b == 0xB9)
{
exprs.Push(new CallResult());
}
else if (b == 0xBA)
{
// This opcode just returns a constant value 0x7FFFFFFF
// But use higher-level representation of it
exprs.Push(new CallOngoing());
}
else if (b == 0xA1)
{
//break;
}
else
{
exprs.Push(new Unknown {
Opcode = b
});
}
}
if (exprs.Count != 1)
{
throw new Exception("Could not parse expr. Remaining stack: " + string.Join("; ", exprs) + $"; = {string.Join(" ", Bytes.Select(x => x.ToString("X2")))}");
}
return(exprs.Pop());
}
private Metadata(IEnumerable <NE> exprsHdr, IEnumerable <NE> exprsRow, IEnumerable <NE> exprsRes, Generator.Ctx parentCtx)
{
var sb = new StringBuilder();
FixSheetExpr = Expr.RecursiveModifier(FixSheetImpl);
FixRowExpr = Expr.RecursiveModifier(FixRowImpl);
#region generate header code
ctxHdr = new Generator.Ctx(parentCtx);
ctxHdr.CreateValue(HdrCellsRef.name, Generator.LazyDummy);
ctxHdr.CreateValue(PrevValsRef.name, Generator.LazyDummy);
foreach (var p in exprsHdr)
{
FCurrName = p.name;
var expr = FixSheetExpr(p.expr);
var val = Generator.Generate(expr, ctxHdr);
ctxHdr.CreateValue(p.name, val);
if (OPs.KindOf(val) == ValueKind.Const)
{
expr = new ConstExpr(val);
}
sb.AppendLine($"{p.name} = {expr}");
}
sb.AppendLine();
RequiredInHdr(sSourceKey);
RequiredInHdr(sSourceName);
RequiredInHdr(sCondition);
RequiredInHdr(sFirstDataRowNum);
{
if (ctxHdr.name2ndx.TryGetValue(sFileContentKind, out int i))
{
FileContentKind = Convert.ToString(ctxHdr.GetConstant(sFileContentKind));
}
else
{
FileContentKind = "XLSX";
}
}
{
if (ctxHdr.name2ndx.TryGetValue(sFileNameExtension, out int i))
{
var val = ctxHdr.GetConstant(sFileNameExtension);
FileNameExtension = Utils.AsIList(val).Cast <object>().Select(Convert.ToString).ToArray();
}
else
{
FileNameExtension = new string[] { "xls", "xlsx", "xlsm" }
};
}
#endregion
#region generate row code
ctxRow = new Generator.Ctx(ctxHdr);
ctxRow.CreateValue(sDataKey, Generator.LazyDummy); // [0] - iKey
ctxRow.CreateValue(sDataSubkey, Generator.LazyDummy); // [1] - iSubkey
ctxRow.CreateValue(CellValsRef.name, Generator.LazyDummy); // [2] - iRowVals
foreach (var p in exprsRow)
{
string name;
if (p.name.StartsWith("~"))
{
// calculated name
name = OPs.TryAsString(Parser.ParseToExpr(p.name.Substring(1)), ctxHdr) ?? p.name;
}
else
{
name = p.name;
}
FCurrName = name;
int i = ctxRow.GetOrCreateIndexOf(name);
var expr = FixRowExpr(p.expr);
var val = Generator.Generate(expr, ctxRow);
ctxRow.values[i] = val;
if (OPs.KindOf(val) == ValueKind.Const)
{
expr = new ConstExpr(val);
}
sb.AppendLine($"{name} = {expr}");
}
RequiredInRow(sDataKey);
RequiredInRow(sDataSubkey);
ctxRow.values[iRowVals] = null;
ctxRow.CheckUndefinedValues();
#endregion
#region generate res code
if (exprsRes != null)
{
// copy all code from row to res
ctxRes = new Generator.Ctx(ctxHdr);
foreach (var p in ctxRow.name2ndx.OrderBy(p => p.Value))
{
ctxRes.values[ctxRes.CreateValue(p.Key)] = ctxRow.values[p.Value];
}
foreach (var p in exprsRes)
{
string name;
if (p.name.StartsWith("~"))
{
// calculated name
name = OPs.TryAsString(Parser.ParseToExpr(p.name.Substring(1)), ctxHdr) ?? p.name;
}
else
{
name = p.name;
}
FCurrName = name;
int i = ctxRes.GetOrCreateIndexOf(name);
var expr = FixRowExpr(p.expr);
var val = Generator.Generate(expr, ctxRes);
ctxRes.values[i] = val;
if (OPs.KindOf(val) == ValueKind.Const)
{
expr = new ConstExpr(val);
}
}
ctxRes.values[iRowVals] = null;
ctxRes.CheckUndefinedValues();
}
#endregion
ctxHdr[HdrCellsRef.name] = null;
ctxHdr[PrevValsRef.name] = null;
ctxHdr.CheckUndefinedValues();
FCodeText = sb.ToString();
if (iHdrMinUsedRow > iHdrMaxUsedRow)
{
iHdrMinUsedRow = iHdrMaxUsedRow;
}
}
public override Expr Normalize()
{
// all children get normalized first
for (int i = 0; i < children_.Count; ++i)
{
Expr x = children_[i];
children_[i] = x.Normalize();
}
Expr l = lchild_();
Expr r = rchild_();
ConstExpr lce = (l is ConstExpr) ? (ConstExpr)l : null;
ConstExpr rce = (r is ConstExpr) ? (ConstExpr)r : null;
switch (op_)
{
case "+":
case "-":
case "*":
case "/":
case ">":
case ">=":
case "<":
case "<=":
case "||":
case "=":
case "<>":
case "!=":
case " and ":
case " or ":
case "is":
case "is not":
if ((lce != null && lce.val_ is null) || (rce != null && rce.val_ is null))
{
if (IsRelOp())
{
// needs to be TRUE or FALSE
if ((op_ == "is") || (op_ == "is not") || (lce != null && TypeBase.IsNumberType(l.type_)) || (rce != null && TypeBase.IsNumberType(r.type_)))
{
return(SimplifyRelop());
}
}
// NULL simplification: if operator is not relational, X op NULL is NULL
if (lce != null && lce.val_ is null)
{
return(lce);
}
if (rce != null && rce.IsNull())
{
return(rce);
}
}
if (lce != null && rce != null)
{
// Simplify Constants: children are not non null constants, evaluate them.
Value val = Exec(null, null);
return(ConstExpr.MakeConst(val, type_, outputName_));
}
if (lce != null && rce == null && isPlainSwappableConstOp())
{
SwapSide();
}
if ((lce != null || rce != null) && (IsArithIdentity(lce, rce)))
{
return(SimplifyArithmetic(lce, rce));
}
if (IsLogicalOp())
{
return(SimplifyLogic());
}
if (IsRelOp())
{
return(SimplifyRelop());
}
// arithmetic operators?
if (l is BinExpr le && le.children_[1].IsConst() && (rce != null) &&
isCommutativeConstOp() && le.isCommutativeConstOp() && TypeBase.SameArithType(l.type_, r.type_))
{
/*
* Here root is distributive operator (only * in this context) left is Commutative
* operator, +, or * right is constant, furthermore, left's right is a constant
* (becuase we swapped cosntant to be on the right).
* if be == + and l == +: add left's right value to root's right value,
* make left's left as left of root
*
* if be == * and l == +: create a expr node as left (x + 10), create (5 * 10)
* as right, change operator to +
* In either case left and right's children must be nulled out
* and since we are going bottom up, this doesn't create any problem.
*
* Here is a pictorial description:
* * root +
* old left / \ old right new left / \ new right
* / \ / \
* + 10 => * 50
* left of old left / \ / \
* / \ ROL LNL / \ RNL (right of New Left)
* x 5 x 10
*/
/*
* Simple case: when current and left are same operators, distributive
* opeartion and node creation is uncessary.
*/
if ((op_ == "+" && le.op_ == "+") || (op_ == "*" && le.op_ == "*"))
{
/* create new right node as constant. */
Expr tmpexp = Clone();
tmpexp.children_[0] = le.children_[1];
tmpexp.children_[1] = r;
tmpexp.type_ = r.type_;
Value val;
bool wasConst = tmpexp.TryEvalConst(out val);
Expr newr = ConstExpr.MakeConst(val, tmpexp.type_, r.outputName_);
// new left is old left's left child
// of left will be right of the root.
children_[0] = l.children_[0];
// new right is the new constant node
children_[1] = newr;
}
else
if (op_ == "*" && le.rchild_() is ConstExpr lrc && (le.op_ == "+" || le.op_ == "-"))
{
/*
* case of (a + const1) * const2 => (a * const2) + (const1 * const2))
* make a newe left node to house (a * const2)
*
* * +
* / \ / \
* / \ / \
* + c2 => * c1 * c2
* / \ / \
* / \ / \
* X c1 X c2
*
*/
/* make a const expr node to evaluate const1 * const 2 */
Expr tmpexp = Clone();
tmpexp.children_[0] = lrc; // right of left is const
tmpexp.children_[1] = r; // our right is const
tmpexp.type_ = r.type_;
Value val;
tmpexp.TryEvalConst(out val);
// set c2 as the value of right child of our left
lrc.val_ = rce.val_;
// val is c1 * c2, set it as the value of our right child
rce.val_ = val;
/* swap the operators */
string op = op_;
op_ = le.op_;
le.op_ = op;
}
/* we can't do any thing about + at the top and * as left child. */
}
return(this);
}
return(this);
}
public bool Equals(ConstExpr o) => Value.Equals(o.Value);
