public static TemplateExpression Parse(ExpressionWalker ew) { ew.IsCurrentOrThrow(ExpressionToken.Template); ew.NextOrThrow(); // ReSharper disable once UseObjectOrCollectionInitializer var ret = new TemplateExpression(); ew.IsCurrentOrThrow(ExpressionToken.StringLiteral); ret._matchPath = new RegexResult[] { ew.Current.Match.AsResult() }; ret.Path = ew.Current.Match.Value.StartsWith("\"") ? ew.Current.Match.Value.Substring(1, ew.Current.Match.Length - 2) : ew.Current.Match.Value; ret._matchForEvery = ew.Next(ExpressionToken.ForEvery, true).Match.AsResult(); ew.NextOrThrow(); ret.Arguments = ArgumentsExpression.Parse(ew, token => token.Token == ExpressionToken.NewLine || token.Token == ExpressionToken.Mod, false, typeof(ForeachExpression)); //we have to fall back because ArgumentsExpression.Parse swallows mod. ew.IsBack(ExpressionToken.Mod, true); if (ew.IsCurrent(ExpressionToken.Mod)) { ew.Back(1); } return(ret); }
public static TernaryExpression Parse(ExpressionWalker ew, Expression condition = null) { if (condition == null) { condition = Expression.ParseExpression(ew, typeof(ExpressionParser)); } ew.IsCurrentOrThrow(ExpressionToken.Or); ew.NextOrThrow(); Expression trueResult; Expression falseResult = null; if (ew.OptionalCurrent(ExpressionToken.Or)) { trueResult = Expression.None; falseResult = Expression.ParseExpression(ew, typeof(TernaryExpression)); } else { trueResult = Expression.ParseExpression(ew, typeof(TernaryExpression)); if (ew.OptionalCurrent(ExpressionToken.Or)) { falseResult = Expression.ParseExpression(ew, typeof(TernaryExpression)); } } return(new TernaryExpression(condition, trueResult, falseResult)); }
public static GroupExpression Parse(ExpressionWalker ew, ExpressionToken left, ExpressionToken right) { ew.IsCurrentOrThrow(left); ew.NextOrThrow(); if (ew.Current.Token == right) { throw new UnexpectedTokenException($"Expected an expression, found end of group of type {right}"); } var expr = ParseExpression(ew, typeof(GroupExpression)); var grp = new GroupExpression() { _matchLeft = AttributeExtensions.GetAttribute <ExpressionTokenAttribute>(left).Emit.AsResult(), _matchRight = AttributeExtensions.GetAttribute <ExpressionTokenAttribute>(right).Emit.AsResult() }; if (ew.IsCurrent(ExpressionToken.Or)) { grp.InnerExpression = TernaryExpression.Parse(ew, expr); } else { grp.InnerExpression = expr; } ew.IsCurrentOrThrow(right); ew.Next(); return(grp); }
public static ImportExpression Parse(ExpressionWalker ew) { ew.IsCurrentOrThrow(ExpressionToken.Import); ew.NextOrThrow(); // ReSharper disable once UseObjectOrCollectionInitializer var ret = new ImportExpression(); if (ew.IsCurrent(ExpressionToken.StringLiteral)) { ret._matchType = new RegexResult[] { ew.Current.Match.AsResult() }; ret.Type = ew.Current.Match.Value.Trim('\"'); } else { ret._matchType = ew.TakeForwardWhile(t => t.Token == ExpressionToken.Period || t.Token == ExpressionToken.Literal) .Select(t => t.Match.AsResult()).ToArray(); ret.Type = ret._matchType.Select(m => m.Value).StringJoin(); if (ew.IsCurrent(ExpressionToken.As)) { ret._matchAlias = ew.Next(ExpressionToken.Literal, true).Match.AsResult(); ret.As = ret._matchAlias.Value; } } ew.Next(); return(ret); }
public static ThrowExpression Parse(ExpressionWalker ew) { ew.IsCurrentOrThrow(ExpressionToken.Throw); //you get it? throw if current is not throw. ew.NextOrThrow(); // ReSharper disable once UseObjectOrCollectionInitializer var ret = new ThrowExpression(); ret.Right = Expression.ParseExpression(ew, typeof(ThrowExpression)); return(ret); }
public static KeyValueExpression Parse(ExpressionWalker ew, Expression left = null, Type caller = null) { var key = left ?? ParseExpression(ew, caller ?? typeof(KeyValueExpression)); ew.IsCurrentOrThrow(ExpressionToken.Colon); ew.NextOrThrow(); var value = ParseExpression(ew, caller ?? typeof(KeyValueExpression)); return(new KeyValueExpression(key, value)); }
public static ArgumentsExpression Parse(ExpressionWalker ew, ExpressionToken left, ExpressionToken right, bool argsOptional, Type caller = null) { var args = new ArgumentsExpression(); ew.IsCurrentOrThrow(left); ew.NextOrThrow(); var exprs = new List <Expression>(); while (ew.Current.Token != right && ew.HasNext) { if (ew.Current.Token == ExpressionToken.Comma) { if (ew.HasBack && ew.PeakBack.Token == ExpressionToken.Comma) { exprs.Add(NullIdentity.Instance); } ew.NextOrThrow(); continue; } var expression = ParseExpression(ew, caller); if (ew.IsCurrent(ExpressionToken.Colon)) { //handle keyvalue item exprs.Add(KeyValueExpression.Parse(ew, expression, caller)); } else { exprs.Add(expression); } } if (exprs.Count == 0 && !argsOptional) { throw new UnexpectedTokenException($"Was expecting an expression between {left} and {right}"); } ew.Next(); args.Arguments = exprs.ToArray(); return(args); }
public static VariableDeclarationExpression Parse(ExpressionWalker ew) { // ReSharper disable once UseObjectOrCollectionInitializer var var = new VariableDeclarationExpression(); var.Name = StringIdentity.Parse(ew); ew.IsCurrentOrThrow(ExpressionToken.Equal); ew.NextOrThrow(); var.Right = Expression.ParseExpression(ew); return(var); }
public static ParserAction Parse(string code) { var walker = new ExpressionWalker(ExpressionLexer.Tokenize(code)); if (walker.Current.Token == ExpressionToken.Mod) { walker.NextOrThrow(); } var lines = new LineBuilder(code); lines.Lines.RemoveAt(0); return(Parse(walker, code, lines)); }
public static LeftOperatorExpression Parse(ExpressionWalker ew, Expression known = null) { if (!IsCurrentAnLeftUniOperation(ew)) { throw new UnexpectedTokenException(ExpressionToken.Increment, ew.Current.Token); } var op = ew.Current.Token; ew.NextOrThrow(); var right = ParseExpression(ew, typeof(LeftOperatorExpression)); return(new LeftOperatorExpression(op, right)); }
public static Expression Parse(ExpressionWalker ew, Type caller = null) { var ret = new HashtagReferenceExpression(); ew.IsCurrentOrThrow(ExpressionToken.Hashtag); ret._hashtagMatch = ew.Current.Match.AsResult(); ew.NextOrThrow(); ew.IsCurrentOrThrow(ExpressionToken.NumberLiteral); ret._numberMatch = ew.Current.Match.AsResult(); ret.Number = NumberLiteral.Parse(ew).Value; return(ret); }
public static Expression Parse(ExpressionWalker ew, Expression known = null) { var left = known ?? ParseExpression(ew, typeof(RightOperatorExpression)); if (!IsCurrentAnRightUniOperation(ew)) { return(left); } var op = ew.Current.Token; ew.NextOrThrow(); return(new RightOperatorExpression(left, op)); }
public static Expression Parse(ExpressionWalker ew, Expression left = null) { // ReSharper disable once UseObjectOrCollectionInitializer var ret = new OperatorExpression(); ret.Left = left ?? ParseExpression(ew, typeof(OperatorExpression)); if (!IsCurrentAnOperation(ew)) { return(left); } ret.Op = ew.Current.Token; ew.NextOrThrow(); ret.Right = ParseExpression(ew, typeof(OperatorExpression)); return(ret); }
public static ArgumentsExpression Parse(ExpressionWalker ew, Func <TokenMatch, bool> parseTill, bool argsOptional, Type caller = null) { var args = new ArgumentsExpression(); var exprs = new List <Expression>(); while (!parseTill(ew.Current) && ew.HasNext) { if (ew.Current.Token == ExpressionToken.Comma) { if (ew.HasBack && ew.PeakBack.Token == ExpressionToken.Comma) { exprs.Add(NullIdentity.Instance); } ew.NextOrThrow(); continue; } var expression = ParseExpression(ew, caller); if (ew.IsCurrent(ExpressionToken.Colon)) { //handle keyvalue item exprs.Add(KeyValueExpression.Parse(ew, expression)); } else { exprs.Add(expression); } } if (exprs.Count == 0 && !argsOptional) { throw new UnexpectedTokenException($"Was expecting arguments but found none while argsOptional is false"); } ew.Next(); args.Arguments = exprs.ToArray(); return(args); }
private void _executeForeach(ForeachExpression expr, List <Line> relatedLines, Line baseLine) { var contents = relatedLines.Select(l => l.Content).ToArray(); var iterateThose = expr.Arguments.Arguments.Select(parseExpr).ToList(); unpackPackedArguments(); //get smallest index and iterate it. var min = iterateThose.Min(i => i.Count); var vars = Context.Variables; for (int i = 0; i < min; i++) { //set variables if (expr.Depth > 0) { vars[$"i{expr.Depth}"] = new NumberScalar(i); } else { vars["i"] = new NumberScalar(i); } for (int j = 0; j < iterateThose.Count; j++) { vars[$"__{j + 1 + expr.Depth * 100}__"] = iterateThose[j][i]; } var variables = new List <string>(); //a list of all added variables that will be cleaned after this i iteration. //now here we iterate contents and set all variables in it. for (var contentIndex = 0; contentIndex < contents.Length; contentIndex++) { var content = contents[contentIndex]; //iterate lines, one at a time // ReSharper disable once RedundantToStringCall var copy = content.ToString().Replace("|#", "#"); bool changed = false; int last_access_index = 0; const string HashtagExpressionRegex = @"(?<!\\)\#\((?:[^()]|(?<open>\()|(?<-open>\)))+(?(open)(?!))\)"; var hashtagExprs = Regex.Matches(copy, HashtagExpressionRegex, Regexes.DefaultRegexOptions).Cast <Match>().ToArray(); //replace all emit commands copy = ExpressionLexer.ReplaceRegex(copy, @"(?<!\\)\#([0-9]+)", match => { var key = $"__{match.Groups[1].Value}__"; if (hashtagExprs.Any(m => m.IsMatchNestedTo(match))) { //it is inside hashtagExpr #(...) return(key); } return(_emit(vars[key])); }); var ew = new ExpressionWalker(ExpressionLexer.Tokenize(copy, ExpressionToken.StringLiteral)); if (ew.HasNext) { do { _restart: if (changed) { changed = false; var cleanedCopy = new string(' ', last_access_index) + copy.Substring(last_access_index); ew = new ExpressionWalker(ExpressionLexer.Tokenize(cleanedCopy, ExpressionToken.StringLiteral)); } var current = ew.Current; //iterate all tokens of that line if (current.Token == ExpressionToken.Mod && ew.HasNext) { if (ew.HasBack && ew.PeakBack.Token == ExpressionToken.Escape) { continue; } var expr_ew = new ExpressionWalker(ExpressionLexer.Tokenize(copy.Substring(current.Match.Index))); //var offset = current.Match.Index; //var hashtag = expr_ew.Current; current = expr_ew.NextToken(); switch (current.Token) { case ExpressionToken.Foreach: var code = contents.SkipWhile(s => s != content).StringJoin(); var e = ForeachExpression.Parse(code); var foreachExpr = (ForeachExpression)e.Related[0]; foreachExpr.Depth = expr.Depth + 1; //no need to mark lines from e for deletion, they are already marked beforehand. _executeForeach(foreachExpr, e.RelatedLines, baseLine); contentIndex += e.RelatedLines.Count + 2 - 1; //first for the %foreach line, second for the closer %, -1 because we increment index by one on next iteration. goto _skipline; default: continue; } } if (current.Token == ExpressionToken.Hashtag && ew.HasNext) { if (ew.HasBack && ew.PeakBack.Token == ExpressionToken.Escape) { continue; } var offset = current.Match.Index; var expr_ew = new ExpressionWalker(ExpressionLexer.Tokenize(copy.Substring(current.Match.Index))); var hashtag = expr_ew.Current; current = expr_ew.NextToken(); switch (current.Token) { case ExpressionToken.Literal: //this is variable declaration %varname = expr var peak = expr_ew.PeakNext.Token; if (peak == ExpressionToken.Equal) { var e = VariableDeclarationExpression.Parse(expr_ew); var varname = e.Name.AsString(); if (!Context.Variables.ContainsKey(varname)) { variables.Add(varname); } CompileAction(new ParserAction(ParserToken.Declaration, new List <Expression>() { e }), new OList <ParserAction>(0)); goto _skipline; } break; case ExpressionToken.LeftParen: { //it is an expression. expr_ew.NextOrThrow(); var expression = Expression.ParseExpression(expr_ew); object val = EvaluateObject(expression, baseLine); if (val is ReferenceData rd) //make sure references are unpacked { val = rd.UnpackReference(Context); } expr_ew.IsCurrentOrThrow(ExpressionToken.RightParen); var emit = val is Data d?d.Emit() : val.ToString(); copy = copy .Remove(offset + hashtag.Match.Index, expr_ew.Current.Match.Index + 1 - hashtag.Match.Index) .Insert(offset + hashtag.Match.Index, emit); last_access_index = hashtag.Match.Index + emit.Length; changed = true; goto _restart; } case ExpressionToken.NumberLiteral: { if (expr_ew.HasNext && expr_ew.PeakNext.Token == ExpressionToken.LeftBracet) { //it is an indexer call. //todo indexer } else { //it is a simple emit var key = $"#{expr_ew.Current.Match.Value}"; object val = vars[$"__{expr_ew.Current.Match.Value}__"]; copy = Regex.Replace(copy, Regex.Escape(key), _emit(val)); changed = true; } goto _restart; } default: continue; } } //incase it is escaped, continue. } while (ew.Next()); } _nextline: //cleanup escapes copy = copy.Replace("\\#", "#"); baseLine.ReplaceOrAppend(copy + (copy.EndsWith("\n") ? "" : "\n")); _skipline :; } foreach (var variable in variables) { Context.Variables.Remove(variable); } } if (expr.Depth == 0) { Context.Variables.Remove("i"); } else { Context.Variables.Remove($"i{expr.Depth}"); } for (var i = 0; i < iterateThose.Count; i++) { Context.Variables.Remove($"__{i + 1 + expr.Depth * 100}__"); } if (!baseLine.ContentWasModified) { baseLine.MarkedForDeletion = true; } IList parseExpr(Expression arg) { var ev = EvaluateObject(arg, baseLine); if (ev is ReferenceData d) { ev = d.UnpackReference(Context); } if (ev is StringScalar ss) { return(ss.ToCharArray()); } if (ev is NetObject no) { ev = no.Value; } return((IList)ev); } void unpackPackedArguments() { //unpack PackedArguments for (var i = iterateThose.Count - 1; i >= 0; i--) { if (iterateThose[i] is PackedArguments pa) { iterateThose.InsertRange(i, pa.Objects.Select(o => (IList)o)); } } iterateThose.RemoveAll(it => it is PackedArguments); } }
public static ParserAction Parse(ExpressionWalker ew, string code, LineBuilder output) { // multiline: // %foreach expr% // code #1 // % // singleline: // %foreach expr // code #1 ew.IsCurrentOrThrow(ExpressionToken.Foreach); ew.NextOrThrow(); //parse the arguments for the foreach var args = ArgumentsExpression.Parse(ew, token => token.Token == ExpressionToken.NewLine || token.Token == ExpressionToken.Mod, false, typeof(ForeachExpression)); //ew.Back(); //agrumentsExpression skips the closer token and we need it to identify if this is a singleline or multiline if (output == null) { output = new LineBuilder(code); } string content; var relatedLines = new List <Line>(); relatedLines.AddRange(output.GetLinesRelated(args.Matches())); if (ew.PeakBack.Token == ExpressionToken.Mod) { //the content is % to % block var leftBorder = ew.Current.Match; var nextMod = ForeachExpression.FindCloser(leftBorder.Index, code); //handle implicit end block (when % is not existing) if (nextMod == -1) { nextMod = code.Length - 1; } ew.SkipForwardWhile(token => token.Match.Index < nextMod); ew.Next(); //skip % itself var l1 = output.GetLineAt(leftBorder.Index) ?? output.Lines.First(); relatedLines.AddRange(new Range(l1.LineNumber, output.GetLineAt(nextMod).LineNumber).EnumerateIndexes().Select(output.GetLineByLineNumber)); content = null; } else { //the content is only next line var leftMod = ew.Current.Match; var nextMod = code.IndexOf('\n', leftMod.Index); relatedLines.Add(output.GetLineByLineNumber(relatedLines.Last().LineNumber + 1)); //next line. content = code.Substring(leftMod.Index, nextMod == -1 ? (code.Length - leftMod.Index) : nextMod - leftMod.Index); } relatedLines = relatedLines.Distinct().OrderBy(l => l.StartIndex).ToList(); if (relatedLines.Count(l => l.Content.Contains("%foreach")) <= relatedLines.Count(l => l.CleanContent() == "%") && relatedLines.Last().CleanContent() == "%") { relatedLines[relatedLines.Count - 1].MarkedForDeletion = true; relatedLines.RemoveAt(relatedLines.Count - 1); } //make sure to clean out % at the end if (content == null) { content = relatedLines.Select(l => l.Content).StringJoin(); } //all lines of the foreach are destined to deletion foreach (var line in relatedLines) { line.MarkedForDeletion = true; } return(new ParserAction(ParserToken.ForeachLoop, relatedLines, new ForeachExpression() { Content = content, Arguments = args })); }
public static Expression ParseExpression(ExpressionWalker ew, Type caller = null) { Expression ret = null; _retry: bool isOperatorCall = caller == typeof(OperatorExpression) || caller == typeof(RightOperatorExpression) || caller == typeof(ForeachExpression); var current = ew.Current.Token; if (current == ExpressionToken.Literal) { //cases like variable(.., variable[.., variable + .., variable if (ew.HasNext) { var peak = ew.PeakNextOrThrow().Token; if (peak == ExpressionToken.LeftParen) { ret = CallExpression.Parse(ew); } else if (peak == ExpressionToken.Period && caller != typeof(IdentityExpression) && caller != typeof(InteractableExpression)) { ret = IdentityExpression.Parse(ew); } else if (peak == ExpressionToken.LeftBracet) { ret = IndexerCallExpression.Parse(ew); } else if (peak == ExpressionToken.New) { ret = NewExpression.Parse(ew); } else if (RightOperatorExpression.IsNextAnRightUniOperation(ew, caller) && caller != typeof(RightOperatorExpression)) { ret = RightOperatorExpression.Parse(ew); } else if (OperatorExpression.IsNextAnOperation(ew) && !isOperatorCall) { ret = OperatorExpression.Parse(ew); } else { ret = IdentityExpression.Parse(ew, caller); } } else { ret = IdentityExpression.Parse(ew, caller); } } else if (LeftOperatorExpression.IsCurrentAnLeftUniOperation(ew)) { ret = LeftOperatorExpression.Parse(ew); } else if (current == ExpressionToken.NumberLiteral) { ret = NumberLiteral.Parse(ew); } else if (current == ExpressionToken.StringLiteral) { ret = StringLiteral.Parse(ew); } else if (current == ExpressionToken.LeftParen) { ret = GroupExpression.Parse(ew, ExpressionToken.LeftParen, ExpressionToken.RightParen); } else if (current == ExpressionToken.LeftBracet) { ret = ArrayExpression.Parse(ew); } else if (current == ExpressionToken.New) { ret = NewExpression.Parse(ew); } else if (current == ExpressionToken.Boolean) { ret = BooleanLiteral.Parse(ew); } else if (current == ExpressionToken.CharLiteral) { ret = CharLiteral.Parse(ew); } else if (current == ExpressionToken.Throw) { ret = ThrowExpression.Parse(ew); } else if (current == ExpressionToken.Hashtag && ew.HasNext && ew.PeakNext.Token == ExpressionToken.NumberLiteral) { ret = HashtagReferenceExpression.Parse(ew, caller); } else if (current == ExpressionToken.Null) { ret = NullIdentity.Parse(ew); } else if (current == ExpressionToken.NewLine) { ew.NextOrThrow(); goto _retry; } else { throw new UnexpectedTokenException($"Token was not expected to be a {ew.Current.Token}"); } //here we parse chained math operations while (OperatorExpression.IsCurrentAnOperation(ew) && !isOperatorCall) { if (RightOperatorExpression.IsCurrentAnRightUniOperation(ew) && caller != typeof(OperatorExpression)) { ret = RightOperatorExpression.Parse(ew, ret); } else if (OperatorExpression.IsCurrentAnOperation(ew)) { ret = OperatorExpression.Parse(ew, ret); } } return(ret); }
public static IdentityExpression Parse(ExpressionWalker ew, Type caller = null, Expression left = null) { var ret = new IdentityExpression(); //types: //justname //justname.accessor //justname.accessor[5].second //justname.method().someval //justname.method().someval[3].thatsfar if (left == null) { ew.IsCurrentOrThrow(ExpressionToken.Literal); } if (ew.HasNext && ew.PeakNext.WhitespacesAfterMatch == 0 && ew.PeakNext.Token == ExpressionToken.Period && caller != typeof(IdentityExpression) || left != null) { var first = left ?? ParseExpression(ew, typeof(IdentityExpression)); ew.NextOrThrow(); //skip the predicted period. var next = ew.PeakNext; switch (next.Token) { //currently we are at Literal case ExpressionToken.Period: ret.Identity = new PropertyIdentity(first, Parse(ew)); return(ret); case ExpressionToken.LeftBracet: ret.Identity = new PropertyIdentity(first, IndexerCallExpression.Parse(ew)); return(ret); case ExpressionToken.LeftParen: ret.Identity = new PropertyIdentity(first, CallExpression.Parse(ew)); return(ret); default: if (left != null) { return(new IdentityExpression(new PropertyIdentity(left, Parse(ew, typeof(IdentityExpression), null)))); } if (first != null) { //just a plain single word! var right = new IdentityExpression(); if (ew.IsCurrent(ExpressionToken.Null)) { right.Identity = NullIdentity.Instance; } else { ew.IsCurrentOrThrow(ExpressionToken.Literal); right.Identity = StringIdentity.Create(ew.Current.Match); } ew.Next(); return(new IdentityExpression(new PropertyIdentity(first, right))); } goto _plain_identity; } } _plain_identity: //just a plain single word! if (ew.IsCurrent(ExpressionToken.Null)) { ret.Identity = NullIdentity.Instance; } else { ew.IsCurrentOrThrow(ExpressionToken.Literal); ret.Identity = StringIdentity.Create(ew.Current.Match); } ew.Next(); return(ret); }
public static NewExpression Parse(ExpressionWalker ew) { ew.IsCurrentOrThrow(ExpressionToken.New); ew.NextOrThrow(); return(new NewExpression(CallExpression.Parse(ew))); }