public static GroupExpression Parse(ExpressionWalker ew, ExpressionToken left, ExpressionToken right)
{
ew.IsCurrentOrThrow(left);
ew.NextOrThrow();
if (ew.Current.Token == right)
{
throw new UnexpectedTokenException($"Expected an expression, found end of group of type {right}");
}
var expr = ParseExpression(ew, typeof(GroupExpression));
var grp = new GroupExpression()
{
_matchLeft = AttributeExtensions.GetAttribute <ExpressionTokenAttribute>(left).Emit.AsResult(), _matchRight = AttributeExtensions.GetAttribute <ExpressionTokenAttribute>(right).Emit.AsResult()
};
if (ew.IsCurrent(ExpressionToken.Or))
{
grp.InnerExpression = TernaryExpression.Parse(ew, expr);
}
else
{
grp.InnerExpression = expr;
}
ew.IsCurrentOrThrow(right);
ew.Next();
return(grp);
}
public static TemplateExpression Parse(ExpressionWalker ew)
{
ew.IsCurrentOrThrow(ExpressionToken.Template);
ew.NextOrThrow();
// ReSharper disable once UseObjectOrCollectionInitializer
var ret = new TemplateExpression();
ew.IsCurrentOrThrow(ExpressionToken.StringLiteral);
ret._matchPath = new RegexResult[] { ew.Current.Match.AsResult() };
ret.Path = ew.Current.Match.Value.StartsWith("\"") ? ew.Current.Match.Value.Substring(1, ew.Current.Match.Length - 2) : ew.Current.Match.Value;
ret._matchForEvery = ew.Next(ExpressionToken.ForEvery, true).Match.AsResult();
ew.NextOrThrow();
ret.Arguments = ArgumentsExpression.Parse(ew, token => token.Token == ExpressionToken.NewLine || token.Token == ExpressionToken.Mod, false, typeof(ForeachExpression));
//we have to fall back because ArgumentsExpression.Parse swallows mod.
ew.IsBack(ExpressionToken.Mod, true);
if (ew.IsCurrent(ExpressionToken.Mod))
{
ew.Back(1);
}
return(ret);
}
public static Expression Parse(ExpressionWalker ew, Type caller = null)
{
var ret = new HashtagReferenceExpression();
ew.IsCurrentOrThrow(ExpressionToken.Hashtag);
ret._hashtagMatch = ew.Current.Match.AsResult();
ew.NextOrThrow();
ew.IsCurrentOrThrow(ExpressionToken.NumberLiteral);
ret._numberMatch = ew.Current.Match.AsResult();
ret.Number = NumberLiteral.Parse(ew).Value;
return(ret);
}
public static TernaryExpression Parse(ExpressionWalker ew, Expression condition = null)
{
if (condition == null)
{
condition = Expression.ParseExpression(ew, typeof(ExpressionParser));
}
ew.IsCurrentOrThrow(ExpressionToken.Or);
ew.NextOrThrow();
Expression trueResult;
Expression falseResult = null;
if (ew.OptionalCurrent(ExpressionToken.Or))
{
trueResult = Expression.None;
falseResult = Expression.ParseExpression(ew, typeof(TernaryExpression));
}
else
{
trueResult = Expression.ParseExpression(ew, typeof(TernaryExpression));
if (ew.OptionalCurrent(ExpressionToken.Or))
{
falseResult = Expression.ParseExpression(ew, typeof(TernaryExpression));
}
}
return(new TernaryExpression(condition, trueResult, falseResult));
}
public static ImportExpression Parse(ExpressionWalker ew)
{
ew.IsCurrentOrThrow(ExpressionToken.Import);
ew.NextOrThrow();
// ReSharper disable once UseObjectOrCollectionInitializer
var ret = new ImportExpression();
if (ew.IsCurrent(ExpressionToken.StringLiteral))
{
ret._matchType = new RegexResult[] { ew.Current.Match.AsResult() };
ret.Type = ew.Current.Match.Value.Trim('\"');
}
else
{
ret._matchType = ew.TakeForwardWhile(t => t.Token == ExpressionToken.Period || t.Token == ExpressionToken.Literal)
.Select(t => t.Match.AsResult()).ToArray();
ret.Type = ret._matchType.Select(m => m.Value).StringJoin();
if (ew.IsCurrent(ExpressionToken.As))
{
ret._matchAlias = ew.Next(ExpressionToken.Literal, true).Match.AsResult();
ret.As = ret._matchAlias.Value;
}
}
ew.Next();
return(ret);
}
public static BooleanLiteral Parse(ExpressionWalker ew)
{
ew.IsCurrentOrThrow(ExpressionToken.Boolean);
var ret = new BooleanLiteral(bool.Parse(ew.Current.Match.Groups[1].Value));
ret._match = ew.Current.Match.AsResult();
ew.Next();
return(ret);
}
public static KeyValueExpression Parse(ExpressionWalker ew, Expression left = null, Type caller = null)
{
var key = left ?? ParseExpression(ew, caller ?? typeof(KeyValueExpression));
ew.IsCurrentOrThrow(ExpressionToken.Colon);
ew.NextOrThrow();
var value = ParseExpression(ew, caller ?? typeof(KeyValueExpression));
return(new KeyValueExpression(key, value));
}
public static ThrowExpression Parse(ExpressionWalker ew)
{
ew.IsCurrentOrThrow(ExpressionToken.Throw); //you get it? throw if current is not throw.
ew.NextOrThrow();
// ReSharper disable once UseObjectOrCollectionInitializer
var ret = new ThrowExpression();
ret.Right = Expression.ParseExpression(ew, typeof(ThrowExpression));
return(ret);
}
public static NumberLiteral Parse(ExpressionWalker ew)
{
ew.IsCurrentOrThrow(ExpressionToken.NumberLiteral);
var ret = new NumberLiteral(ew.Current.Match.Value)
{
_match = ew.Current.Match.AsResult()
};
ew.Next();
return(ret);
}
public static StringLiteral Parse(ExpressionWalker ew)
{
ew.IsCurrentOrThrow(ExpressionToken.StringLiteral);
var ret = new StringLiteral(ew.Current.Match.Groups[1].Value)
{
_match = ew.Current.Match.AsResult()
};
ew.Next();
return(ret);
}
public static VariableDeclarationExpression Parse(ExpressionWalker ew)
{
// ReSharper disable once UseObjectOrCollectionInitializer
var var = new VariableDeclarationExpression();
var.Name = StringIdentity.Parse(ew);
ew.IsCurrentOrThrow(ExpressionToken.Equal);
ew.NextOrThrow();
var.Right = Expression.ParseExpression(ew);
return(var);
}
public static ReferenceIdentity Parse(ExpressionWalker ew)
{
//types:
//justname
ew.IsCurrentOrThrow(ExpressionToken.Literal);
var ret = new ReferenceIdentity(ew.Current.Match.Value)
{
_match = ew.Current.Match.AsResult()
};
ew.Next();
return(ret);
}
public static ArgumentsExpression Parse(ExpressionWalker ew, ExpressionToken left, ExpressionToken right, bool argsOptional, Type caller = null)
{
var args = new ArgumentsExpression();
ew.IsCurrentOrThrow(left);
ew.NextOrThrow();
var exprs = new List <Expression>();
while (ew.Current.Token != right && ew.HasNext)
{
if (ew.Current.Token == ExpressionToken.Comma)
{
if (ew.HasBack && ew.PeakBack.Token == ExpressionToken.Comma)
{
exprs.Add(NullIdentity.Instance);
}
ew.NextOrThrow();
continue;
}
var expression = ParseExpression(ew, caller);
if (ew.IsCurrent(ExpressionToken.Colon))
{
//handle keyvalue item
exprs.Add(KeyValueExpression.Parse(ew, expression, caller));
}
else
{
exprs.Add(expression);
}
}
if (exprs.Count == 0 && !argsOptional)
{
throw new UnexpectedTokenException($"Was expecting an expression between {left} and {right}");
}
ew.Next();
args.Arguments = exprs.ToArray();
return(args);
}
public static NewExpression Parse(ExpressionWalker ew)
{
ew.IsCurrentOrThrow(ExpressionToken.New);
ew.NextOrThrow();
return(new NewExpression(CallExpression.Parse(ew)));
}
public static Expression Parse(ExpressionWalker ew)
{
ew.IsCurrentOrThrow(ExpressionToken.Null);
ew.Next();
return(new IdentityExpression(Instance));
}
private void _executeForeach(ForeachExpression expr, List <Line> relatedLines, Line baseLine)
{
var contents = relatedLines.Select(l => l.Content).ToArray();
var iterateThose = expr.Arguments.Arguments.Select(parseExpr).ToList();
unpackPackedArguments();
//get smallest index and iterate it.
var min = iterateThose.Min(i => i.Count);
var vars = Context.Variables;
for (int i = 0; i < min; i++)
{
//set variables
if (expr.Depth > 0)
{
vars[$"i{expr.Depth}"] = new NumberScalar(i);
}
else
{
vars["i"] = new NumberScalar(i);
}
for (int j = 0; j < iterateThose.Count; j++)
{
vars[$"__{j + 1 + expr.Depth * 100}__"] = iterateThose[j][i];
}
var variables = new List <string>(); //a list of all added variables that will be cleaned after this i iteration.
//now here we iterate contents and set all variables in it.
for (var contentIndex = 0; contentIndex < contents.Length; contentIndex++)
{
var content = contents[contentIndex];
//iterate lines, one at a time
// ReSharper disable once RedundantToStringCall
var copy = content.ToString().Replace("|#", "#");
bool changed = false;
int last_access_index = 0;
const string HashtagExpressionRegex = @"(?<!\\)\#\((?:[^()]|(?<open>\()|(?<-open>\)))+(?(open)(?!))\)";
var hashtagExprs = Regex.Matches(copy, HashtagExpressionRegex, Regexes.DefaultRegexOptions).Cast <Match>().ToArray();
//replace all emit commands
copy = ExpressionLexer.ReplaceRegex(copy, @"(?<!\\)\#([0-9]+)", match => {
var key = $"__{match.Groups[1].Value}__";
if (hashtagExprs.Any(m => m.IsMatchNestedTo(match)))
{
//it is inside hashtagExpr #(...)
return(key);
}
return(_emit(vars[key]));
});
var ew = new ExpressionWalker(ExpressionLexer.Tokenize(copy, ExpressionToken.StringLiteral));
if (ew.HasNext)
{
do
{
_restart:
if (changed)
{
changed = false;
var cleanedCopy = new string(' ', last_access_index) + copy.Substring(last_access_index);
ew = new ExpressionWalker(ExpressionLexer.Tokenize(cleanedCopy, ExpressionToken.StringLiteral));
}
var current = ew.Current;
//iterate all tokens of that line
if (current.Token == ExpressionToken.Mod && ew.HasNext)
{
if (ew.HasBack && ew.PeakBack.Token == ExpressionToken.Escape)
{
continue;
}
var expr_ew = new ExpressionWalker(ExpressionLexer.Tokenize(copy.Substring(current.Match.Index)));
//var offset = current.Match.Index;
//var hashtag = expr_ew.Current;
current = expr_ew.NextToken();
switch (current.Token)
{
case ExpressionToken.Foreach:
var code = contents.SkipWhile(s => s != content).StringJoin();
var e = ForeachExpression.Parse(code);
var foreachExpr = (ForeachExpression)e.Related[0];
foreachExpr.Depth = expr.Depth + 1;
//no need to mark lines from e for deletion, they are already marked beforehand.
_executeForeach(foreachExpr, e.RelatedLines, baseLine);
contentIndex += e.RelatedLines.Count + 2 - 1; //first for the %foreach line, second for the closer %, -1 because we increment index by one on next iteration.
goto _skipline;
default:
continue;
}
}
if (current.Token == ExpressionToken.Hashtag && ew.HasNext)
{
if (ew.HasBack && ew.PeakBack.Token == ExpressionToken.Escape)
{
continue;
}
var offset = current.Match.Index;
var expr_ew = new ExpressionWalker(ExpressionLexer.Tokenize(copy.Substring(current.Match.Index)));
var hashtag = expr_ew.Current;
current = expr_ew.NextToken();
switch (current.Token)
{
case ExpressionToken.Literal:
//this is variable declaration %varname = expr
var peak = expr_ew.PeakNext.Token;
if (peak == ExpressionToken.Equal)
{
var e = VariableDeclarationExpression.Parse(expr_ew);
var varname = e.Name.AsString();
if (!Context.Variables.ContainsKey(varname))
{
variables.Add(varname);
}
CompileAction(new ParserAction(ParserToken.Declaration, new List <Expression>()
{
e
}), new OList <ParserAction>(0));
goto _skipline;
}
break;
case ExpressionToken.LeftParen: {
//it is an expression.
expr_ew.NextOrThrow();
var expression = Expression.ParseExpression(expr_ew);
object val = EvaluateObject(expression, baseLine);
if (val is ReferenceData rd) //make sure references are unpacked
{
val = rd.UnpackReference(Context);
}
expr_ew.IsCurrentOrThrow(ExpressionToken.RightParen);
var emit = val is Data d?d.Emit() : val.ToString();
copy = copy
.Remove(offset + hashtag.Match.Index, expr_ew.Current.Match.Index + 1 - hashtag.Match.Index)
.Insert(offset + hashtag.Match.Index, emit);
last_access_index = hashtag.Match.Index + emit.Length;
changed = true;
goto _restart;
}
case ExpressionToken.NumberLiteral: {
if (expr_ew.HasNext && expr_ew.PeakNext.Token == ExpressionToken.LeftBracet)
{
//it is an indexer call.
//todo indexer
}
else
{
//it is a simple emit
var key = $"#{expr_ew.Current.Match.Value}";
object val = vars[$"__{expr_ew.Current.Match.Value}__"];
copy = Regex.Replace(copy, Regex.Escape(key), _emit(val));
changed = true;
}
goto _restart;
}
default:
continue;
}
}
//incase it is escaped, continue.
} while (ew.Next());
}
_nextline:
//cleanup escapes
copy = copy.Replace("\\#", "#");
baseLine.ReplaceOrAppend(copy + (copy.EndsWith("\n") ? "" : "\n"));
_skipline :;
}
foreach (var variable in variables)
{
Context.Variables.Remove(variable);
}
}
if (expr.Depth == 0)
{
Context.Variables.Remove("i");
}
else
{
Context.Variables.Remove($"i{expr.Depth}");
}
for (var i = 0; i < iterateThose.Count; i++)
{
Context.Variables.Remove($"__{i + 1 + expr.Depth * 100}__");
}
if (!baseLine.ContentWasModified)
{
baseLine.MarkedForDeletion = true;
}
IList parseExpr(Expression arg)
{
var ev = EvaluateObject(arg, baseLine);
if (ev is ReferenceData d)
{
ev = d.UnpackReference(Context);
}
if (ev is StringScalar ss)
{
return(ss.ToCharArray());
}
if (ev is NetObject no)
{
ev = no.Value;
}
return((IList)ev);
}
void unpackPackedArguments()
{
//unpack PackedArguments
for (var i = iterateThose.Count - 1; i >= 0; i--)
{
if (iterateThose[i] is PackedArguments pa)
{
iterateThose.InsertRange(i, pa.Objects.Select(o => (IList)o));
}
}
iterateThose.RemoveAll(it => it is PackedArguments);
}
}
public static ParserAction Parse(ExpressionWalker ew, string code, LineBuilder output)
{
// multiline:
// %foreach expr%
// code #1
// %
// singleline:
// %foreach expr
// code #1
ew.IsCurrentOrThrow(ExpressionToken.Foreach);
ew.NextOrThrow();
//parse the arguments for the foreach
var args = ArgumentsExpression.Parse(ew, token => token.Token == ExpressionToken.NewLine || token.Token == ExpressionToken.Mod, false, typeof(ForeachExpression));
//ew.Back(); //agrumentsExpression skips the closer token and we need it to identify if this is a singleline or multiline
if (output == null)
{
output = new LineBuilder(code);
}
string content;
var relatedLines = new List <Line>();
relatedLines.AddRange(output.GetLinesRelated(args.Matches()));
if (ew.PeakBack.Token == ExpressionToken.Mod)
{
//the content is % to % block
var leftBorder = ew.Current.Match;
var nextMod = ForeachExpression.FindCloser(leftBorder.Index, code);
//handle implicit end block (when % is not existing)
if (nextMod == -1)
{
nextMod = code.Length - 1;
}
ew.SkipForwardWhile(token => token.Match.Index < nextMod);
ew.Next(); //skip % itself
var l1 = output.GetLineAt(leftBorder.Index) ?? output.Lines.First();
relatedLines.AddRange(new Range(l1.LineNumber, output.GetLineAt(nextMod).LineNumber).EnumerateIndexes().Select(output.GetLineByLineNumber));
content = null;
}
else
{
//the content is only next line
var leftMod = ew.Current.Match;
var nextMod = code.IndexOf('\n', leftMod.Index);
relatedLines.Add(output.GetLineByLineNumber(relatedLines.Last().LineNumber + 1)); //next line.
content = code.Substring(leftMod.Index, nextMod == -1 ? (code.Length - leftMod.Index) : nextMod - leftMod.Index);
}
relatedLines = relatedLines.Distinct().OrderBy(l => l.StartIndex).ToList();
if (relatedLines.Count(l => l.Content.Contains("%foreach")) <= relatedLines.Count(l => l.CleanContent() == "%") && relatedLines.Last().CleanContent() == "%")
{
relatedLines[relatedLines.Count - 1].MarkedForDeletion = true;
relatedLines.RemoveAt(relatedLines.Count - 1);
}
//make sure to clean out % at the end
if (content == null)
{
content = relatedLines.Select(l => l.Content).StringJoin();
}
//all lines of the foreach are destined to deletion
foreach (var line in relatedLines)
{
line.MarkedForDeletion = true;
}
return(new ParserAction(ParserToken.ForeachLoop, relatedLines, new ForeachExpression()
{
Content = content, Arguments = args
}));
}
public static IdentityExpression Parse(ExpressionWalker ew, Type caller = null, Expression left = null)
{
var ret = new IdentityExpression();
//types:
//justname
//justname.accessor
//justname.accessor[5].second
//justname.method().someval
//justname.method().someval[3].thatsfar
if (left == null)
{
ew.IsCurrentOrThrow(ExpressionToken.Literal);
}
if (ew.HasNext && ew.PeakNext.WhitespacesAfterMatch == 0 && ew.PeakNext.Token == ExpressionToken.Period && caller != typeof(IdentityExpression) || left != null)
{
var first = left ?? ParseExpression(ew, typeof(IdentityExpression));
ew.NextOrThrow(); //skip the predicted period.
var next = ew.PeakNext;
switch (next.Token)
{
//currently we are at Literal
case ExpressionToken.Period:
ret.Identity = new PropertyIdentity(first, Parse(ew));
return(ret);
case ExpressionToken.LeftBracet:
ret.Identity = new PropertyIdentity(first, IndexerCallExpression.Parse(ew));
return(ret);
case ExpressionToken.LeftParen:
ret.Identity = new PropertyIdentity(first, CallExpression.Parse(ew));
return(ret);
default:
if (left != null)
{
return(new IdentityExpression(new PropertyIdentity(left, Parse(ew, typeof(IdentityExpression), null))));
}
if (first != null)
{
//just a plain single word!
var right = new IdentityExpression();
if (ew.IsCurrent(ExpressionToken.Null))
{
right.Identity = NullIdentity.Instance;
}
else
{
ew.IsCurrentOrThrow(ExpressionToken.Literal);
right.Identity = StringIdentity.Create(ew.Current.Match);
}
ew.Next();
return(new IdentityExpression(new PropertyIdentity(first, right)));
}
goto _plain_identity;
}
}
_plain_identity:
//just a plain single word!
if (ew.IsCurrent(ExpressionToken.Null))
{
ret.Identity = NullIdentity.Instance;
}
else
{
ew.IsCurrentOrThrow(ExpressionToken.Literal);
ret.Identity = StringIdentity.Create(ew.Current.Match);
}
ew.Next();
return(ret);
}