public JittedKernel(JitContext ctx)
{
_cfg = ctx.Cfg.Clone();
_ptx = ctx.Ptx;
_hir = ctx.Hir.Fluent(blk => blk.FreezeForever());
var flds = ctx.Allocator.Fields;
_memcpyHostToDevice = kernel =>
{
var args = new KernelArguments();
flds.Keys.ForEach(fld =>
{
var value = fld.GetValue(kernel);
if (flds[fld] is SlotLayout)
{
args.Add(value.In());
}
else if (flds[fld] is ArrayLayout)
{
var arr = value.AssertCast<Array>();
args.Add(arr.InOut());
var rank = arr.GetType().GetArrayRank();
0.UpTo(rank - 1).ForEach(i => args.Add(arr.GetLength(i).In()));
}
else
{
throw AssertionHelper.Fail();
}
});
return args;
};
_memcpyDeviceToHost = (result, kernel) =>
{
var idx = 0;
flds.Keys.ForEach(fld =>
{
Object value;
if (flds[fld] is SlotLayout)
{
value = result[idx];
idx += 1;
}
else if (flds[fld] is ArrayLayout)
{
value = result[idx];
idx += 3;
}
else
{
throw AssertionHelper.Fail();
}
fld.SetValue(kernel, value);
});
};
}
[DebuggerNonUserCode] public BlockExpander(Block source, ExpansionContext ctx)
{
Source = source;
Ctx = ctx;
Stmts = new Block();
ctx.Scope = Stmts;
var cloned_locals = source.Locals.Select(l => l.DeepClone());
cloned_locals.ForEach(local => ctx.Scope.Locals.Add(local));
var is_root = Ctx.Parent == null || (Ctx.Parent.Stack.Count() != Ctx.Stack.Count());
if (is_root) RetLabel = new Label();
else RetLabel.AssertNotNull();
source.ForEach(Expand);
if (is_root && Stmts.LastOrDefault() is Goto) Stmts.RemoveLast();
var gotos = Stmts.Family().OfType<Goto>().Where(@goto => @goto.LabelId == RetLabel.Id).ToReadOnly();
if (is_root && gotos.IsNotEmpty()) Stmts.Add(RetLabel);
}
protected internal override void TraverseBlock(Block block)
{
var prefixes = block.ToDictionary(s => s, s => new List<Action<IndentedWriter>>());
var evalOrder = block.CSharpEvaluationOrder();
var allRefs = evalOrder.OfType<Ref>().Where(
@ref => @ref.Sym != null && @ref.Sym.IsLocal()).ToReadOnly();
var r_scopes = allRefs.ToDictionary(@ref => @ref, @ref => @ref.Scope());
block.Locals.ForEach(loc =>
{
var refs = allRefs.Where(@ref => @ref.Sym == loc).ToReadOnly();
if (refs.IsEmpty()) return;
var firstUsage = refs.First();
var firstAss = firstUsage.Parent.AssertCast<Assign>().AssertNotNull();
(firstAss.Lhs is Ref && ((Ref)firstAss.Lhs).Sym == loc).AssertTrue();
if (ReferenceEquals(r_scopes[firstUsage], block))
{
prefixes[firstAss].Add(w => w.Write("var "));
}
else
{
var stmt = firstUsage.Hierarchy().SkipWhile(n => n.Parent != block).First();
prefixes[stmt].Add(w => w.WriteLine(String.Format("{0} {1};",
loc.Type == null ? "?" : loc.Type.GetCSharpRef(ToCSharpOptions.Informative), loc.Name)));
}
});
_writer.WriteLine("{");
_writer.Indent++;
block.ForEach(c =>
{
prefixes[c].ForEach(action => action(_writer));
Traverse(c);
if (IsOneliner(c))
{
if (c is Label) _writer.WriteLine(":");
else _writer.WriteLine(";");
}
});
_writer.Indent--;
_writer.WriteLine("}");
}
private JitCompiler(CpuConfig config, Type t_kernel, TypeBuilder t_xformed)
{
_config = config;
_t_kernel = t_kernel;
_m_kernel = _t_kernel.GetMethod("RunKernel", BF.All);
ValidateInputParameters();
// todo. think how to support multiple methods: *Dims, *Idxes, synchronization points
// todo. I'd even say that we should embed constants instead of Dims and cache such bodies between calls
// todo. if some dimension is equal to 1, then strip off corresponding loop
var lam = _m_kernel.Decompile();
_hir = lam.Body;
_params = lam.Sig.Syms;
_locals = lam.Body.LocalsRecursive();
_xhir = new Block();
_tids.Add("x", new Local("tid_x", typeof(int)));
_tids.Add("y", new Local("tid_y", typeof(int)));
_tids.Add("z", new Local("tid_z", typeof(int)));
_xhir.Locals.AddElements(_tids.Values);
InferLocalAllocationHints();
ReplicatePrivatelyAllocatedLocals();
LiftLocallyAllocatedLocals();
HoistGloballyAllocatedLocals();
_callsToSyncThreads = _hir.Family().OfType<Eval>().Where(eval =>
{
var m1 = eval.InvokedMethod();
var syncapi_syncThreads = typeof(ISyncApi).GetMethod("SyncThreads");
return m1.Api() == syncapi_syncThreads;
}).ToReadOnly();
TransformBlock(_hir, _xhir, false);
// todo. currently we don't support debuggable IL
// the latter implies correct PDB-mappings and working watch for replicated vars
_config.EmitDebuggableIL.AssertFalse();
_t_xformed = t_xformed;
_t_xformed.AddInterfaceImplementation(typeof(IBlockRunner));
_m_xformed = _t_xformed.DefineMethod("RunBlock", MA.Public, typeof(void), typeof(int3).MkArray());
_m_xformed.DefineParameter(1, ParmA.None, "blockIdx");
CompileTransformedHir();
}
public static Block DoRestoreTypeIs(Block hir)
{
if (hir.Family().OfType<TypeAs>().None()) return hir;
else
{
return hir.Transform((Operator op) =>
{
var is_rel = op.OperatorType.IsRelational();
if (is_rel)
{
var bin = op.AssertCast<BinaryOperator>();
var opt = bin.OperatorType;
var needs_xform = bin.Lhs is TypeAs ^ bin.Rhs is TypeAs;
if (needs_xform)
{
var type_as = bin.Lhs as TypeAs ?? bin.Rhs as TypeAs;
var other = bin.Lhs == type_as ? bin.Rhs : bin.Lhs;
other.AssertCast<Const>().AssertThat(c => c.Value == null);
var pos = opt == OperatorType.NotEqual || opt == OperatorType.GreaterThan;
var neg = opt == OperatorType.Equal || opt == OperatorType.LessThanOrEqual;
(pos || neg).AssertTrue();
var type_is = new TypeIs(type_as.Type, type_as.Target) as Expression;
return pos ? type_is : Operator.Negate(type_is);
}
else
{
return op.DefaultTransform();
}
}
else
{
return op.DefaultTransform();
}
}).AssertCast<Block>();
}
}
protected override void TraverseBlock(Block block)
{
block.ForEach(stmt => _ptx.emit(stmt));
}
protected internal override void TraverseBlock(Block block)
{
Dispatch(block);
}
public static Block DoSimplifyConditions(Block hir)
{
return hir.SimplifyConditions().AssertCast<Block>();
}
protected internal override Node TransformBlock(Block block)
{
return Dispatch(block);
}
public static Block DoRestoreUsings(Block hir)
{
// todo. to be implemented
return hir;
}
private void TransformBlock(Block hir, Block xhir, bool insideThreadLoop)
{
if (hir == null) return;
var deepFission = new List<Node>();
var regions = new List<ReadOnlyCollection<Node>>();
var curr_region = new List<Node>();
foreach (var stmt in hir)
{
if (_callsToSyncThreads.Contains(stmt as Eval))
{
if (curr_region.IsNotEmpty()) regions.Add(curr_region.ToReadOnly());
curr_region = new List<Node>();
}
else if (_callsToSyncThreads.Any(c => Set.Intersect(c.Hierarchy(), stmt.MkArray()).IsNotEmpty()))
{
if (curr_region.IsNotEmpty()) regions.Add(curr_region.ToReadOnly());
curr_region = new List<Node>();
deepFission.Add(stmt);
regions.Add(stmt.MkArray().ToReadOnly());
}
else
{
curr_region.Add(stmt);
}
}
if (curr_region.IsNotEmpty()) regions.Add(curr_region.ToReadOnly());
curr_region = null;
var privateLocals = hir.Locals.Where(l => Alloc(l) == allocPrivate).ToReadOnly();
var privateUsages = privateLocals.ToDictionary(l => l,
l => regions.Where(r => r.UsagesOfLocal(l).IsNotEmpty()).ToReadOnly());
privateUsages.ForEach(kvp =>
{
var needsReplication = kvp.Value.Count() > 1;
_needsReplication[kvp.Key] = needsReplication;
if (needsReplication)
{
_xhir.Insert(_lastIndex++, _replicatedInits[kvp.Key]);
_xhir.Locals.Add(_replicatedLocals[kvp.Key]);
}
});
foreach (var region in regions)
{
var xregion = new Block();
var needsToBeWrapped = !deepFission.Contains(region.SingleOrDefault2());
var regionIsInsideThreadLoop = insideThreadLoop || needsToBeWrapped;
foreach (var stmt in region)
{
if (stmt is Expression)
{
TransformExpression(((Expression)stmt), xregion, regionIsInsideThreadLoop);
}
else if (stmt is If)
{
TransformIf(((If)stmt), xregion, regionIsInsideThreadLoop);
}
else if (stmt is Loop)
{
TransformLoop(((Loop)stmt), xregion, regionIsInsideThreadLoop);
}
else
{
throw AssertionHelper.Fail();
}
}
if (needsToBeWrapped && !insideThreadLoop)
{
xhir.Add(new Loop
{
Init = new Block(new Assign(new Ref(_tids["z"]), new Const(0))),
Test = Operator.LessThan(new Ref(_tids["z"]), new Fld(typeof(int3).GetField("Z"), new Prop(typeof(IGridApi).GetProperty("BlockDim"), new Ref(_this), true))),
Body = new Block(new Loop
{
Init = new Block(new Assign(new Ref(_tids["y"]), new Const(0))),
Test = Operator.LessThan(new Ref(_tids["y"]), new Fld(typeof(int3).GetField("Y"), new Prop(typeof(IGridApi).GetProperty("BlockDim"), new Ref(_this), true))),
Body = new Block(new Loop
{
Init = new Block(new Assign(new Ref(_tids["x"]), new Const(0))),
Test = Operator.LessThan(new Ref(_tids["x"]), new Fld(typeof(int3).GetField("X"), new Prop(typeof(IGridApi).GetProperty("BlockDim"), new Ref(_this), true))),
Body = new Block(xregion.Children),
Iter = new Block(Operator.PreIncrement(new Ref(_tids["x"]))),
}),
Iter = new Block(Operator.PreIncrement(new Ref(_tids["y"]))),
}),
Iter = new Block(Operator.PreIncrement(new Ref(_tids["z"]))),
});
}
else
{
xhir.AddElements(xregion.Children);
}
}
var locals = hir.Locals.Except(_liftedLocals).ToReadOnly();
locals.ForEach(l => xhir.Locals.Add(l.DeepClone()));
}
private void Expand(Node node)
{
if (node is Expression)
{
var expr = (Expression)node;
var inlined = expr.Expand(Ctx);
inlined.Stmts.ForEach(Stmts.Add);
if (inlined.Result != null) Stmts.Add(inlined.Result);
}
else if (node is Block)
{
var block = (Block)node;
Stmts.Add(block.Expand(Ctx.SpinOff()));
}
else if (node is Break)
{
Stmts.Add(node);
}
else if (node is Continue)
{
Stmts.Add(node);
}
else if (node is Goto)
{
Stmts.Add(node);
}
else if (node is Label)
{
Stmts.Add(node);
}
else if (node is If)
{
var @if = (If)node;
var test = @if.Test.Expand(Ctx);
test.Stmts.ForEach(Stmts.Add);
test.Result.AssertNotNull();
var if_true = @if.IfTrue.Expand(Ctx.SpinOff());
var if_false = @if.IfFalse.Expand(Ctx.SpinOff());
var expanded = new If(test.Result, if_true, if_false);
Stmts.Add(expanded);
}
else if (node is Loop)
{
var loop = (Loop)node;
var test = loop.Test.Expand(Ctx);
test.Result.AssertNotNull();
var init = loop.Init.Expand(Ctx.SpinOff());
var iter = loop.Iter.Expand(Ctx.SpinOff());
var body = loop.Body.Expand(Ctx.SpinOff());
var prepend_test = loop.IsWhileDo && test.Stmts.IsNotEmpty();
if (init.IsNotEmpty() && prepend_test) { Stmts.Add(init); init = new Block(); }
if (prepend_test) test.Stmts.ForEach(Stmts.Add);
test.Stmts.ForEach(iter.Add);
var xloop = new Loop(test.Result, body, loop.IsWhileDo){Init = init, Iter = iter};
var cloned_locals = loop.Locals.Select(l => l.DeepClone());
cloned_locals.ForEach(local => xloop.Locals.Add(local));
Stmts.Add(xloop);
}
else if (node is Return)
{
var ret = (Return)node;
(ret.Value == null).AssertEquiv(Ret == null);
if (ret.Value != null) Expand(new Assign(Ret, ret.Value));
Stmts.Add(new Goto(RetLabel));
}
else if (node is Try || node is Clause || node is Throw ||
node is Using || node is Iter)
{
// todo. implement support for non-linear control flow
// this is only possible when we fully implement decompilation of tries
// until now I leave this marked as "to be implemented"
throw AssertionHelper.Fail();
}
else
{
throw AssertionHelper.Fail();
}
}
[DebuggerNonUserCode] public static Block ExpandBlock(Block source, ExpansionContext ctx) { return new BlockExpander(source, ctx).Stmts; }
public static Block DoRestoreBooleans(Block hir)
{
// todo. also think about how to support nullables
var types = hir.InferTypes();
return (Block)hir.Transform(
(If @if) =>
{
var is_bool = types[@if.Test] == typeof(bool);
var ensuredTest = is_bool ? @if.Test.CurrentTransform() : @if.Test.EnsureBoolean(types);
var ifTrue = @if.IfTrue.CurrentTransform();
var ifFalse = @if.IfFalse.CurrentTransform();
return new If(ensuredTest, ifTrue, ifFalse);
},
(Assign ass) =>
{
if (types[ass.Lhs] == typeof(bool))
{
var lhs = ass.Lhs.CurrentTransform();
var ensuredRhs = ass.Rhs.EnsureBoolean(types);
return new Assign(lhs, ensuredRhs);
}
else
{
return ass.DefaultTransform();
}
},
(Apply app) =>
{
ReadOnlyCollection<Type> targs = null;
if (app.Callee is Prop)
{
var p = app.Callee.InvokedProperty();
if (p != null) targs = p.GetIndexParameters().Select(pi => pi.ParameterType).ToReadOnly();
}
else
{
var ftype = types[app.Callee];
if (ftype != null) targs = ftype.DasmFType().SkipLast(1).ToReadOnly();
}
if (targs == null) return app.DefaultTransform();
var ensuredArgs = targs.Select((targ, i) =>
{
var arg = app.Args[i];
var is_bool = targ == typeof(bool);
return is_bool ? arg.EnsureBoolean(types) : arg.CurrentTransform();
});
var callee = app.Callee.CurrentTransform();
return new Apply(callee, ensuredArgs);
},
(Conditional cond) =>
{
var anyBool = types[cond.IfTrue] == typeof(bool) || types[cond.IfFalse] == typeof(bool);
var is_bool = types[cond.Test] == typeof(bool);
var ensuredTest = is_bool ? cond.Test.CurrentTransform() : cond.Test.EnsureBoolean(types);
var ifTrue = anyBool ? cond.IfTrue.EnsureBoolean(types) : cond.IfTrue.CurrentTransform();
var ifFalse = anyBool ? cond.IfFalse.EnsureBoolean(types) : cond.IfFalse.CurrentTransform();
return new Conditional(ensuredTest, ifTrue, ifFalse);
},
(Operator op) =>
{
var opt = op.OperatorType;
var anyBool = op.Args.Any(arg => types[arg] == typeof(bool));
var alwaysBool = opt == OperatorType.AndAlso ||
opt == OperatorType.OrElse || opt == OperatorType.Not;
var ensureBool = anyBool || alwaysBool;
if (ensureBool)
{
if (opt == OperatorType.And) opt = OperatorType.AndAlso;
if (opt == OperatorType.Or) opt = OperatorType.OrElse;
if (opt == OperatorType.Negate) opt = OperatorType.Not;
var ensuredArgs = op.Args.Select(arg => arg.EnsureBoolean(types));
return Operator.Create(opt, ensuredArgs);
}
else
{
return op.DefaultTransform();
}
});
}
public static void DoInferLoopIters(Block hir)
{
foreach (var loop in hir.Family().OfType<Loop>())
{
if (loop.Iter.IsNotEmpty())
{
loop.Iter.RemoveElements(n => n == null);
}
else
{
if (loop.Init.IsNotEmpty())
{
Func<Expression, bool> isLvalueOfLool = e =>
{
if (e == null)
{
return false;
}
else if (e is Ref)
{
var @ref = e.AssertCast<Ref>();
var local = @ref == null ? null : @ref.Sym as Local;
return local != null && loop.Locals.Contains(local);
}
else if (e is Slot)
{
var slot = e.AssertCast<Slot>();
var @ref = slot == null ? null : slot.This as Ref;
var local = @ref == null ? null : @ref.Sym as Local;
return local != null && loop.Locals.Contains(local);
}
else if (e is Apply)
{
var app = e.AssertCast<Apply>();
var callee = app == null ? null : app.Callee;
var prop = callee as Prop;
var @ref = prop == null ? null : prop.This as Ref;
var local = @ref == null ? null : @ref.Sym as Local;
return local != null && loop.Locals.Contains(local);
}
else
{
return false;
}
};
var lil = loop.Body.LastOrDefault() as Expression;
var lilIsIter = false;
if (lil is Assign)
{
var ass = lil.AssertCast<Assign>();
lilIsIter = isLvalueOfLool(ass.Lhs);
}
else if (lil is Operator)
{
var op = lil.AssertCast<Operator>();
if (op.OperatorType.IsAssign())
{
lilIsIter = isLvalueOfLool(op.Args.First());
}
}
else if (lil is Eval)
{
var m = lil.InvokedMethod();
if (m != null)
{
var args = lil.InvocationArgs();
if (m.IsInstance() || m.IsExtension())
{
lilIsIter = isLvalueOfLool(args.FirstOrDefault());
}
else
{
lilIsIter = args.Any(isLvalueOfLool);
}
}
}
if (lilIsIter)
{
loop.Iter = new Block();
loop.Iter.Add(lil);
loop.Body.Remove(lil);
}
}
}
}
}
// todo. verify that init and iter can live outside the thread loop (when insideThreadLoop is false)
private void TransformLoop(Loop loop, Block xregion, bool insideThreadLoop)
{
var x_init = new Block(); TransformBlock(loop.Init, x_init, true);
var x_test = TransformExpression(loop.Test, insideThreadLoop);
var x_body = new Block(); TransformBlock(loop.Body, x_body, insideThreadLoop);
var x_iter = new Block(); TransformBlock(loop.Iter, x_iter, true);
var x_loop = new Loop(x_test, x_body);
x_loop.Locals.SetElements(loop.Locals);
x_loop.Init = new Block(x_init.Children);
x_loop.Iter = new Block(x_iter.Children);
xregion.Add(x_loop);
}
private void TransformIf(If @if, Block xregion, bool insideThreadLoop)
{
var x_test = TransformExpression(@if.Test, insideThreadLoop);
var x_iftrue = new Block(); TransformBlock(@if.IfTrue, x_iftrue, insideThreadLoop);
var x_iffalse = new Block(); TransformBlock(@if.IfFalse, x_iffalse, insideThreadLoop);
xregion.Add(new If(x_test, x_iftrue, x_iffalse));
}
private void TransformExpression(Expression e, Block xregion, bool insideThreadLoop)
{
var xe = TransformExpression(e, insideThreadLoop);
if (xe != null) xregion.Add(xe);
}
protected override void TraverseBlock(Block block)
{
Func<Type, Type> safeType = t => t.IsRectMdArray() ? typeof(Object) : t;
block.Locals.ForEach(l => locals.Add(l, il.DeclareLocal(safeType(l.Type.AssertNotNull()))));
block.ForEach(Traverse);
}
protected internal override void TraverseBlock(Block block)
{
block.ForEach(Traverse);
Types.Add(block, null);
}
protected internal virtual void TraverseBlock(Block block) { block.Unsupported(); }
protected internal virtual Node TransformBlock(Block block) { return block.AcceptTransformer(this, true); }