public void Infer(layers.Quantize layer, QuantizeLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From];
var outputAlloc = context.MainMemoryMap[layer.Output];
argument.MainMemoryInputAddress = inputAlloc.GetAddress();
argument.MainMemoryOutputAddress = outputAlloc.GetAddress();
}
public void Infer(K210Upload layer, K210UploadLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From];
var outputAlloc = context.KPUMemoryMap[layer.Output];
argument.MainMemoryInputAddress = inputAlloc.GetAddress();
argument.KPUMemoryOutputAddress = outputAlloc.GetAddress();
}
public void Infer(AveragePool2d layer, AveragePool2dLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From];
var outputAlloc = context.MainMemoryMap[layer.Output];
argument.Flags = K210LayerFlags.MainMemoryOutput;
argument.MainMemoryInputAddress = inputAlloc.GetAddress();
argument.MainMemoryOutputAddress = outputAlloc.GetAddress();
}
public void Infer(QuantizedAdd layer, QuantizedAddLayerArgument argument, InferenceContext context)
{
var inputAAlloc = context.MainMemoryMap[layer.InputA.Connection.From];
var inputBAlloc = context.MainMemoryMap[layer.InputB.Connection.From];
var outputAlloc = context.MainMemoryMap[layer.Output];
argument.Flags = K210LayerFlags.MainMemoryOutput;
argument.MainMemoryInputAAddress = inputAAlloc.GetAddress();
argument.MainMemoryInputBAddress = inputBAlloc.GetAddress();
argument.MainMemoryOutputAddress = outputAlloc.GetAddress();
}
public void Infer(Concatenation layer, ConcatenationLayerArgument argument, InferenceContext context)
{
var outputAlloc = context.MainMemoryMap[layer.Output];
argument.Flags = K210LayerFlags.MainMemoryOutput;
argument.MainMemoryOutputAddress = outputAlloc.GetAddress();
argument.InputsMainMemory = (from i in layer.Inputs
let a = context.MainMemoryMap[i.Connection.From]
select new MemoryRange
{
Start = a.GetAddress(),
Size = a.Size
}).ToList();
}
public void Infer(K210Conv2d layer, K210Conv2dLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.KPUMemoryMap[layer.Input.Connection.From];
MemoryAllocation outputAlloc;
argument.Config.InputAddress = inputAlloc.GetAddress();
if (context.MainMemoryMap.TryGetValue(layer.Output, out var mainAlloc))
{
argument.Flags = K210LayerFlags.MainMemoryOutput;
argument.MainMemoryOutputAddress = mainAlloc.GetAddress();
outputAlloc = context.GetOrAllocateKPUMemory(layer.Output);
}
else
{
argument.Flags = K210LayerFlags.None;
outputAlloc = context.KPUMemoryMap[layer.Output];
}
argument.Config.OutputAddress = outputAlloc.GetAddress();
}
public void AllocateInputMemory(Reshape layer, OutputConnector input, InferenceContext context)
{
context.MainMemoryMap.Add(layer.Output, context.GetOrAllocateMainMemory(layer.Input.Connection.From));
}
public void Infer(ResizeNearestNeighbor layer, ResizeNearestNeighborLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From];
var outputAlloc = context.MainMemoryMap[layer.Output];
argument.MainMemoryInputAddress = inputAlloc.GetAddress();
argument.MainMemoryOutputAddress = outputAlloc.GetAddress();
}
public void Infer(TensorflowFlatten layer, TensorflowFlattenLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From];
var outputAlloc = context.MainMemoryMap[layer.Output];
argument.Flags = K210LayerFlags.MainMemoryOutput;
argument.MainMemoryInputAddress = inputAlloc.GetAddress();
argument.MainMemoryOutputAddress = outputAlloc.GetAddress();
}
public static K210BinGenerationContext GenerateBin(Graph graph, Stream stream, int weightsBits, string prefix, InferenceContext inferenceContext)
{
var context = new K210BinGenerationContext
{
Prefix = prefix,
MaxStartAddress = inferenceContext.KPUMemoryAllocator.MaxStart,
MainMemoryUsage = inferenceContext.MainMemoryAllocator.MaxEnd,
Outputs = (from o in graph.Outputs
let m = inferenceContext.MainMemoryMap[o.Input.Connection.From]
select new K210OutputAddress
{
Address = m.GetAddress(),
Size = (uint)o.Input.Dimensions.GetSize() * 4
}).ToList(),
Stream = stream,
WeightsBits = weightsBits
};
var converters = (from t in typeof(Generator).Assembly.ExportedTypes
let attrs = t.GetCustomAttributes <LayerConverterAttribute>()
where attrs.Any()
from attr in attrs
where attr.LayerType != K210LayerType.Invalid
select new
{
Key = attr.LayerType,
Value = new { Type = t, Method = t.GetMethod("GenerateBin") }
}).ToDictionary(x => x.Key, x => x.Value);
var layers = inferenceContext.InferenceOrders;
var bw = new BinaryWriter(stream);
void GenerateBinLayerBody(K210Layer layer)
{
var type = layer.Header.Type;
if (converters.TryGetValue(type, out var info))
{
if (info.Method != null)
{
var converter = Activator.CreateInstance(info.Type);
info.Method.Invoke(converter, new object[] { bw, layer.Body, context });
}
else
{
GenerateBinDefault(bw, layer.Body);
}
}
else
{
throw new LayerNotSupportedException(type.ToString());
}
context.AlignStreamPosition(8);
}
uint version = 3;
uint flags = weightsBits == 8 ? 1u : 0u;
bw.Write(version);
bw.Write(flags);
bw.Write(0);
bw.Write(layers.Count);
bw.Write(context.MaxStartAddress);
bw.Write(context.MainMemoryUsage);
// Outputs
bw.Write(context.Outputs.Count);
foreach (var output in context.Outputs)
{
bw.Write(output.Address);
bw.Write(output.Size);
}
// Headers
var fixPosition = bw.BaseStream.Position;
bw.BaseStream.Position += 4 * 2 * layers.Count;
for (int i = 0; i < layers.Count; i++)
{
var layer = layers[i];
// BodySize
var beginPosition = bw.BaseStream.Position;
GenerateBinLayerBody(layer);
layer.Header.BodySize = (uint)(bw.BaseStream.Position - beginPosition);
}
var newPosition = bw.BaseStream.Position;
bw.BaseStream.Position = fixPosition;
for (int i = 0; i < layers.Count; i++)
{
var header = layers[i].Header;
bw.Write((uint)header.Type);
bw.Write((uint)header.BodySize);
}
bw.BaseStream.Position = newPosition;
return(context);
}
public void Infer(K210AddPadding layer, K210AddPaddingLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From];
var outputAlloc = context.KPUMemoryMap[layer.Output];
argument.Flags = K210LayerFlags.None;
argument.MainMemoryInputAddress = inputAlloc.GetAddress();
argument.KPUMemoryOutputAddress = outputAlloc.GetAddress();
}
public void AllocateInputMemory(QuantizedExclusiveConcatenation layer, OutputConnector input, InferenceContext context)
{
var totalAlloc = context.GetOrAllocateMainMemory(layer.Output);
uint offset = 0;
foreach (var node in layer.Inputs.Select(x => x.Connection.From))
{
if (context.MainMemoryMap.ContainsKey(node))
{
return;
}
uint size = (uint)node.Dimensions.GetSize();
context.MainMemoryMap.Add(node, new MemoryAllocation(totalAlloc.Node, offset, size));
offset += size;
}
}
public void Infer(ChannelwiseDequantize layer, ChannelwiseDequantizeLayerArgument argument, InferenceContext context)
{
var inputAlloc = context.MainMemoryMap[layer.Input.Connection.From];
var outputAlloc = context.MainMemoryMap[layer.Output];
argument.Flags = K210LayerFlags.MainMemoryOutput;
argument.MainMemoryInputAddress = inputAlloc.GetAddress();
argument.MainMemoryOutputAddress = outputAlloc.GetAddress();
}
