private static string StringMap(Model.Input inp)
{
string s = "";
for (int i = 0; i < s.Length; i++)
{
s = s + s[i] + " ";
}
return(s);
}
public void ModelBuilderTest()
{
TensorCachingAllocator tca = new TensorCachingAllocator();
ModelBuilder mb = new ModelBuilder();
Model.Input inputLayer = mb.Input("Input", new int[] { -1, 1, 1, 1 });
Layer prevLayer = null;
prevLayer = mb.Dense(MultiLayerPerception.LayerNames.Hidden, inputLayer, tca.Alloc(new TensorShape(1, 1)), tca.Alloc(new TensorShape(1, 1)));
prevLayer.weights[0] = 1;
prevLayer.weights[1] = 1;
Debug.Log(prevLayer.weights.Length + ": " + string.Join(",", prevLayer.weights));
for (int i = 0; i < prevLayer.datasets.Length; i++)
{
Debug.Log(prevLayer.datasets[i].name + ":" + prevLayer.datasets[i].offset);
}
prevLayer = mb.Identity("hiddenAct", prevLayer);
Debug.Log(prevLayer.weights.Length + ": " + string.Join(",", prevLayer.weights));
prevLayer = mb.Dense("output", prevLayer, tca.Alloc(new TensorShape(1, 1)), tca.Alloc(new TensorShape(1, 1)));
prevLayer.weights[0] = 3;
prevLayer.weights[1] = 5;
Debug.Log(prevLayer.weights.Length + ": " + string.Join(",", prevLayer.weights));
prevLayer = mb.Identity("outputActive", prevLayer);
Debug.Log(prevLayer.weights.Length + ": " + string.Join(",", prevLayer.weights));
mb.Output(prevLayer);
IWorker worker = WorkerFactory.CreateWorker(mb.model, WorkerFactory.Device.GPU);
Tensor input = tca.Alloc(new TensorShape(4, 1, 1, 1));
for (int i = 0; i < 4; i++)
{
input[i] = i;
}
IWorker ex = worker.Execute(input);
ex.FlushSchedule(true);
Tensor output = ex.PeekOutput();
for (int i = 0; i < 4; i++)
{
Debug.Log($"output[{i}] = {output[i]}");
}
tca.Dispose();
ex.Dispose();
worker.Dispose();
Debug.Assert(true); // Just getting here is good enough
}
private void CreateBarracudaWorker()
{
int savedAlphaBetasIndex = 0;
model = ModelLoader.Load(nnModel, verbose);
layerNameToPatch = new List <string>();
List <Layer> layerList = new List <Layer>(model.layers);
// Pre-process Network for run-time use
Layer lastConv = null;
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
// Remove Style_Prediction_Network: constant with style, executed once in Setup()
if (layer.name.Contains("Style_Prediction_Network/"))
{
layerList.RemoveAt(i);
i--;
continue;
}
// Fix Upsample2D size parameters
if (layer.type == Layer.Type.Upsample2D)
{
layer.pool = new[] { 2, 2 };
//ref model is supposed to be nearest sampling but bilinear scale better when network is applied at lower resoltions
bool useBilinearUpsampling = internalSetup.forceBilinearUpsample2DInModel || (modelToUse != UsedModel.Reference);
layer.axis = useBilinearUpsampling ? 1 : -1;
}
// Remove Mirror padding layers (not supported, TODO)
if (layer.name.Contains("reflect_padding"))
{
layerList[i + 1].inputs = layer.inputs;
layerList[i + 1].pad = layer.pad.ToArray();
layerList.RemoveAt(i);
i--;
}
else if (layer.type == Layer.Type.Conv2D || layer.type == Layer.Type.Conv2DTrans)
{
lastConv = layer;
}
else if (layer.type == Layer.Type.Normalization)
{
// Manually set alpha/betas from Style_Prediction_Network as scale/bias tensors for InstanceNormalization
if (layerList[i - 1].type == Layer.Type.StridedSlice)
{
int channels = predictionAlphasBetasData[savedAlphaBetasIndex].Length;
layer.datasets = new Layer.DataSet[2];
layer.datasets[0].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[0].offset = 0;
layer.datasets[0].length = channels;
layer.datasets[1].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[1].offset = channels;
layer.datasets[1].length = channels;
layerNameToPatch.Add(layer.name);
float[] data = new float[channels * 2];
for (int j = 0; j < data.Length / 2; j++)
{
data[j] = predictionAlphasBetasData[savedAlphaBetasIndex][j];
}
for (int j = data.Length / 2; j < data.Length; j++)
{
data[j] = predictionAlphasBetasData[savedAlphaBetasIndex + 1][j - data.Length / 2];
}
layer.weights = data;
savedAlphaBetasIndex += 2;
}
// Else initialize scale/bias tensors of InstanceNormalization to default 1/0
else
{
int channels = lastConv.datasets[1].shape.channels;
layer.datasets = new Layer.DataSet[2];
layer.datasets[0].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[0].offset = 0;
layer.datasets[0].length = channels;
layer.datasets[1].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[1].offset = channels;
layer.datasets[1].length = channels;
float[] data = new float[channels * 2];
for (int j = 0; j < data.Length / 2; j++)
{
data[j] = 1.0f;
}
for (int j = data.Length / 2; j < data.Length; j++)
{
data[j] = 0.0f;
}
layer.weights = data;
}
}
}
// Remove Slice layers originally used to get alpha/beta tensors into Style_Network
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
if (layer.type == Layer.Type.StridedSlice)
{
layerList.RemoveAt(i);
i--;
}
}
// Fold Relu into instance normalisation
Dictionary <string, string> reluToInstNorm = new Dictionary <string, string>();
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
if (layer.type == Layer.Type.Activation && layer.activation == Layer.Activation.Relu)
{
if (layerList[i - 1].type == Layer.Type.Normalization)
{
layerList[i - 1].activation = layer.activation;
reluToInstNorm[layer.name] = layerList[i - 1].name;
layerList.RemoveAt(i);
i--;
}
}
}
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
for (int j = 0; j < layer.inputs.Length; j++)
{
if (reluToInstNorm.ContainsKey(layer.inputs[j]))
{
layer.inputs[j] = reluToInstNorm[layer.inputs[j]];
}
}
}
// Feed first convolution directly with input (no need for normalisation from the model)
string firstConvName = "StyleNetwork/conv1/convolution_conv1/convolution";
int firstConv = FindLayerIndexByName(layerList, firstConvName);
layerList[firstConv].inputs = new[] { model.inputs[1].name };
if (modelToUse == UsedModel.Reference)
{
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/add"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/add/y"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/normalized_contentFrames"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/normalized_contentFrames/y"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/sub"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/sub/y"));
}
if (modelToUse == UsedModel.RefBut32Channels)
{
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalized_contentFrames"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalized_contentFrames/y"));
}
// Remove final model post processing, post process happen in tensor to texture instead
int postAdd = FindLayerIndexByName(layerList, "StyleNetwork/clamp_0_255/add");
layerList.RemoveRange(postAdd, 5);
// Correct wrong output layer list
model.outputs = new List <string>()
{
layerList[postAdd - 1].name
};
model.layers = layerList;
Model.Input input = model.inputs[1];
input.shape[0] = 0;
input.shape[1] = 1080;//TODO get framebuffer size rather than hardcoded value
input.shape[2] = 1920;
input.shape[3] = 3;
model.inputs = new List <Model.Input> {
model.inputs[1]
};
//Create worker and execute it once at target resolution to prime all memory allocation (however in editor resolution can still change at runtime)
worker = WorkerFactory.CreateWorker(WorkerFactory.ValidateType(internalSetup.workerType), model, verbose);
Dictionary <string, Tensor> temp = new Dictionary <string, Tensor>();
var inputTensor = new Tensor(input.shape, input.name);
temp.Add("frame", inputTensor);
worker.Execute(temp);
inputTensor.Dispose();
}
