public static IOps CreateOps(WorkerFactory.Type type, bool verbose = false)
{
WorkerFactory.ValidateType(type);
switch (type)
{
case WorkerFactory.Type.ComputePrecompiled:
return new PrecompiledComputeOps(ComputeShaderSingleton.Instance.kernels,
ComputeShaderSingleton.Instance.referenceKernels, verbose: verbose);
case WorkerFactory.Type.Compute:
return new ComputeOps(ComputeShaderSingleton.Instance.kernels,
ComputeShaderSingleton.Instance.referenceKernels, verbose: verbose);
case WorkerFactory.Type.ComputeRef:
return new ReferenceComputeOps(ComputeShaderSingleton.Instance.referenceKernels);
case WorkerFactory.Type.CSharp:
return new UnsafeArrayCPUOps();
default:
return new ReferenceCPUOps();
}
}
/// <summary>
/// Updates the output layer names based on the selected model architecture
/// and initializes the Barracuda inference engine witht the selected model.
/// </summary>
private void InitializeBarracuda()
{
// The compiled model used for performing inference
Model m_RunTimeModel;
if (modelType == ModelType.MobileNet)
{
preProcessFunction = Utils.PreprocessMobileNet;
// Compile the model asset into an object oriented representation
m_RunTimeModel = ModelLoader.Load(mobileNetModelAsset);
displacementFWDLayer = m_RunTimeModel.outputs[2];
displacementBWDLayer = m_RunTimeModel.outputs[3];
}
else
{
preProcessFunction = Utils.PreprocessResNet;
// Compile the model asset into an object oriented representation
m_RunTimeModel = ModelLoader.Load(resnetModelAsset);
displacementFWDLayer = m_RunTimeModel.outputs[3];
displacementBWDLayer = m_RunTimeModel.outputs[2];
}
heatmapLayer = m_RunTimeModel.outputs[0];
offsetsLayer = m_RunTimeModel.outputs[1];
// Create a model builder to modify the m_RunTimeModel
ModelBuilder modelBuilder = new ModelBuilder(m_RunTimeModel);
// Add a new Sigmoid layer that takes the output of the heatmap layer
modelBuilder.Sigmoid(predictionLayer, heatmapLayer);
// Validate if backend is supported, otherwise use fallback type.
workerType = WorkerFactory.ValidateType(workerType);
// Create a worker that will execute the model with the selected backend
engine = new Engine(workerType, modelBuilder.model, modelType);
}
private void CreateBarracudaWorker()
{
if (_styleTexture == null || _predictionAlphasBetasData == null)
{
return;
}
int savedAlphaBetasIndex = 0;
_layerNameToPatch = new List <string>();
List <Layer> layerList = new List <Layer>(_model.layers);
// Pre-process Network for run-time use
Layer lastConv = null;
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
// Remove Style_Prediction_Network: constant with style, executed once in Setup()
if (layer.name.Contains("Style_Prediction_Network/"))
{
layerList.RemoveAt(i);
i--;
continue;
}
// Fix Upsample2D size parameters
if (layer.type == Layer.Type.Upsample2D)
{
layer.pool = new[] { 2, 2 };
//ref model is supposed to be nearest sampling but bilinear scale better when network is applied at lower resoltions
bool useBilinearUpsampling = forceBilinearUpsample2DInModel.value || (modelType.value != ModelType.Reference);
layer.axis = useBilinearUpsampling ? 1 : -1;
}
// Remove Mirror padding layers (not supported, TODO)
if (layer.name.Contains("reflect_padding"))
{
layerList[i + 1].inputs = layer.inputs;
layerList[i + 1].pad = layer.pad.ToArray();
layerList.RemoveAt(i);
i--;
}
else if (layer.type == Layer.Type.Conv2D || layer.type == Layer.Type.Conv2DTrans)
{
lastConv = layer;
}
else if (layer.type == Layer.Type.Normalization)
{
// Manually set alpha/betas from Style_Prediction_Network as scale/bias tensors for InstanceNormalization
if (layerList[i - 1].type == Layer.Type.StridedSlice)
{
int channels = _predictionAlphasBetasData[savedAlphaBetasIndex].Length;
layer.datasets = new Layer.DataSet[2];
layer.datasets[0].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[0].offset = 0;
layer.datasets[0].length = channels;
layer.datasets[1].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[1].offset = channels;
layer.datasets[1].length = channels;
_layerNameToPatch.Add(layer.name);
float[] data = new float[channels * 2];
for (int j = 0; j < data.Length / 2; j++)
{
data[j] = _predictionAlphasBetasData[savedAlphaBetasIndex][j];
}
for (int j = data.Length / 2; j < data.Length; j++)
{
data[j] = _predictionAlphasBetasData[savedAlphaBetasIndex + 1][j - data.Length / 2];
}
layer.weights = new BarracudaArrayFromManagedArray(data);
savedAlphaBetasIndex += 2;
}
// Else initialize scale/bias tensors of InstanceNormalization to default 1/0
else
{
int channels = lastConv.datasets[1].shape.channels;
layer.datasets = new Layer.DataSet[2];
layer.datasets[0].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[0].offset = 0;
layer.datasets[0].length = channels;
layer.datasets[1].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[1].offset = channels;
layer.datasets[1].length = channels;
float[] data = new float[channels * 2];
for (int j = 0; j < data.Length / 2; j++)
{
data[j] = 1.0f;
}
for (int j = data.Length / 2; j < data.Length; j++)
{
data[j] = 0.0f;
}
layer.weights = new BarracudaArrayFromManagedArray(data);
}
}
}
// Remove Slice layers originally used to get alpha/beta tensors into Style_Network
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
if (layer.type == Layer.Type.StridedSlice)
{
layerList.RemoveAt(i);
i--;
}
}
// Fold Relu into instance normalisation
Dictionary <string, string> reluToInstNorm = new Dictionary <string, string>();
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
if (layer.type == Layer.Type.Activation && layer.activation == Layer.Activation.Relu)
{
if (layerList[i - 1].type == Layer.Type.Normalization)
{
layerList[i - 1].activation = layer.activation;
reluToInstNorm[layer.name] = layerList[i - 1].name;
layerList.RemoveAt(i);
i--;
}
}
}
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
for (int j = 0; j < layer.inputs.Length; j++)
{
if (reluToInstNorm.ContainsKey(layer.inputs[j]))
{
layer.inputs[j] = reluToInstNorm[layer.inputs[j]];
}
}
}
// Feed first convolution directly with input (no need for normalisation from the model)
string firstConvName = "StyleNetwork/conv1/convolution_conv1/convolution";
int firstConv = FindLayerIndexByName(layerList, firstConvName);
layerList[firstConv].inputs = new[] { _model.inputs[1].name };
if (modelType.value == ModelType.Reference)
{
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/add"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/add/y"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/normalized_contentFrames"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/normalized_contentFrames/y"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/sub"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalisation/sub/y"));
}
if (modelType.value == ModelType.RefBut32Channels)
{
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalized_contentFrames"));
layerList.RemoveAt(FindLayerIndexByName(layerList, "StyleNetwork/normalized_contentFrames/y"));
}
// Remove final model post processing, post process happen in tensor to texture instead
int postAdd = FindLayerIndexByName(layerList, "StyleNetwork/clamp_0_255/add");
layerList.RemoveRange(postAdd, 5);
// Correct wrong output layer list
_model.outputs = new List <string>()
{
layerList[postAdd - 1].name
};
_model.layers = layerList;
Model.Input input = _model.inputs[1];
input.shape[0] = 0;
input.shape[1] = _rtHandle.rt.height;
input.shape[2] = _rtHandle.rt.width;
input.shape[3] = 3;
_model.inputs = new List <Model.Input> {
_model.inputs[1]
};
//Create worker and execute it once at target resolution to prime all memory allocation (however in editor resolution can still change at runtime)
_worker = WorkerFactory.CreateWorker(WorkerFactory.ValidateType(workerType.value), _model, debugModelLoading.value);
Dictionary <string, Tensor> temp = new Dictionary <string, Tensor>();
var inputTensor = new Tensor(input.shape, input.name);
temp.Add("frame", inputTensor);
_worker.Execute(temp);
inputTensor.Dispose();
Debug.Log("Style Transfer Model: \n" + _model.ToString());
}
// https://github.com/JasonMa0012/barracuda-style-transfer/blob/fb61d2d8172e3150f6ebbfb78443d0fe9def66db/Assets/BarracudaStyleTransfer/BarracudaStyleTransfer.cs#L575
private void PrepareStylePrediction()
{
if (_styleTexture == null)
{
return;
}
Model tempModel = ModelLoader.Load(_nnModels[(int)modelType.value], debugModelLoading.value); //_model.ShallowCopy();
List <Layer> predictionAlphasBetas = new List <Layer>();
List <Layer> layerList = new List <Layer>(tempModel.layers);
// Remove Divide by 255, Unity textures are in [0, 1] already
int firstDivide = FindLayerIndexByName(layerList, "Style_Prediction_Network/normalized_image");
if (firstDivide < 0)
{
Debug.Log(0);
}
layerList[firstDivide + 1].inputs[0] = layerList[firstDivide].inputs[0];
layerList.RemoveAt(firstDivide);
// Pre-process network to get it to run and extract Style alpha/beta tensors
Layer lastConv = null;
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
// Remove Mirror padding layers (not supported, TODO)
if (layer.name.Contains("reflect_padding"))
{
layerList[i + 1].inputs = layer.inputs;
layerList[i + 1].pad = layer.pad.ToArray();
layerList.RemoveAt(i);
i--;
continue;
}
// Placeholder instance norm bias + scale tensors
if (layer.type == Layer.Type.Conv2D || layer.type == Layer.Type.Conv2DTrans)
{
lastConv = layer;
}
else if (layer.type == Layer.Type.Normalization)
{
int channels = lastConv.datasets[1].shape.channels;
layer.datasets = new Layer.DataSet[2];
layer.datasets[0].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[0].offset = 0;
layer.datasets[0].length = channels;
layer.datasets[1].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[1].offset = channels;
layer.datasets[1].length = channels;
float[] data = new float[channels * 2];
for (int j = 0; j < data.Length / 2; j++)
{
data[j] = 1.0f;
}
for (int j = data.Length / 2; j < data.Length; j++)
{
data[j] = 0.0f;
}
layer.weights = new BarracudaArrayFromManagedArray(data);
}
if (layer.type != Layer.Type.StridedSlice && layer.name.Contains("StyleNetwork/"))
{
layerList.RemoveAt(i);
i--;
}
if (layer.type == Layer.Type.StridedSlice)
{
predictionAlphasBetas.Add(layer);
}
}
tempModel.layers = layerList;
// Run Style_Prediction_Network on given style
var styleInput = new Tensor(_styleTexture);
Dictionary <string, Tensor> temp = new Dictionary <string, Tensor>();
temp.Add("frame", styleInput);
temp.Add("style", styleInput);
IWorker tempWorker = WorkerFactory.CreateWorker(WorkerFactory.ValidateType(workerType.value), tempModel, debugModelLoading.value);
tempWorker.Execute(temp);
// Store alpha/beta tensors from Style_Prediction_Network to feed into the run-time network
_predictionAlphasBetasData = new List <float[]>();
foreach (var layer in predictionAlphasBetas)
{
_predictionAlphasBetasData.Add(tempWorker.PeekOutput(layer.name).ToReadOnlyArray());
}
tempWorker.Dispose();
styleInput.Dispose();
Debug.Log("Style Prediction Model: \n" + tempModel.ToString());
}
private void PrepareStylePrediction()
{
//Try to load style tensors from disk first
StyleData loadedData = Resources.Load <StyleData>(GetStyleResourceName());
if (loadedData)
{
predictionAlphasBetasData = new List <float[]>();
for (int i = 0; i < loadedData.layerData.Count; i++)
{
predictionAlphasBetasData.Add(loadedData.layerData[i].data);
}
Resources.UnloadAsset(loadedData);
return;
}
//Tensors not found on disk, compute them (and eventually store them on disk)
Model tempModel = ModelLoader.Load(nnModel, verbose);
List <Layer> predictionAlphasBetas = new List <Layer>();
List <Layer> layerList = new List <Layer>(tempModel.layers);
// Remove Divide by 255, Unity textures are in [0, 1] already
int firstDivide = FindLayerIndexByName(layerList, "Style_Prediction_Network/normalized_image");
layerList[firstDivide + 1].inputs[0] = layerList[firstDivide].inputs[0];
layerList.RemoveAt(firstDivide);
// Pre-process network to get it to run and extract Style alpha/beta tensors
Layer lastConv = null;
for (int i = 0; i < layerList.Count; i++)
{
Layer layer = layerList[i];
// Remove Mirror padding layers (not supported, TODO)
if (layer.name.Contains("reflect_padding"))
{
layerList[i + 1].inputs = layer.inputs;
layerList[i + 1].pad = layer.pad.ToArray();
layerList.RemoveAt(i);
i--;
continue;
}
// Placeholder instance norm bias + scale tensors
if (layer.type == Layer.Type.Conv2D || layer.type == Layer.Type.Conv2DTrans)
{
lastConv = layer;
}
else if (layer.type == Layer.Type.Normalization)
{
int channels = lastConv.datasets[1].shape.channels;
layer.datasets = new Layer.DataSet[2];
layer.datasets[0].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[0].offset = 0;
layer.datasets[0].length = channels;
layer.datasets[1].shape = new TensorShape(1, 1, 1, channels);
layer.datasets[1].offset = channels;
layer.datasets[1].length = channels;
float[] data = new float[channels * 2];
for (int j = 0; j < data.Length / 2; j++)
{
data[j] = 1.0f;
}
for (int j = data.Length / 2; j < data.Length; j++)
{
data[j] = 0.0f;
}
layer.weights = data;
}
if (layer.type != Layer.Type.StridedSlice && layer.name.Contains("StyleNetwork/"))
{
layerList.RemoveAt(i);
i--;
}
if (layer.type == Layer.Type.StridedSlice)
{
predictionAlphasBetas.Add(layer);
}
}
tempModel.layers = layerList;
// Run Style_Prediction_Network on given style
styleInput = new Tensor(StyleImage);
CustomPinTensorFromTexture(styleInput);
Dictionary <string, Tensor> temp = new Dictionary <string, Tensor>();
temp.Add("frame", styleInput);
temp.Add("style", styleInput);
IWorker tempWorker = WorkerFactory.CreateWorker(WorkerFactory.ValidateType(internalSetup.workerType), tempModel, verbose);
tempWorker.Execute(temp);
// Store alpha/beta tensors from Style_Prediction_Network to feed into the run-time network
predictionAlphasBetasData = new List <float[]>();
for (int i = 0; i < predictionAlphasBetas.Count; i++)
{
Tensor O = tempWorker.PeekOutput(predictionAlphasBetas[i].name);
predictionAlphasBetasData.Add(new float[O.length]);
for (int j = 0; j < O.length; j++)
{
predictionAlphasBetasData[i][j] = O[j];
}
O.Dispose();
}
tempWorker.Dispose();
#if UNITY_EDITOR
//Store to disk
if (internalSetup.shouldSaveStyleTransferDataAsAsset)
{
SaveStyleTransferDataAsAsset();
}
#endif
}
