private static void PackConstants(Model model, Dictionary <string, Layer> constantLayers)
{
for (int l = 0; l < model.layers.Count; ++l)
{
var layer = model.layers[l];
if (!LinearLayerFusing.IsLayerLinearMathOp(layer))
{
continue;
}
var constInputs = layer.inputs.Count(x => constantLayers.ContainsKey(x));
// @TODO fuse multi const inputs here
if (!(layer.inputs.Length == 2 && constInputs == 1))
{
continue;
}
var constInput = layer.inputs.ToList().Find(x => constantLayers.ContainsKey(x));
layer.datasets = new Layer.DataSet[constantLayers[constInput].datasets.Length];
Array.Copy(constantLayers[constInput].datasets, layer.datasets, constantLayers[constInput].datasets.Length);
layer.weights = new BarracudaArray(constantLayers[constInput].weights.Length);
BarracudaArray.Copy(constantLayers[constInput].weights, layer.weights, constantLayers[constInput].weights.Length);
model.layers[l].inputs = layer.inputs.Where(x => x != constInput).ToArray();
}
}
public static void BakeConstantWRBIntoLSTMLayer(Layer layer, Tensor W, Tensor R, Tensor B)
{
string name = layer.name;
// Bake out constant tensors into layer
void AddDataset(List <Layer.DataSet> datasets, BarracudaArray weights, string tensorName, Tensor t, ref int offset)
{
var dataset = new Layer.DataSet();
dataset.name = $"{name}/{tensorName}";
dataset.shape = t.shape;
dataset.itemSizeInBytes = 4;
dataset.length = t.shape.length;
dataset.offset = offset;
datasets.Add(dataset);
t.ToReadOnlyArray().CopyToBarracudaArray(weights, offset);
offset += t.shape.length;
}
var layerDatasets = new List <Layer.DataSet>();
var layerWeights = new BarracudaArray(W.shape.length + R.shape.length + B.shape.length);
int dataOffset = 0;
var ops = new ReferenceCPUOps();
using (var td = new TensorScope())
{
TensorScope.F _ = td._;
Tensor[] w_iofj, r_iofj, wb_iofj, rb_iofj;
SplitWRBForLSTM(ops, W, R, B, out w_iofj, out r_iofj, out wb_iofj, out rb_iofj);
var indexName = new[] { "i", "o", "f", "j" };
for (int i = 0; i < w_iofj.Length; i++)
{
AddDataset(layerDatasets, layerWeights, $"w_{indexName[i]}", _(w_iofj[i]), ref dataOffset);
}
for (int i = 0; i < w_iofj.Length; i++)
{
AddDataset(layerDatasets, layerWeights, $"r_{indexName[i]}", _(r_iofj[i]), ref dataOffset);
}
for (int i = 0; i < w_iofj.Length; i++)
{
AddDataset(layerDatasets, layerWeights, $"wb_{indexName[i]}", _(wb_iofj[i]), ref dataOffset);
}
for (int i = 0; i < w_iofj.Length; i++)
{
AddDataset(layerDatasets, layerWeights, $"rb_{indexName[i]}", _(rb_iofj[i]), ref dataOffset);
}
}
layer.datasets = layerDatasets.ToArray();
layer.weights = layerWeights;
}
/// <summary>
/// Converts Tensor to DataSet
/// </summary>
/// <param name="X">input `Tensor`</param>
/// <param name="index">dataset index</param>
public void ApplyTensorToDataSet(Tensor X, int index)
{
Assert.IsTrue(index < datasets.Length);
var ds = datasets[index];
ds.shape = X.shape;
BarracudaArray.Copy(X.ToReadOnlyArray(), 0, weights, ds.offset, ds.shape.length);
datasets[index] = ds;
}
public void Allocate(int numElement, DataType dataType, int alignment, Allocator allocator)
{
m_ReadFence = new JobHandle();
m_WriteFence = new JobHandle();
elementCount = numElement;
elementSize = BarracudaArray.DataItemSize(dataType);
type = dataType;
Assert.IsTrue(data == null, "Please call ClearState() when freeing underlying memory.");
Assert.IsTrue(alignment % elementSize == 0);
data = UnsafeUtility.Malloc(elementCount * elementSize, alignment, allocator);
Assert.IsTrue(data != null);
}
/// <summary>
/// Convert in place all model weights to given data type
/// </summary>
/// <param name="type">target type for moodel weights</param>
internal void ConvertWeights(DataType type)
{
foreach (var layer in layers)
{
if (layer.weights != null && layer.weights.Type != type)
{
var sourceWeights = layer.weights;
var targetWeights = new BarracudaArray(layer.weights.Length, type);
BarracudaArray.Copy(sourceWeights, targetWeights);
layer.weights = targetWeights;
}
}
}
private Layer(string layerName)
{
name = layerName;
type = Type.Nop;
activation = Activation.None;
pad = new int[0];
stride = new int[0];
pool = new int[0];
axis = -1;
alpha = 1.0f;
beta = 0.0f;
inputs = new string[0];
datasets = new DataSet[0];
weights = new BarracudaArray(0);//TODO fp16?
}
private static void UnpackConstants(Model model)
{
List <Layer> newConstants = new List <Layer>();
for (int l = 0; l < model.layers.Count; ++l)
{
var layer = model.layers[l];
if (!LinearLayerFusing.IsLayerLinearMathOp(layer))
{
continue;
}
if (layer.datasets == null || layer.datasets.Length != 1)
{
continue;
}
var name = "c" + layer.name;
Layer constInput = new Layer(name, Layer.Type.Load);
constInput.datasets = new Layer.DataSet[layer.datasets.Length];
Array.Copy(layer.datasets, constInput.datasets, layer.datasets.Length);
for (int d = 0; d < constInput.datasets.Length; ++d)
{
constInput.datasets[d].name = name;
}
constInput.weights = new BarracudaArray(layer.weights.Length);
BarracudaArray.Copy(layer.weights, constInput.weights, layer.weights.Length);
Array.Resize(ref layer.inputs, layer.inputs.Length + 1);
layer.inputs[layer.inputs.Length - 1] = constInput.name;
newConstants.Add(constInput);
layer.datasets = new Layer.DataSet[0];
layer.weights = new BarracudaArray(0);//TODO fp16
}
newConstants.AddRange(model.layers);
model.layers = newConstants;
}
/// <summary>
/// Save model to file
/// </summary>
/// <param name="writer">`BinaryWriter`</param>
/// <param name="model">`Model`</param>
/// <param name="verbose">verbose flag</param>
public static void Save(BinaryWriter writer, Model model, bool verbose = false)
{
Profiler.BeginSample("Barracuda.ModelWriter.Save");
writer.Write((long)Model.Version);
writer.Write(model.inputs.Count);
for (var i = 0; i < model.inputs.Count; ++i)
{
WriteString(writer, model.inputs[i].name);
WriteInt32Array(writer, model.inputs[i].shape);
}
WriteStringArray(writer, model.outputs);
writer.Write(model.memories.Count);
for (var m = 0; m < model.memories.Count; ++m)
{
WriteInt32Array(writer, model.memories[m].shape.ToArray());
WriteString(writer, model.memories[m].input);
WriteString(writer, model.memories[m].output);
}
// Write layers
long offsetFromModelStartToLayer = 0;
writer.Write(model.layers.Count);
for (var l = 0; l < model.layers.Count; ++l)
{
Layer layer = model.layers[l];
WriteString(writer, layer.name);
writer.Write((Int32)layer.type);
writer.Write((Int32)layer.activation);
writer.Write(0); //dummy 0 size array
writer.Write(0); //dummy 0 size array
WriteInt32Array(writer, layer.pad);
WriteInt32Array(writer, layer.stride);
WriteInt32Array(writer, layer.pool);
writer.Write(layer.axis);
writer.Write(layer.alpha);
writer.Write(layer.beta);
writer.Write(0); //dummy 0 size array
WriteStringArray(writer, layer.inputs);
long offsetFromLayerStart = 0;
writer.Write(layer.datasets.Length);
for (var i = 0; i < layer.datasets.Length; ++i)
{
WriteString(writer, layer.datasets[i].name);
WriteInt32Array(writer, layer.datasets[i].shape.ToArray());
// Recalculate all offsets to be global inside the model
// this way weights can be stored in one block at the end of the file
Assert.AreEqual(offsetFromLayerStart, layer.datasets[i].offset - layer.datasets[0].offset);
writer.Write(offsetFromModelStartToLayer + offsetFromLayerStart);
writer.Write(layer.datasets[i].itemSizeInBytes);
writer.Write(layer.datasets[i].length);
offsetFromLayerStart += layer.datasets[i].length;
}
offsetFromModelStartToLayer += offsetFromLayerStart;
if (verbose)
{
D.Log("layer " + l + ", " + layer.name + " type: " + layer.type.ToString() +
((layer.activation != Layer.Activation.None) ? " activation " + layer.activation : "") +
" tensors: " + layer.datasets.Length +
" inputs: " + String.Join(",", layer.inputs));
}
if (verbose)
{
foreach (var t in layer.datasets)
{
D.Log(" Tensor: " + t.shape + " offset: " + t.offset + " len: " + t.length);
}
}
}
//Version 20 introduce weights type but full model need to be in the same type. Per layer no supported yet.
Assert.IsTrue(model.layers.Count >= 0);
var weightsDataType = model.layers[0].weights.Type;
var sizeOfDataItem = BarracudaArray.DataItemSize(weightsDataType);
writer.Write((int)weightsDataType);
//Pad to 4 bytes
long writerCurrentPosition = writer.BaseStream.Position;
long paddingForAlignment = Model.WeightsAlignment - (writerCurrentPosition % Model.WeightsAlignment);
writer.Write(new byte[paddingForAlignment]);
// Write tensor data
for (var l = 0; l < model.layers.Count; ++l)
{
for (var d = 0; d < model.layers[l].datasets.Length; ++d)
{
Assert.AreEqual(weightsDataType, model.layers[0].weights.Type);
byte[] dst = new byte[model.layers[l].datasets[d].length * sizeOfDataItem];
BarracudaArray.BlockCopy(model.layers[l].weights, (int)(model.layers[l].datasets[d].offset * sizeOfDataItem), dst, 0, dst.Length);
writer.Write(dst);
}
}
WriteString(writer, model.IrSource);
WriteString(writer, model.IrVersion);
WriteString(writer, model.ProducerName);
int numWarnings = model.Warnings.Count;
writer.Write(numWarnings);
for (var i = 0; i < numWarnings; ++i)
{
WriteString(writer, model.Warnings[i].LayerName);
WriteString(writer, model.Warnings[i].Message);
}
int numMetadataProps = model.Metadata.Count;
writer.Write(numMetadataProps);
foreach (KeyValuePair <string, string> kvp in model.Metadata)
{
WriteString(writer, kvp.Key);
WriteString(writer, kvp.Value);
}
Profiler.EndSample();
}
private static BarracudaArray ReadLargeWeightArray(BinaryReader file, Int64 count, DataType dataType)
{
int bytesToRead;
Int64 bytesToReadInt64 = count * BarracudaArray.DataItemSize(dataType);
try
{
bytesToRead = Convert.ToInt32(bytesToReadInt64); // throws OverflowException
}
catch (OverflowException)
{
throw new OverflowException($"Files larger than 2GB currently are not supported. Attempt to read {bytesToReadInt64} bytes.");
}
//1-Try to remap byte[] stream to avoid allocation
Profiler.BeginSample("Barracuda.RemapWeights");
BarracudaArray remappedWeights = null;
try
{
Stream stream = file.BaseStream;
var memoryStream = stream as MemoryStream;
var sourceBuffer = memoryStream?.GetBuffer();
int currentPosition = (int)memoryStream?.Position;
remappedWeights = new BarracudaArrayFromManagedArray(sourceBuffer, currentPosition, dataType, (int)count);
}
#if UNITY_EDITOR
catch (InvalidOperationException e)
{
UnityEngine.Debug.Log("ModelLoader: Can't remap memory stream to underlying data type, allocation and copy will occurs. Exception: " + e);
}
#else
catch (InvalidOperationException) {}
#endif
if (remappedWeights != null)
{
//We remapped memory. Need to advance stream position to be consistent with read behavior.
file.BaseStream.Position += bytesToRead;
Profiler.EndSample();
return(remappedWeights);
}
Profiler.EndSample();
//2-Can't remap will copy from managed memory to native
Profiler.BeginSample("Barracuda.AllocWeights");
BarracudaArray loadedWeights = new BarracudaArray((int)count, dataType);
Profiler.EndSample();
Profiler.BeginSample("Barracuda.LoadWeights");
try
{
var readBuffer = new byte[4096]; // 4Kb is close to optimal read size.
// See for measurements: https://www.jacksondunstan.com/articles/3568
// Read size vs relative read-time:
// 64b: x10, 128b: x6, 256b: x4, 1Kb: x3, 4Kb: x3
int writeOffset = 0;
while (writeOffset < bytesToRead)
{
var bytesLeftToRead = bytesToRead - writeOffset;
var readSizeInBytes = Math.Min(readBuffer.Length, bytesLeftToRead);
Assert.IsTrue(readSizeInBytes > 0);
Assert.IsTrue(readSizeInBytes <= readBuffer.Length);
readSizeInBytes = file.BaseStream.Read(readBuffer, offset: 0, count: readSizeInBytes);
if (readSizeInBytes == 0)
{
throw new IOException($"Unexpected EOF reached. Read {writeOffset / sizeof(float)} out of expected {count} floats before reaching end of file.");
}
BarracudaArray.BlockCopy(
sourceArray: readBuffer, sourceByteOffset: 0,
destinationArray: loadedWeights, destinationByteOffset: writeOffset,
lengthInBytes: readSizeInBytes);
writeOffset += readSizeInBytes;
}
Assert.AreEqual(writeOffset, bytesToRead);
}
finally
{
Profiler.EndSample();
}
return(loadedWeights);
}
static IEnumerator LoadAsync(BinaryReader fileReader, Model model, bool verbose = true, bool applyPatching = true, bool skipWeights = false, float maxTimePerYield = 0f)
{
using (BinaryReader file = fileReader)
{
Profiler.BeginSample("Barracuda.LoadLayers");
float timeStart = Time.realtimeSinceStartup;
if (model == null)
{
model = new Model();
}
List <Layer> layers = new List <Layer>();
long version = file.ReadInt64() % 0xff; // magic
if (version != Model.Version && version != Model.LastVersionWithout8DSupport && version != Model.LastVersionWithoutWeightsAlignmentSupport)
{
throw new NotSupportedException($"Format version not supported: {version}");
}
var count = file.ReadInt32();
model.inputs = new List <Model.Input>(count);
for (var i = 0; i < count; ++i)
{
model.inputs.Add(new Model.Input {
name = ReadString(file), shape = ReadInt32Array(file)
});
if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
{
#if DEBUG_TIMING
UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
yield return(null);
timeStart = Time.realtimeSinceStartup;
}
}
model.outputs = ReadStringArray(file).ToList();
count = file.ReadInt32();
model.memories = new List <Model.Memory>(count);
for (var m = 0; m < count; ++m)
{
model.memories.Add(new Model.Memory
{
shape = new TensorShape(ReadInt32Array(file)),
input = ReadString(file),
output = ReadString(file)
});
if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
{
#if DEBUG_TIMING
UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
yield return(null);
timeStart = Time.realtimeSinceStartup;
}
}
int numberOfLayers = file.ReadInt32();
for (var l = 0; l < numberOfLayers; ++l)
{
var name = ReadString(file);
var layerType = (Layer.Type)file.ReadInt32();
var activation = (Layer.Activation)file.ReadInt32();
Layer layer = new Layer(name, layerType, activation);
ReadInt32Array(file); // dummy
ReadInt32Array(file); // dummy
layer.pad = ReadInt32Array(file);
layer.stride = ReadInt32Array(file);
layer.pool = ReadInt32Array(file);
layer.axis = ConvertLayerAxisFor8DShapeSupportIfNeeded(file.ReadInt32(), version, layerType);
layer.alpha = file.ReadSingle();
layer.beta = file.ReadSingle();
ReadInt32Array(file); // dummy
layer.inputs = ReadStringArray(file);
if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
{
#if DEBUG_TIMING
UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
yield return(null);
timeStart = Time.realtimeSinceStartup;
}
layer.datasets = new Layer.DataSet[file.ReadInt32()];
for (var i = 0; i < layer.datasets.Length; ++i)
{
if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
{
#if DEBUG_TIMING
UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
yield return(null);
timeStart = Time.realtimeSinceStartup;
}
layer.datasets[i].name = ReadString(file);
layer.datasets[i].shape = new TensorShape(ReadInt32Array(file));
layer.datasets[i].offset = file.ReadInt64();
layer.datasets[i].itemSizeInBytes = file.ReadInt32();
layer.datasets[i].length = file.ReadInt32();
}
layers.Add(layer);
if (verbose)
{
D.Log(
$"layer {l}, {layer.name} type: {layer.type} " +
$"{((layer.activation != Layer.Activation.None) ? $"activation {layer.activation} " : "")}" +
$"tensors: {layer.datasets.Length} inputs: {String.Join(",", layer.inputs)}");
}
if (verbose)
{
foreach (var t in layer.datasets)
{
D.Log($" Tensor: {t.shape} offset: {t.offset} len: {t.length}");
}
}
if (applyPatching)
{
PatchLayer(layers, layer);
}
if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
{
#if DEBUG_TIMING
UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart + ": " + l);
#endif
yield return(null);
timeStart = Time.realtimeSinceStartup;
}
}
model.layers = layers;
Int64 numWeightsToRead = 0;
for (var l = 0; l < model.layers.Count; ++l)
{
for (var d = 0; d < model.layers[l].datasets.Length; ++d)
{
numWeightsToRead += model.layers[l].datasets[d].length;
if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
{
#if DEBUG_TIMING
UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
yield return(null);
timeStart = Time.realtimeSinceStartup;
}
}
}
Profiler.EndSample();
DataType weightsDataType = DataType.Float;
if (version >= 20)
{
//Version 20 introduce weights type but full model need to be in the same type. Per layer no supported yet.
weightsDataType = (DataType)file.ReadInt32();
}
if (version >= 19)
{
//Padding so weights are aligned on Model.WeightsAlignment bytes
long streamCurrentPosition = file.BaseStream.Position;
long paddingForAlignment = Model.WeightsAlignment - (streamCurrentPosition % Model.WeightsAlignment);
file.BaseStream.Seek(paddingForAlignment, SeekOrigin.Current);
}
if (skipWeights)
{
SkipLargeByteArray(file, numWeightsToRead * BarracudaArray.DataItemSize(weightsDataType));
}
else
{
if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
{
#if DEBUG_TIMING
UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
yield return(null);
timeStart = Time.realtimeSinceStartup;
}
var sharedWeightsArray = ReadLargeWeightArray(file, numWeightsToRead, weightsDataType);
Assert.AreEqual(weightsDataType, sharedWeightsArray.Type);
for (var l = 0; l < model.layers.Count; ++l)
{
model.layers[l].weights = sharedWeightsArray;
if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
{
#if DEBUG_TIMING
UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
yield return(null);
timeStart = Time.realtimeSinceStartup;
}
}
}
// Importer Reporting
try
{
model.IrSource = ReadString(file);
model.IrVersion = ReadString(file);
model.ProducerName = ReadString(file);
int numWarnings = file.ReadInt32();
for (var i = 0; i < numWarnings; ++i)
{
model.Warnings.Add(new Model.ImporterWarning(ReadString(file), ReadString(file)));
}
if (version >= 18)
{
int numMetadataProps = file.ReadInt32();
for (var i = 0; i < numMetadataProps; ++i)
{
model.Metadata.Add(ReadString(file), ReadString(file));
}
}
}
catch (EndOfStreamException)
{
//Do nothing Importer Reporting data might not be present for backward compatibility reasons
}
yield return(model);
}
}