Exemplo n.º 1
0
        private static void PackConstants(Model model, Dictionary <string, Layer> constantLayers)
        {
            for (int l = 0; l < model.layers.Count; ++l)
            {
                var layer = model.layers[l];

                if (!LinearLayerFusing.IsLayerLinearMathOp(layer))
                {
                    continue;
                }
                var constInputs = layer.inputs.Count(x => constantLayers.ContainsKey(x));
                // @TODO fuse multi const inputs here
                if (!(layer.inputs.Length == 2 && constInputs == 1))
                {
                    continue;
                }

                var constInput = layer.inputs.ToList().Find(x => constantLayers.ContainsKey(x));

                layer.datasets = new Layer.DataSet[constantLayers[constInput].datasets.Length];
                Array.Copy(constantLayers[constInput].datasets, layer.datasets, constantLayers[constInput].datasets.Length);
                layer.weights = new BarracudaArray(constantLayers[constInput].weights.Length);
                BarracudaArray.Copy(constantLayers[constInput].weights, layer.weights, constantLayers[constInput].weights.Length);

                model.layers[l].inputs = layer.inputs.Where(x => x != constInput).ToArray();
            }
        }
Exemplo n.º 2
0
        public static void BakeConstantWRBIntoLSTMLayer(Layer layer, Tensor W, Tensor R, Tensor B)
        {
            string name = layer.name;

            // Bake out constant tensors into layer
            void AddDataset(List <Layer.DataSet> datasets, BarracudaArray weights, string tensorName, Tensor t, ref int offset)
            {
                var dataset = new Layer.DataSet();

                dataset.name            = $"{name}/{tensorName}";
                dataset.shape           = t.shape;
                dataset.itemSizeInBytes = 4;
                dataset.length          = t.shape.length;
                dataset.offset          = offset;
                datasets.Add(dataset);

                t.ToReadOnlyArray().CopyToBarracudaArray(weights, offset);

                offset += t.shape.length;
            }

            var layerDatasets = new List <Layer.DataSet>();
            var layerWeights  = new BarracudaArray(W.shape.length + R.shape.length + B.shape.length);
            int dataOffset    = 0;

            var ops = new ReferenceCPUOps();

            using (var td = new TensorScope())
            {
                TensorScope.F _ = td._;

                Tensor[] w_iofj, r_iofj, wb_iofj, rb_iofj;
                SplitWRBForLSTM(ops, W, R, B, out w_iofj, out r_iofj, out wb_iofj, out rb_iofj);

                var indexName = new[] { "i", "o", "f", "j" };

                for (int i = 0; i < w_iofj.Length; i++)
                {
                    AddDataset(layerDatasets, layerWeights, $"w_{indexName[i]}", _(w_iofj[i]), ref dataOffset);
                }

                for (int i = 0; i < w_iofj.Length; i++)
                {
                    AddDataset(layerDatasets, layerWeights, $"r_{indexName[i]}", _(r_iofj[i]), ref dataOffset);
                }

                for (int i = 0; i < w_iofj.Length; i++)
                {
                    AddDataset(layerDatasets, layerWeights, $"wb_{indexName[i]}", _(wb_iofj[i]), ref dataOffset);
                }

                for (int i = 0; i < w_iofj.Length; i++)
                {
                    AddDataset(layerDatasets, layerWeights, $"rb_{indexName[i]}", _(rb_iofj[i]), ref dataOffset);
                }
            }

            layer.datasets = layerDatasets.ToArray();
            layer.weights  = layerWeights;
        }
Exemplo n.º 3
0
        /// <summary>
        /// Converts Tensor to DataSet
        /// </summary>
        /// <param name="X">input `Tensor`</param>
        /// <param name="index">dataset index</param>
        public void ApplyTensorToDataSet(Tensor X, int index)
        {
            Assert.IsTrue(index < datasets.Length);
            var ds = datasets[index];

            ds.shape = X.shape;
            BarracudaArray.Copy(X.ToReadOnlyArray(), 0, weights, ds.offset, ds.shape.length);
            datasets[index] = ds;
        }
Exemplo n.º 4
0
 public void Allocate(int numElement, DataType dataType, int alignment, Allocator allocator)
 {
     m_ReadFence  = new JobHandle();
     m_WriteFence = new JobHandle();
     elementCount = numElement;
     elementSize  = BarracudaArray.DataItemSize(dataType);
     type         = dataType;
     Assert.IsTrue(data == null, "Please call ClearState() when freeing underlying memory.");
     Assert.IsTrue(alignment % elementSize == 0);
     data = UnsafeUtility.Malloc(elementCount * elementSize, alignment, allocator);
     Assert.IsTrue(data != null);
 }
Exemplo n.º 5
0
 /// <summary>
 /// Convert in place all model weights to given data type
 /// </summary>
 /// <param name="type">target type for moodel weights</param>
 internal void ConvertWeights(DataType type)
 {
     foreach (var layer in layers)
     {
         if (layer.weights != null && layer.weights.Type != type)
         {
             var sourceWeights = layer.weights;
             var targetWeights = new BarracudaArray(layer.weights.Length, type);
             BarracudaArray.Copy(sourceWeights, targetWeights);
             layer.weights = targetWeights;
         }
     }
 }
Exemplo n.º 6
0
 private Layer(string layerName)
 {
     name       = layerName;
     type       = Type.Nop;
     activation = Activation.None;
     pad        = new int[0];
     stride     = new int[0];
     pool       = new int[0];
     axis       = -1;
     alpha      = 1.0f;
     beta       = 0.0f;
     inputs     = new string[0];
     datasets   = new DataSet[0];
     weights    = new BarracudaArray(0);//TODO fp16?
 }
Exemplo n.º 7
0
        private static void UnpackConstants(Model model)
        {
            List <Layer> newConstants = new List <Layer>();

            for (int l = 0; l < model.layers.Count; ++l)
            {
                var layer = model.layers[l];
                if (!LinearLayerFusing.IsLayerLinearMathOp(layer))
                {
                    continue;
                }

                if (layer.datasets == null || layer.datasets.Length != 1)
                {
                    continue;
                }

                var   name       = "c" + layer.name;
                Layer constInput = new Layer(name, Layer.Type.Load);

                constInput.datasets = new Layer.DataSet[layer.datasets.Length];
                Array.Copy(layer.datasets, constInput.datasets, layer.datasets.Length);
                for (int d = 0; d < constInput.datasets.Length; ++d)
                {
                    constInput.datasets[d].name = name;
                }

                constInput.weights = new BarracudaArray(layer.weights.Length);
                BarracudaArray.Copy(layer.weights, constInput.weights, layer.weights.Length);

                Array.Resize(ref layer.inputs, layer.inputs.Length + 1);
                layer.inputs[layer.inputs.Length - 1] = constInput.name;

                newConstants.Add(constInput);

                layer.datasets = new Layer.DataSet[0];
                layer.weights  = new BarracudaArray(0);//TODO fp16
            }
            newConstants.AddRange(model.layers);
            model.layers = newConstants;
        }
Exemplo n.º 8
0
        /// <summary>
        /// Save model to file
        /// </summary>
        /// <param name="writer">`BinaryWriter`</param>
        /// <param name="model">`Model`</param>
        /// <param name="verbose">verbose flag</param>
        public static void Save(BinaryWriter writer, Model model, bool verbose = false)
        {
            Profiler.BeginSample("Barracuda.ModelWriter.Save");

            writer.Write((long)Model.Version);

            writer.Write(model.inputs.Count);
            for (var i = 0; i < model.inputs.Count; ++i)
            {
                WriteString(writer, model.inputs[i].name);
                WriteInt32Array(writer, model.inputs[i].shape);
            }
            WriteStringArray(writer, model.outputs);

            writer.Write(model.memories.Count);
            for (var m = 0; m < model.memories.Count; ++m)
            {
                WriteInt32Array(writer, model.memories[m].shape.ToArray());
                WriteString(writer, model.memories[m].input);
                WriteString(writer, model.memories[m].output);
            }

            // Write layers
            long offsetFromModelStartToLayer = 0;

            writer.Write(model.layers.Count);
            for (var l = 0; l < model.layers.Count; ++l)
            {
                Layer layer = model.layers[l];
                WriteString(writer, layer.name);
                writer.Write((Int32)layer.type);
                writer.Write((Int32)layer.activation);
                writer.Write(0); //dummy 0 size array
                writer.Write(0); //dummy 0 size array
                WriteInt32Array(writer, layer.pad);
                WriteInt32Array(writer, layer.stride);
                WriteInt32Array(writer, layer.pool);
                writer.Write(layer.axis);
                writer.Write(layer.alpha);
                writer.Write(layer.beta);
                writer.Write(0); //dummy 0 size array

                WriteStringArray(writer, layer.inputs);

                long offsetFromLayerStart = 0;
                writer.Write(layer.datasets.Length);
                for (var i = 0; i < layer.datasets.Length; ++i)
                {
                    WriteString(writer, layer.datasets[i].name);
                    WriteInt32Array(writer, layer.datasets[i].shape.ToArray());
                    // Recalculate all offsets to be global inside the model
                    // this way weights can be stored in one block at the end of the file
                    Assert.AreEqual(offsetFromLayerStart, layer.datasets[i].offset - layer.datasets[0].offset);
                    writer.Write(offsetFromModelStartToLayer + offsetFromLayerStart);
                    writer.Write(layer.datasets[i].itemSizeInBytes);
                    writer.Write(layer.datasets[i].length);
                    offsetFromLayerStart += layer.datasets[i].length;
                }
                offsetFromModelStartToLayer += offsetFromLayerStart;

                if (verbose)
                {
                    D.Log("layer " + l + ", " + layer.name + " type: " + layer.type.ToString() +
                          ((layer.activation != Layer.Activation.None) ? " activation " + layer.activation : "") +
                          " tensors: " + layer.datasets.Length +
                          " inputs: " + String.Join(",", layer.inputs));
                }

                if (verbose)
                {
                    foreach (var t in layer.datasets)
                    {
                        D.Log("        Tensor: " + t.shape + " offset: " + t.offset + " len: " + t.length);
                    }
                }
            }

            //Version 20 introduce weights type but full model need to be in the same type. Per layer no supported yet.
            Assert.IsTrue(model.layers.Count >= 0);
            var weightsDataType = model.layers[0].weights.Type;
            var sizeOfDataItem  = BarracudaArray.DataItemSize(weightsDataType);

            writer.Write((int)weightsDataType);

            //Pad to 4 bytes
            long writerCurrentPosition = writer.BaseStream.Position;
            long paddingForAlignment   = Model.WeightsAlignment - (writerCurrentPosition % Model.WeightsAlignment);

            writer.Write(new byte[paddingForAlignment]);

            // Write tensor data
            for (var l = 0; l < model.layers.Count; ++l)
            {
                for (var d = 0; d < model.layers[l].datasets.Length; ++d)
                {
                    Assert.AreEqual(weightsDataType, model.layers[0].weights.Type);
                    byte[] dst = new byte[model.layers[l].datasets[d].length * sizeOfDataItem];
                    BarracudaArray.BlockCopy(model.layers[l].weights, (int)(model.layers[l].datasets[d].offset * sizeOfDataItem), dst, 0, dst.Length);
                    writer.Write(dst);
                }
            }

            WriteString(writer, model.IrSource);
            WriteString(writer, model.IrVersion);
            WriteString(writer, model.ProducerName);
            int numWarnings = model.Warnings.Count;

            writer.Write(numWarnings);
            for (var i = 0; i < numWarnings; ++i)
            {
                WriteString(writer, model.Warnings[i].LayerName);
                WriteString(writer, model.Warnings[i].Message);
            }

            int numMetadataProps = model.Metadata.Count;

            writer.Write(numMetadataProps);
            foreach (KeyValuePair <string, string> kvp in model.Metadata)
            {
                WriteString(writer, kvp.Key);
                WriteString(writer, kvp.Value);
            }

            Profiler.EndSample();
        }
Exemplo n.º 9
0
        private static BarracudaArray ReadLargeWeightArray(BinaryReader file, Int64 count, DataType dataType)
        {
            int   bytesToRead;
            Int64 bytesToReadInt64 = count * BarracudaArray.DataItemSize(dataType);

            try
            {
                bytesToRead = Convert.ToInt32(bytesToReadInt64); // throws OverflowException
            }
            catch (OverflowException)
            {
                throw new OverflowException($"Files larger than 2GB currently are not supported. Attempt to read {bytesToReadInt64} bytes.");
            }

            //1-Try to remap byte[] stream to avoid allocation
            Profiler.BeginSample("Barracuda.RemapWeights");
            BarracudaArray remappedWeights = null;

            try
            {
                Stream stream          = file.BaseStream;
                var    memoryStream    = stream as MemoryStream;
                var    sourceBuffer    = memoryStream?.GetBuffer();
                int    currentPosition = (int)memoryStream?.Position;
                remappedWeights = new BarracudaArrayFromManagedArray(sourceBuffer, currentPosition, dataType, (int)count);
            }
        #if UNITY_EDITOR
            catch (InvalidOperationException e)
            {
                UnityEngine.Debug.Log("ModelLoader: Can't remap memory stream to underlying data type, allocation and copy will occurs. Exception: " + e);
            }
        #else
            catch (InvalidOperationException) {}
        #endif
            if (remappedWeights != null)
            {
                //We remapped memory. Need to advance stream position to be consistent with read behavior.
                file.BaseStream.Position += bytesToRead;
                Profiler.EndSample();
                return(remappedWeights);
            }
            Profiler.EndSample();

            //2-Can't remap will copy from managed memory to native
            Profiler.BeginSample("Barracuda.AllocWeights");
            BarracudaArray loadedWeights = new BarracudaArray((int)count, dataType);
            Profiler.EndSample();

            Profiler.BeginSample("Barracuda.LoadWeights");
            try
            {
                var readBuffer = new byte[4096]; // 4Kb is close to optimal read size.
                                                 // See for measurements: https://www.jacksondunstan.com/articles/3568
                                                 // Read size vs relative read-time:
                                                 // 64b: x10, 128b: x6, 256b: x4, 1Kb: x3, 4Kb: x3
                int writeOffset = 0;
                while (writeOffset < bytesToRead)
                {
                    var bytesLeftToRead = bytesToRead - writeOffset;
                    var readSizeInBytes = Math.Min(readBuffer.Length, bytesLeftToRead);

                    Assert.IsTrue(readSizeInBytes > 0);
                    Assert.IsTrue(readSizeInBytes <= readBuffer.Length);
                    readSizeInBytes = file.BaseStream.Read(readBuffer, offset: 0, count: readSizeInBytes);
                    if (readSizeInBytes == 0)
                    {
                        throw new IOException($"Unexpected EOF reached. Read {writeOffset / sizeof(float)} out of expected {count} floats before reaching end of file.");
                    }

                    BarracudaArray.BlockCopy(
                        sourceArray: readBuffer, sourceByteOffset: 0,
                        destinationArray: loadedWeights, destinationByteOffset: writeOffset,
                        lengthInBytes: readSizeInBytes);
                    writeOffset += readSizeInBytes;
                }
                Assert.AreEqual(writeOffset, bytesToRead);
            }
            finally
            {
                Profiler.EndSample();
            }

            return(loadedWeights);
        }
Exemplo n.º 10
0
        static IEnumerator LoadAsync(BinaryReader fileReader, Model model, bool verbose = true, bool applyPatching = true, bool skipWeights = false, float maxTimePerYield = 0f)
        {
            using (BinaryReader file = fileReader)
            {
                Profiler.BeginSample("Barracuda.LoadLayers");
                float timeStart = Time.realtimeSinceStartup;

                if (model == null)
                {
                    model = new Model();
                }
                List <Layer> layers = new List <Layer>();

                long version = file.ReadInt64() % 0xff; // magic
                if (version != Model.Version && version != Model.LastVersionWithout8DSupport && version != Model.LastVersionWithoutWeightsAlignmentSupport)
                {
                    throw new NotSupportedException($"Format version not supported: {version}");
                }

                var count = file.ReadInt32();
                model.inputs = new List <Model.Input>(count);
                for (var i = 0; i < count; ++i)
                {
                    model.inputs.Add(new Model.Input {
                        name = ReadString(file), shape = ReadInt32Array(file)
                    });

                    if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
                    {
#if DEBUG_TIMING
                        UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
                        yield return(null);

                        timeStart = Time.realtimeSinceStartup;
                    }
                }

                model.outputs = ReadStringArray(file).ToList();

                count          = file.ReadInt32();
                model.memories = new List <Model.Memory>(count);
                for (var m = 0; m < count; ++m)
                {
                    model.memories.Add(new Model.Memory
                    {
                        shape  = new TensorShape(ReadInt32Array(file)),
                        input  = ReadString(file),
                        output = ReadString(file)
                    });

                    if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
                    {
#if DEBUG_TIMING
                        UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
                        yield return(null);

                        timeStart = Time.realtimeSinceStartup;
                    }
                }

                int numberOfLayers = file.ReadInt32();
                for (var l = 0; l < numberOfLayers; ++l)
                {
                    var   name       = ReadString(file);
                    var   layerType  = (Layer.Type)file.ReadInt32();
                    var   activation = (Layer.Activation)file.ReadInt32();
                    Layer layer      = new Layer(name, layerType, activation);
                    ReadInt32Array(file);                   // dummy
                    ReadInt32Array(file);                   // dummy
                    layer.pad    = ReadInt32Array(file);
                    layer.stride = ReadInt32Array(file);
                    layer.pool   = ReadInt32Array(file);
                    layer.axis   = ConvertLayerAxisFor8DShapeSupportIfNeeded(file.ReadInt32(), version, layerType);
                    layer.alpha  = file.ReadSingle();
                    layer.beta   = file.ReadSingle();
                    ReadInt32Array(file);                   // dummy

                    layer.inputs = ReadStringArray(file);

                    if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
                    {
#if DEBUG_TIMING
                        UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
                        yield return(null);

                        timeStart = Time.realtimeSinceStartup;
                    }

                    layer.datasets = new Layer.DataSet[file.ReadInt32()];
                    for (var i = 0; i < layer.datasets.Length; ++i)
                    {
                        if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
                        {
#if DEBUG_TIMING
                            UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
                            yield return(null);

                            timeStart = Time.realtimeSinceStartup;
                        }

                        layer.datasets[i].name            = ReadString(file);
                        layer.datasets[i].shape           = new TensorShape(ReadInt32Array(file));
                        layer.datasets[i].offset          = file.ReadInt64();
                        layer.datasets[i].itemSizeInBytes = file.ReadInt32();
                        layer.datasets[i].length          = file.ReadInt32();
                    }

                    layers.Add(layer);

                    if (verbose)
                    {
                        D.Log(
                            $"layer {l}, {layer.name} type: {layer.type} " +
                            $"{((layer.activation != Layer.Activation.None) ? $"activation {layer.activation} " : "")}" +
                            $"tensors: {layer.datasets.Length} inputs: {String.Join(",", layer.inputs)}");
                    }

                    if (verbose)
                    {
                        foreach (var t in layer.datasets)
                        {
                            D.Log($"        Tensor: {t.shape} offset: {t.offset} len: {t.length}");
                        }
                    }

                    if (applyPatching)
                    {
                        PatchLayer(layers, layer);
                    }

                    if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
                    {
#if DEBUG_TIMING
                        UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart + ": " + l);
#endif
                        yield return(null);

                        timeStart = Time.realtimeSinceStartup;
                    }
                }
                model.layers = layers;

                Int64 numWeightsToRead = 0;
                for (var l = 0; l < model.layers.Count; ++l)
                {
                    for (var d = 0; d < model.layers[l].datasets.Length; ++d)
                    {
                        numWeightsToRead += model.layers[l].datasets[d].length;

                        if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
                        {
#if DEBUG_TIMING
                            UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
                            yield return(null);

                            timeStart = Time.realtimeSinceStartup;
                        }
                    }
                }

                Profiler.EndSample();

                DataType weightsDataType = DataType.Float;
                if (version >= 20)
                {
                    //Version 20 introduce weights type but full model need to be in the same type. Per layer no supported yet.
                    weightsDataType = (DataType)file.ReadInt32();
                }

                if (version >= 19)
                {
                    //Padding so weights are aligned on Model.WeightsAlignment bytes
                    long streamCurrentPosition = file.BaseStream.Position;
                    long paddingForAlignment   = Model.WeightsAlignment - (streamCurrentPosition % Model.WeightsAlignment);
                    file.BaseStream.Seek(paddingForAlignment, SeekOrigin.Current);
                }

                if (skipWeights)
                {
                    SkipLargeByteArray(file, numWeightsToRead * BarracudaArray.DataItemSize(weightsDataType));
                }
                else
                {
                    if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
                    {
#if DEBUG_TIMING
                        UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
                        yield return(null);

                        timeStart = Time.realtimeSinceStartup;
                    }

                    var sharedWeightsArray = ReadLargeWeightArray(file, numWeightsToRead, weightsDataType);

                    Assert.AreEqual(weightsDataType, sharedWeightsArray.Type);
                    for (var l = 0; l < model.layers.Count; ++l)
                    {
                        model.layers[l].weights = sharedWeightsArray;

                        if (maxTimePerYield > 0 && Time.realtimeSinceStartup - timeStart > maxTimePerYield)
                        {
#if DEBUG_TIMING
                            UnityEngine.Debug.Log(Time.realtimeSinceStartup - timeStart);
#endif
                            yield return(null);

                            timeStart = Time.realtimeSinceStartup;
                        }
                    }
                }

                // Importer Reporting
                try
                {
                    model.IrSource     = ReadString(file);
                    model.IrVersion    = ReadString(file);
                    model.ProducerName = ReadString(file);
                    int numWarnings = file.ReadInt32();
                    for (var i = 0; i < numWarnings; ++i)
                    {
                        model.Warnings.Add(new Model.ImporterWarning(ReadString(file), ReadString(file)));
                    }

                    if (version >= 18)
                    {
                        int numMetadataProps = file.ReadInt32();
                        for (var i = 0; i < numMetadataProps; ++i)
                        {
                            model.Metadata.Add(ReadString(file), ReadString(file));
                        }
                    }
                }
                catch (EndOfStreamException)
                {
                    //Do nothing Importer Reporting data might not be present for backward compatibility reasons
                }

                yield return(model);
            }
        }