public async Task<IEnumerable<Prediction>> ClassifyAsync(byte[] bytes)
        {
            var mappedByteBuffer = GetModelAsMappedByteBuffer();

            //var interpreter = new Xamarin.TensorFlow.Lite.Interpreter(mappedByteBuffer);

            System.Console.WriteLine($"Running Tensorflow interpreter");
            System.Console.WriteLine($"Tensorflow runtime version {TensorFlowLite.RuntimeVersion()}");
            System.Console.WriteLine($"Tensorflow schema version {TensorFlowLite.SchemaVersion()}");
            
            var interpreterOptions = new Interpreter.Options();
            //TODO: Pass from UI?
            var numThreads = 1;
            interpreterOptions.SetNumThreads(numThreads);
            //TODO: Look into use of GPU delegate vs NNAPI
            // https://developer.android.com/ndk/guides/neuralnetworks
            interpreterOptions.SetUseNNAPI(true);
            interpreterOptions.SetAllowFp16PrecisionForFp32(true);

            //var interpreter = new Interpreter(mappedByteBuffer);
            var interpreter = new Interpreter(mappedByteBuffer, interpreterOptions);

            var tensor = interpreter.GetInputTensor(0);

            var shape = tensor.Shape();

            var width = shape[1];
            var height = shape[2];

            var labels = await LoadLabelsAsync(LabelsFileName);
            var byteBuffer = GetPhotoAsByteBuffer(bytes, width, height);

            //var outputLocations = new float[1][] { new float[labels.Count] };
            var outputLocations = new[] { new float[labels.Count] };

            var outputs = Java.Lang.Object.FromArray(outputLocations);

            interpreter.Run(byteBuffer, outputs);

            var classificationResult = outputs.ToArray<float[]>();

            var result = new List<Prediction>();

            for (var i = 0; i < labels.Count; i++)
            {
                var label = labels[i];
                result.Add(new Prediction(label, classificationResult[0][i]));
            }

            //TODO: Consider using this event or MediatR to return results to view model
            //https://blog.duijzer.com/posts/mvvmcross_with_mediatr/
            PredictionCompleted?.Invoke(this, new PredictionCompletedEventArgs(result));

            return result;
        }
Exemplo n.º 2
0
    void Start()
    {
        var options = new Interpreter.Options()
        {
            threads     = 2,
            gpuDelegate = null,
        };

        interpreter = new Interpreter(FileUtil.LoadFile(fileName), options);
        interpreter.ResizeInputTensor(0, new int[] { 1, 28, 28, 1 });
        interpreter.AllocateTensors();

        inputBuffer = new ComputeBuffer(28 * 28, sizeof(float));
    }
        private void LoadTensorflowModel()
        {
            var assets = Android.App.Application.Context.Assets;
            AssetFileDescriptor fileDescriptor = assets.OpenFd(MODEL_FILE);
            FileInputStream     inputStream    = new FileInputStream(fileDescriptor.FileDescriptor);
            FileChannel         fileChannel    = inputStream.Channel;
            long startOffset    = fileDescriptor.StartOffset;
            long declaredLength = fileDescriptor.DeclaredLength;

            _model = fileChannel.Map(FileChannel.MapMode.ReadOnly, startOffset, declaredLength);
            Interpreter.Options options = new Interpreter.Options();
            options.SetNumThreads(THREAD_NUM);
            _interpreter = new Interpreter(_model, options);
        }
    void Start()
    {
        // recursively add children to list
        componentList.Add(character);
        AddDescendants(character.transform, componentList);

        // options
        var options = new Interpreter.Options()
        {
            threads     = 2,
            gpuDelegate = CreateGpuDelegate()// useGPU ? CreateGpuDelegate() : null,
        };

        // load decoder
        string path = Path.Combine(Application.streamingAssetsPath, tfliteFileName);

        Debug.Log(path);
        interpreter = new Interpreter(FileUtil.LoadFile(path), options);

        Debug.LogFormat(
            "InputCount: {0}, OutputCount: {1}",
            interpreter.GetInputTensorCount(), // always 1,1
            interpreter.GetOutputTensorCount()
            );

        // add variable for input size
        inputs = new float[2];

        // load mean features from file
        //path = Path.Combine(Application.streamingAssetsPath, vectorFileName);
        //string[] numbers = File.ReadAllLines(path);
        TextAsset meanfile = Resources.Load(vectorFileName) as TextAsset;

        string[] numbers = meanfile.text.Split(new string[] { "\n" }, StringSplitOptions.None);
        vmean = new float[componentList.Count * 4 + 3];
        for (int i = 0; i < numbers.Length; i++)
        {
            if (float.TryParse(numbers[i], out float parsedValue))
            {
                vmean[i] = parsedValue;
            }
        }
    }
Exemplo n.º 5
0
    //bool _isStoppedDrag = true;
    //bool _hasPrinted = false;

    public void Start()
    {
        shouldClearText = true;

        var options = new Interpreter.Options()
        {
            threads     = 2,
            gpuDelegate = null,
        };

        interpreter = new Interpreter(FileUtil.LoadFile(fileName), options);
        interpreter.ResizeInputTensor(0, new int[] { 1, 28, 28, 1 });
        interpreter.AllocateTensors();

        inputBuffer = new ComputeBuffer(28 * 28, sizeof(float));

        //_hasPrinted = false;

        //GameObject inputImage = GameObject.Find("Input Image");
        //LineDrawer lineDrawer = inputImage.GetComponent<LineDrawer>();
        //_isStoppedDrag = lineDrawer.isStoppedDrag;
    }
Exemplo n.º 6
0
    void Start()
    {
        Debug.LogFormat("TensorFlow Lite Verion: {0}", Interpreter.GetVersion());

        var options = new Interpreter.Options()
        {
            threads = 2,
        };

        interpreter = new Interpreter(model.bytes, options);

        int inputCount  = interpreter.GetInputTensorCount();
        int outputCount = interpreter.GetOutputTensorCount();

        for (int i = 0; i < inputCount; i++)
        {
            Debug.LogFormat("Input {0}: {1}", i, interpreter.GetInputTensorInfo(i));
        }
        for (int i = 0; i < inputCount; i++)
        {
            Debug.LogFormat("Output {0}: {1}", i, interpreter.GetOutputTensorInfo(i));
        }
    }
Exemplo n.º 7
0
        public async Task <PredictionResult> PredictAsync(Stream stream)
        {
            //https://www.tensorflow.org/lite/models/image_classification/android
            //Code https://github.com/tensorflow/examples/blob/master/lite/examples/image_classification/android/app/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java

            //Kotlin example https://medium.com/@teresa.wu/tensorflow-image-recognition-on-android-with-kotlin-cee8d977ae9
            // - code at https://github.com/teresawu/random

            //var tf = new Tensor();
            _labels = _labels ?? (_labels = await LoadLabels(LabelsFileName));
            //_model = _model ?? (_model = await LoadModel(ModelFileName));
            //_modelFile = _modelFile ?? (_modelFile = await LoadModelFile(ModelFileName));
            _modelBuffer = _modelBuffer ?? (_modelBuffer = await LoadModelFileBuffer(ModelFileName));
            //var device = Android.App.Application.Context.Dev
            //Xamarin.T.TensorFlow.Lite.
            //var x = new TensorFlowInferenceInterface()
            //TODO: Look into GPU - GpuHelperDelegate g;

            //TODO: Pass from UI?
            var numThreads = 1;

            var tfOptions = new Interpreter.Options();

            tfOptions.SetNumThreads(numThreads);
            //tfOptions.SetUseNNAPI(true);
            tfOptions.SetAllowFp16PrecisionForFp32(true);

            //TODO: Look into use of GPU delegate vs NNAPI
            // https://developer.android.com/ndk/guides/neuralnetworks

            //https://stackoverflow.com/questions/30716027/c-sharp-equivalent-of-java-memory-mapping-methods
            //ByteBuffer.FromArray()
            //var buff = new ByteBuffer()
            var interpreter = new Interpreter(_modelBuffer, tfOptions);

            //var imgData =  new int[ImageWidth * ImageHeight];
            //bitmap.GetPixels(argbPixelArray, 0, bitmap.Width, 0, 0, bitmap.Width, bitmap.Height);
            var bitmap  = CreateBitmap(stream);
            var imgData = ConvertBitmapToByteBuffer(bitmap);

            //var normalizedPixelComponents = new float[argbPixelArray.Length * 3];

            //var arr = new float[1, _labels.Count];
            //var arr1 = new float[_labels.Count];
            //var labelProbArray = new float[_labels.Count];
            //var objectArray = new object[1];
            var labelProbArray = new float[1][] {
                new float[_labels.Count]
            };

            //https://forums.xamarin.com/discussion/1930/creating-arrays-of-custom-java-objects-in-jni
            //var labelProbArray = Java.Lang.Object.FromArray<IJavaObject>();
            //var javaProbArray = Java.Lang.Object.FromArray(labelArray);
            //var javaObjectArray = Java.Lang.Object.FromArray(objectArray);
            var javaLabelProbArray = Java.Lang.Object.FromArray(labelProbArray);
            //var labelProbArray = Java.Lang.Object.To(labelArray);

            var javaLabelProbArray1 = Java.Lang.Object.FromArray(labelProbArray[0]);
            //var javaLabelProbArray1 = new
            var javaProbArrayContainer = new Java.Lang.Object[] { javaLabelProbArray1 };

            var floatArrayArray = new Java.Lang.Object(JNIEnv.NewArray(labelProbArray), JniHandleOwnership.TransferLocalRef);

            //var array = JNIEnv.NewArray(arr);
            //Java.Lang.Object[] values = arr
            //    .Select(b => new Java.Lang.Object(b.ToLocalJniHandle(), JniHandleOwnership.TransferLocalRef))
            //    .ToArray();

            //var intPtrFloatArray = JNIEnv.NewArray<float>(labelProbArray);
            //var labelProbArrayContainer = new Java.Lang.Object[] { javaLabelProbArray };

            //values2[0] = labelProbArray;

            //Array<ByteArray>

            try
            {
                // Run the inference
                //Try RunForMultipleInputsOutputs
                //    ?? GetOutputTensor
                //  - see https://github.com/tensorflow/tensorflow/issues/25751
                // or https://devblogs.microsoft.com/xamarin/android-apps-tensorflow/
                interpreter.Run(imgData, floatArrayArray);
            }
            catch (Exception e)
            {
                System.Console.WriteLine(e);
                throw;
            }

            //var outputArrayFirstDimension = javaProbArrayContainer[0];
            //var outputs = outputArrayFirstDimension.ToArray<float[][]>();

            //https://stackoverflow.com/questions/17305522/mono-for-android-binding-jagged-array

            //float[][] payload = (float[][])JNIEnv.GetArray(floatArrayArray, JniHandleOwnership.DoNotTransfer, typeof(float[]));
            //float[][] payload = JNIEnv.GetArray<float[]>(javaLabelProbArray);
            float[][] outputs = JNIEnv.GetArray <float[]>(floatArrayArray.Handle);
            for (int i = 0; i < outputs[0].Length; i++)
            {
                if (!float.IsNaN(outputs[0][i]))
                {
                }

                if (outputs[0][i] > 0)
                {
                }
            }


            /*
             * using (var byteArrayArray = new Java.Lang.Object(JNIEnv.NewArray(data), JniHandleOwnership.TransferLocalRef))
             * {
             *  Console.WriteLine("# jonp [[b: {0}", JNIEnv.GetClassNameFromInstance(byteArrayArray.Handle));
             *  byte[][] data2 = JNIEnv.GetArray<byte[]>(byteArrayArray.Handle);
             */

            //https://stackoverflow.com/questions/6594250/type-cast-from-java-lang-object-to-native-clr-type-in-monodroid
            var obj          = javaLabelProbArray;
            var propertyInfo = obj.GetType().GetProperty("Instance");
            var result       = propertyInfo == null
                ? null
                : propertyInfo.GetValue(obj, null) as float[][];

            //var outputs = javaLabelProbArray.ToArray<float[][]>();


            // Find the best classifications.
            var recognitions = new List <Recognition>(_labels.Count);

            for (var i = 0; i < _labels.Count; i++)
            {
                recognitions.Add(new Recognition
                                 (
                                     i.ToString(),
                                     _labels[i],
                                     labelProbArray[0][i],
                                     null
                                 ));
            }
            // Sort high-to-low via confidence
            //Array.Sort(results, (x, y) => y.Confidence.CompareTo(x.Confidence));

            var orderedRecognitions = recognitions.OrderByDescending(x => x.Confidence).ToList();

            foreach (var recognition in orderedRecognitions)
            {
                System.Console.WriteLine($"Result {recognition.Title} with confidence {recognition.Confidence}");
            }

            /*
             * var pq =
             *  new PriorityQueue<Recognition>(
             *      3,
             *      new Comparator<Recognition>() {
             *          @Override
             *          public int compare(Recognition lhs, Recognition rhs)
             *          {
             *          // Intentionally reversed to put high confidence at the head of the queue.
             *          return Float.compare(rhs.getConfidence(), lhs.getConfidence());
             *      }
             * });
             * for (int i = 0; i < labels.size(); ++i) {
             *  pq.add(
             *      new Recognition(
             *          "" + i,
             *          labels.size() > i? labels.get(i) : "unknown",
             *          getNormalizedProbability(i),
             *          null));
             * }
             * final ArrayList<Recognition> recognitions = new ArrayList<Recognition>();
             * int recognitionsSize = Math.min(pq.size(), MAX_RESULTS);
             * for (int i = 0; i<recognitionsSize; ++i) {
             *  recognitions.add(pq.poll());
             * }
             */

            return
                (await Task.FromResult(
                     new PredictionResult
            {
                Success = true,
                Answer = $"{orderedRecognitions.First().Title} with confidence {orderedRecognitions.First().Confidence:0.##}"
            }));
        }