コード例 #1
0
        /**
         * Train segmentor and recognizer with next participant's data
         */
        void Prepare()
        {
            ResetParticipantsCommands();
            GetNewParticipantSamples();

            machete = new Machete(deviceType, cr_options);
            foreach (Sample sample in trainSets[currentParticipantIndex])
            {
                machete.AddSample(sample, filtered: true);
            }

            blades = new JackknifeBlades();
            blades.SetIPDefaults();
            blades.ResampleCnt = Global.GetResampleCnt(deviceType);

            blades.Radius = (int)(blades.ResampleCnt * .1f);

            blades.LowerBound = false;
            jackknife         = new Jackknife.Jackknife(blades);
            foreach (Jackknife.Sample sample in jkTrainSets[currentParticipantIndex])
            {
                jackknife.AddTemplate(sample);
            }

            jackknife.SetRejectionThresholds(knownThreshold);

            frames = participantsFrames[currentParticipantIndex];

            double cutoff = 3.0;

            if (deviceType == DeviceType.MOUSE)
            {
                cutoff = 5.0;
            }
            ema_filter = new ExponentialMovingAverage(frames[0].pt, cutoff);

            video = new List <Vector>();

            rresults = new List <RecognitionResult>();

            frame_idx = 0;

            currentParticipantID = participants[currentParticipantIndex];

            timer = 0.0;
        }
コード例 #2
0
        void PrepareWindow()
        {
            ResetParticipantsCommands();

            // Set up Jackknife first because window will use it's values
            blades = new JackknifeBlades();
            blades.SetIPDefaults();
            blades.ResampleCnt = Global.GetResampleCnt(deviceType);

            blades.Radius = (int)(blades.ResampleCnt * .1f);

            blades.LowerBound = false;
            jackknife         = new Jackknife.Jackknife(blades);
            foreach (Jackknife.Sample sample in jkTrainSets[currentParticipantIndex])
            {
                jackknife.AddTemplate(sample);
            }

            jackknife.SetRejectionThresholds(knownThreshold);

            frames = participantsFrames[currentParticipantIndex];

            double cutoff = 3.0;

            if (deviceType == DeviceType.MOUSE)
            {
                cutoff = 5.0;
            }
            ema_filter = new ExponentialMovingAverage(frames[0].pt, cutoff);

            video = new List <Vector>();

            rresults = new List <RecognitionResult>();

            frame_idx = 0;

            currentParticipantID = participants[currentParticipantIndex];

            window = new Window(jackknife, m: (int)windowMode);

            timer = 0.0;
        }
コード例 #3
0
ファイル: Session.cs プロジェクト: shayan-taheri/Jackknife
        /**
         * Load a participant's dataset and session.
         * Train the recognizer with the training data
         * and run the video through. See what happens...
         */
        public static ConfusionMatrices evaluate_session(DeviceType device, int subject_id)
        {
            // Load up the training dataset.
            Dataset ds = load_subject_dataset(
                device,
                subject_id);

            // Load up the session.
            List <Frame> frames = new List <Frame>();

            load_session(
                device,
                subject_id,
                frames);

            // Create a new recognizer.
            JackknifeBlades blades = new JackknifeBlades();

            blades.SetIPDefaults();
            Jackknife jk = new Jackknife(blades);

            // Train the recognizer, without 'bad' gestures.
            for (int ii = 0;
                 ii < ds.Samples.Count;
                 ii++)
            {
                int    gesture_id   = ds.Samples[ii].GestureId;
                string gesture_name = ds.Gestures[gesture_id];
                if (bad_gesture(gesture_name))
                {
                    continue;
                }
                jk.AddTemplate(ds.Samples[ii]);
            }


            // Get device and application parameters
            // based on the device type.
            configuartion_parameters_t parameters = new configuartion_parameters_t(device);

            // We originally used n=4, r=2 for Kinect data
            // and n=6, r=2 for Leap Motion data, but
            // here we just set the average. There is barely
            // any effect on the results.
            jk.Train(6, 2, 1.00);

            // Play session video through
            // the recognizer.
            List <Vector>         buffer     = new List <Vector>();
            List <int>            detections = new List <int>();
            List <CommandResults> cmds       = new List <CommandResults>();
            int last_cmd_id = -1;
            int next_update = parameters.update_interval;

            int frame_no = 0;

            ExponentialMovingAverage filter = new ExponentialMovingAverage(frames[0].pt);
            Vector pt;

            for (int ii = 0;
                 ii < frames.Count;
                 ii++)
            {
                // skip this frame if its bad
                if (frames[ii].bad_pt)
                {
                    continue;
                }

                // Low pass filter the input.
                // Note, we originally didn't smooth the data,
                // so results now are a little higher than in
                // the paper.
                pt = filter.Filter(
                    frames[ii].pt,
                    1 / (double)parameters.fps);

                //pt = frames[ii].pt;

                frame_no += 1;

                // start a new command
                if (frames[ii].cmd_id != last_cmd_id)
                {
                    last_cmd_id = frames[ii].cmd_id;

                    int gid = convert_gesture_id(
                        ds,
                        frames[ii].gesture_id);

                    CommandResults cmd = new CommandResults(
                        frames[ii].cmd_id,
                        gid);

                    if (bad_gesture(frames[ii].gesture_id))
                    {
                        cmd.ignore = true;
                    }

                    cmds.Add(cmd);
                }

                // This buffering approach is really
                // inefficient, but since this off-line,
                // performance is not important.
                buffer.Add(pt);
                if (buffer.Count > parameters.sliding_window_frame_cnt)
                {
                    buffer.RemoveAt(0);
                }

                // We need to have a couple points before
                // calling the recognizer.
                if (buffer.Count < 2)
                {
                    continue;
                }

                // Wait a few frames again before trying
                // to recognize again.
                if (frame_no < next_update)
                {
                    continue;
                }

                next_update = frame_no + parameters.update_interval;

                // Run the recognizer.
                int gesture_id = jk.Classify(buffer);

                // Add recognition result.
                detections.Add(gesture_id);
                if (detections.Count > parameters.repeat_cnt)
                {
                    detections.RemoveAt(0);
                }

                // Count how many times this gesture was recognized.
                int winner_cnt = 0;
                for (int jj = 0;
                     jj < detections.Count;
                     jj++)
                {
                    if (detections[jj] == gesture_id)
                    {
                        winner_cnt += 1;
                    }
                }

                // Ensure we have enough recognitions.
                if (winner_cnt < parameters.repeat_cnt)
                {
                    continue;
                }

                // If nothing was detected, skip rest.
                if (gesture_id == -1)
                {
                    continue;
                }

                // Hurray! A gesture is recognized!
                // Hopefully it's the right one too!!
                cmds[cmds.Count - 1].add(gesture_id);
                detections.Clear();
                buffer.Clear();
            }

            // Mark bad commands, situations where the participant
            // made a mistake or tracking was lost. We know the
            // command was bad because the protector asked the
            // participant to repeat the gesture, but a new command
            // ID is assigned to the sequence.
            for (int ii = 1;
                 ii < cmds.Count;
                 ii++)
            {
                if (cmds[ii].expected_id == cmds[ii - 1].expected_id)
                {
                    CommandResults temp = cmds[ii - 1];
                    temp.ignore  = true;
                    cmds[ii - 1] = temp;
                }
            }

            // Put all results in confusion matrices.
            ConfusionMatrices ret = new ConfusionMatrices(ds);

            for (int ii = 0; ii < cmds.Count; ii++)
            {
                if (cmds[ii].ignore)
                {
                    continue;
                }

                cmds[ii].update_confusion_matrices(ret);
            }

            return(ret);
        }
コード例 #4
0
        public static Results EvaluateSessionWindowed(DeviceType device, int subject_id)
        {
            configuartion_parameters_t parameneters = new configuartion_parameters_t(device);

            // Load subject dataset
            Dataset       ds        = Global.load_subject_dataset(device, subject_id);
            List <Sample> train_set = Global.GetTrainSet(ds, 1);

            // Covert the dataset to format accepted by Jackknife
            List <Jackknife.Sample> jk_train_set = JackknifeConnector.GetJKTrainSet(train_set);

            // Load subject session
            List <Frame> frames = new List <Frame>();

            Global.load_session(device, subject_id, frames, ds);

            // Load ground truth
            List <GestureCommand> cmds = new List <GestureCommand>();

            GestureCommand.GetAllCommands(cmds, ds, device, subject_id);

            // Train the recognizer
            JackknifeBlades blades = new JackknifeBlades();

            blades.SetIPDefaults();

            blades.ResampleCnt = 20;

            Jackknife.Jackknife jk = new Jackknife.Jackknife(blades);
            foreach (Jackknife.Sample s in jk_train_set)
            {
                jk.AddTemplate(s);
            }

            // Set between 2.0 and 10.0 in steps of .25
            // to find the best result
            jk.SetRejectionThresholds(5.25f);

            // Set up filter for session points
            ExponentialMovingAverage ema_filter = new ExponentialMovingAverage(frames[0].pt);
            Vector pt;

            WindowSegmentor windowSegmentor = new WindowSegmentor(jk);

            //List<RecognitionResult> rresults = new List<RecognitionResult>();

            List <ContinuousResult> continuous_results = new List <ContinuousResult>();

            // Go through session
            for (int session_pt = 0; session_pt < frames.Count; session_pt++)
            {
                long ts1 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // at beginning

                pt = ema_filter.Filter(frames[session_pt].pt, 1 / (double)parameneters.fps);

                long ts2 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after filter

                Jackknife.Vector jkpt = JackknifeConnector.ToJKVector(pt);

                windowSegmentor.Update(jkpt);
                windowSegmentor.Segment(continuous_results);

                if (session_pt % 2000 == 0)
                {
                    Debug.Log(string.Format("{0}% Done", (double)session_pt / (double)frames.Count * 100.0));
                }
            }

            foreach (ContinuousResult cr in continuous_results)
            {
                Debug.Log(string.Format("st {0}, en {1}, gid {2}", cr.startFrameNo, cr.endFrameNo, cr.gid));
            }
            // Per gesture confusion matrix
            List <ConfisionMatrix> cm = new List <ConfisionMatrix>();

            for (int ii = 0; ii < ds.Gestures.Count; ii++)
            {
                cm.Add(new ConfisionMatrix());
            }

            for (int ii = 0; ii < continuous_results.Count; ii++)
            {
                ContinuousResult result = continuous_results[ii];

                bool found = false;
                int  cidx  = 0;

                for (cidx = 0; cidx < cmds.Count; cidx++)
                {
                    found = cmds[cidx].Hit(result);

                    if (found == true)
                    {
                        break;
                    }
                }

                if (found == true)
                {
                    // true positive
                    if (cmds[cidx].detected == false)
                    {
                        cmds[cidx].detected = true;
                        cm[result.gid].tp  += 1.0f;
                    }
                }
                else
                {
                    bool bad = GestureCommand.IsBadCommand(
                        frames,
                        result);

                    if (bad == true)
                    {
                        continue;
                    }
                    // false positive
                    cm[result.gid].fp += 1.0f;
                }
            }

            // false negatives
            for (int cidx = 0; cidx < cmds.Count; cidx++)
            {
                if (cmds[cidx].detected == true)
                {
                    continue;
                }

                cm[cmds[cidx].gid].fn += 1.0;
            }

            Results ret = new Results();

            for (int ii = 0; ii < cm.Count; ii++)
            {
                ret.AppendResults(cm[ii]);
            }

            ret.PrintF();
            return(ret);
        }
コード例 #5
0
        public static Results EvaluateSession(DeviceType device, int subject_id)
        {
            configuartion_parameters_t parameneters = new configuartion_parameters_t(device);

            // Load subject dataset
            Dataset       ds        = Global.load_subject_dataset(device, subject_id);
            List <Sample> train_set = Global.GetTrainSet(ds, 1);
            // Covert the dataset to format accepted by Jackknife
            List <Jackknife.Sample> jk_train_set = JackknifeConnector.GetJKTrainSet(train_set);

            // Load subject session
            List <Frame> frames = new List <Frame>();

            Global.load_session(device, subject_id, frames, ds);
            //Debug.Log("Frame_cnt = " + frames.Count());
            // Load ground truth
            List <GestureCommand> cmds = new List <GestureCommand>();

            GestureCommand.GetAllCommands(cmds, ds, device, subject_id);

            // Train the segmentor
            ContinuousResultOptions cr_options = new ContinuousResultOptions();

            //COREY FIX latency framecount
            cr_options.latencyFrameCount = 1;
            Machete yeah = new Machete(device, cr_options);

            foreach (Sample s in train_set)
            {
                yeah.AddSample(s, filtered: true);
            }

            //PrintYeahStats(yeah);

            // Train the recognizer
            JackknifeBlades blades = new JackknifeBlades();

            blades.SetIPDefaults();

            //COREY FIX RESAMPLECNT
            blades.ResampleCnt = 20;
            blades.LowerBound  = false;

            Jackknife.Jackknife jk = new Jackknife.Jackknife(blades);
            foreach (Jackknife.Sample s in jk_train_set)
            {
                jk.AddTemplate(s);
            }

            // Set between 2.0 and 10.0 in steps of .25
            // to find the best result
            // Best at 7.5 with around 66% :/
            jk.SetRejectionThresholds(7.0);


            // Set up filter for session points
            ExponentialMovingAverage ema_filter = new ExponentialMovingAverage(frames[0].pt, 5.0);
            Vector        pt;
            List <Vector> video = new List <Vector>();

            List <RecognitionResult> rresults = new List <RecognitionResult>();

            int triggered_count = 0;

            // Go through session
            for (int session_pt = 0; session_pt < frames.Count; session_pt++)
            {
                long ts1 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // at beginning

                List <ContinuousResult> continuous_results = new List <ContinuousResult>();

                pt = ema_filter.Filter(frames[session_pt].pt, 1 / (double)parameneters.fps);

                long ts2 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after filter

                video.Add(pt);

                //Debug.Log(string.Format("Pt: {0} {1} {2}", pt.Data[0], pt.Data[1], pt.Data[2]));

                yeah.ProcessFrame(pt, session_pt, continuous_results);

                long ts3 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after processing frame

                bool             cancel_if_better_score = false;
                ContinuousResult result = ContinuousResult.SelectResult(
                    continuous_results,
                    cancel_if_better_score);

                long ts4 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after looking for result

                //Debug.Log(string.Format("{0} {1} {2}", ts2 - ts1, ts3 - ts2, ts4 - ts3));
                //Debug.Log(string.Format("FRAME NO: {0}", frame_no));

                if (result == null)
                {
                    continue;
                }


                //COREY FIX For comparing against the original code
                //if (result.sample.GestureId == 0)
                //    Debug.Log(string.Format("start {0}, end {1}", result.startFrameNo, result.endFrameNo + 1));

                triggered_count += 1;

                //Debug.Log(string.Format("Frame: {3} Result: {0}, Sample: {1}, Score: {2}", result.gid, result.sample.GestureName, result.score, frame_no));
                //Debug.Log(string.Format("Best result as of: {0} {1}", ii, yeah.bestScore));

                // Run recognizer on segmented result
                double recognizer_d = 0.0f;
                bool   match        = false;

                // Save a buffer to pass to recognizer
                List <Jackknife.Vector> jkbuffer = JackknifeConnector.GetJKBufferFromVideo(
                    video,
                    result.startFrameNo,
                    result.endFrameNo);

                long ts5 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // before passing to recognizer

                match = jk.IsMatch(jkbuffer, result.sample.GestureId, out recognizer_d);

                //COREY Fix print out scores

                if (result.sample.GestureId == 0)
                {
                    //Debug.Log(string.Format("start {0}, end {1} ", result.startFrameNo, result.endFrameNo + 1) + string.Format("Is match = {0}, score = {1}",
                    //    match,
                    //    recognizer_d));
                }
                // Matched to template with this gid
                if (match == false)
                {
                    continue;
                }

                long ts6 = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond; // after classifying

                // Gesture was accepted
                RecognitionResult rresult = new RecognitionResult();
                rresult.gid   = result.gid;
                rresult.start = result.startFrameNo;
                rresult.end   = result.endFrameNo;
                rresult.score = recognizer_d;

                match = false;

                for (int ii = 0; ii < rresults.Count; ii++)
                {
                    if (rresults[ii].Update(rresult) == true)
                    {
                        match = true;
                        break;
                    }
                }

                // if some result was updated for better, continue
                if (match == true)
                {
                    continue;
                }

                rresults.Add(rresult);
            }

            // Per gesture confusion matrix
            List <ConfisionMatrix> cm = new List <ConfisionMatrix>();

            for (int ii = 0; ii < ds.Gestures.Count; ii++)
            {
                cm.Add(new ConfisionMatrix());
            }

            for (int ii = 0; ii < rresults.Count; ii++)
            {
                RecognitionResult result = rresults[ii];

                bool found = false;
                int  cidx  = 0;

                for (cidx = 0; cidx < cmds.Count; cidx++)
                {
                    found = cmds[cidx].Hit(result);

                    if (found == true)
                    {
                        break;
                    }
                }

                if (found == true)
                {
                    // true positive
                    if (cmds[cidx].detected == false)
                    {
                        cmds[cidx].detected = true;
                        cm[result.gid].tp  += 1.0f;
                    }
                }
                else
                {
                    bool bad = GestureCommand.IsBadCommand(
                        frames,
                        result);

                    if (bad == true)
                    {
                        continue;
                    }
                    // false positive
                    cm[result.gid].fp += 1.0f;
                }
            }

            // false negatives
            for (int cidx = 0; cidx < cmds.Count; cidx++)
            {
                if (cmds[cidx].detected == true)
                {
                    continue;
                }

                cm[cmds[cidx].gid].fn += 1.0;
            }

            Results ret = new Results();

            for (int ii = 0; ii < cm.Count; ii++)
            {
                ret.AppendResults(cm[ii]);
                //temp += string.Format("{5}:\t A: {0:N6}, E: {1:N6}, P: {2:N6}, R: {3:N6}, F1: {4:N6}\n", ret.accuracy/ret.total, ret.error / ret.total, ret.precision / ret.total, ret.recall / ret.total, ret.f1_0 / ret.total, ii);
            }
            //Debug.Log(temp);

            return(ret);
        }
コード例 #6
0
        /**
         * A simple user independent test.
         */
        public static void user_indepedent_test(DeviceType device)
        {
            // First, load all training data for one device type.
            string path = "";

            if (device == DeviceType.KINECT)
            {
                path = Global.GetRootDirectory() + "datasets/jk2017/kinect/training/";
            }
            else if (device == DeviceType.LEAP_MOTION)
            {
                path = Global.GetRootDirectory() + "datasets/jk2017/leap_motion/training/";
            }


            Dataset ds = Dataset.LoadDataset(path);

            int subject_cnt      = ds.Subjects.Count;
            int sample_cnt       = ds.Samples.Count;
            ConfusionMatrices cm = new ConfusionMatrices(ds);

            // iterate through all subjects
            for (int subject_id = 0; subject_id < subject_cnt; subject_id++)
            {
                ConfusionMatrices cm_individual = new ConfusionMatrices(ds);

                Console.WriteLine("Participant: " + subject_id);

                // train a recognizer with the selected subject
                JackknifeBlades blades = new JackknifeBlades();

                blades.SetIPDefaults();

                Jackknife jk = new Jackknife(blades);

                for (int sample_id = 0; sample_id < sample_cnt; sample_id++)
                {
                    Sample sample = ds.Samples[sample_id];

                    if (sample.SubjectId != subject_id)
                    {
                        continue;
                    }

                    jk.AddTemplate(ds.Samples[sample_id]);
                }

                // only train the recognize if you need

                // test the recognizer with all other samples
                for (int sample_id = 0; sample_id < sample_cnt; sample_id++)
                {
                    Sample sample = ds.Samples[sample_id];

                    if (sample.SubjectId == subject_id)
                    {
                        continue;
                    }

                    int gid = jk.Classify(sample);

                    cm_individual.AddResult(
                        sample.GestureId,
                        gid);
                }

                cm.AddResult(cm_individual);

                ResultT resultss = cm_individual.Results();
                resultss.Print();
            }

            Console.WriteLine("Aggregate Results: ");
            ResultT results = cm.Results();

            results.Print();
        }