コード例 #1
0
        private void EvaluateAndSortCandidates(List <Core.Tokenization.PrioritizedToken> candidates)
        {
            if (candidates == null || candidates.Count == 0)
            {
                return;
            }

            int basePriority = _Priority;

            foreach (Core.Tokenization.PrioritizedToken pt in candidates)
            {
                Core.Tokenization.NumberToken nt = pt.Token as Core.Tokenization.NumberToken;
                System.Diagnostics.Debug.Assert(nt != null);

                int malus = 0;

                if (nt.DecimalSeparator == NumericSeparator.Alternate ||
                    nt.GroupSeparator == NumericSeparator.Alternate)
                {
                    ++malus;
                }

                pt.Priority = _Priority - malus;
            }

            candidates.Sort(delegate(PrioritizedToken a, PrioritizedToken b)
            {
                // sort by decreasing priority
                return(b.Priority - a.Priority);
            });
        }
コード例 #2
0
        public override Core.Tokenization.Token Recognize(string s,
                                                          int from, bool allowTokenBundles, ref int consumedLength)
        {
            consumedLength = 0;

            List <FSTMatch> matches = _FSTRecognizer.ComputeMatches(s, from, false, 0, true);

            if (matches == null || matches.Count == 0)
            {
                return(null);
            }

            List <Core.Tokenization.PrioritizedToken> candidates = null;

            foreach (FSTMatch m in matches)
            {
                System.Diagnostics.Debug.Assert(m.Index == from);

                if (VerifyContextConstraints(s, m.Index + m.Length))
                {
                    if (m.Output == null || m.Output.Length != m.Length || m.Length == 0)
                    {
                        throw new Exception("Internal error: invalid number FST");
                    }

                    if (consumedLength == 0)
                    {
                        consumedLength = m.Length;
                    }
                    else
                    {
                        System.Diagnostics.Debug.Assert(consumedLength == m.Length);
                    }

                    Core.Tokenization.NumberToken nt = ParseNumber(s.Substring(m.Index, m.Length),
                                                                   m.Output);
                    if (nt != null)
                    {
                        if (candidates == null)
                        {
                            candidates = new List <PrioritizedToken>();
                        }
                        candidates.Add(new Core.Tokenization.PrioritizedToken(nt, 0));
                    }
                    else
                    {
                        System.Diagnostics.Debug.Assert(false, "Expect to be able to parse a number if returned by the FST");
                    }
                }
            }

            if (candidates == null || candidates.Count == 0)
            {
                return(null);
            }

            EvaluateAndSortCandidates(candidates);
            if (allowTokenBundles && candidates.Count > 1)
            {
                return(new Core.Tokenization.TokenBundle(candidates));
            }
            else
            {
                return(candidates[0].Token);
            }
        }