Пример #1
0
        unsafe private double GetWordProbability(ref NativeOffset no)
        {
            if ((no.Length == 1) && !StringsHelper.IsLettersOrDash(no.BasePtr[no.StartIndex]))
            {
                return(1);
            }

            return(_Model.TryGetProbability(ref no, out var prob) ? prob : 0);
        }
Пример #2
0
        public bool TryGetProbability(ref NativeOffset no, out double probability)
        {
            if (_Set.TryGetValue(ref no, out var existsValue))
            {
#if DEBUG
                var len = StringsHelper.GetLength(existsValue);
                Debug.Assert(len == no.Length);
#endif
                probability = ToProbability(existsValue, no.Length + 1);   //--- len + 1 );
                return(true);
            }
            probability = default(double);
            return(false);
        }
Пример #3
0
        public int GetHashCode(ref NativeOffset no)
        {
            char *ptr    = no.BasePtr + no.StartIndex;
            char *endPtr = ptr + no.Length;
            int   n1     = 5381;
            int   n2     = 5381;

            for ( ; ;)
            {
                var n3 = (int)(*(ushort *)ptr);
                n1 = ((n1 << 5) + n1 ^ n3);
                n2 = ((n2 << 5) + n2 ^ n3);
                ptr++;

                if (ptr == endPtr)
                {
                    break;
                }
            }
            return(n1 + n2 * 1566083941);
        }
Пример #4
0
        unsafe public List <TermProbability_Offset> Run(char *text, int length)
        {
            var probs   = stackalloc double      [length + 1]; probs[0] = 1;
            var offsets = stackalloc NativeOffset[length + 1];
            var no      = new NativeOffset()
            {
                BasePtr = text
            };

            for (var i = 0; i <= length; i++)
            {
                for (var j = 0; j < i; j++)
                {
                    no.StartIndex = j;
                    no.Length     = i - j;
                    var term_prob = probs[i - no.Length] * GetWordProbability(ref no);
                    if (probs[i] <= term_prob)
                    {
                        probs  [i] = term_prob;
                        offsets[i] = no;
                    }
                }
            }

            var tuples = new List <TermProbability_Offset>(length >> 2);

            for (var i = length; 0 < i;)
            {
                var no_ptr = &offsets[i];
                tuples.Add(new TermProbability_Offset()
                {
                    StartIndex  = no_ptr->StartIndex,
                    Length      = no_ptr->Length,
                    Probability = probs[i],
                });
                i = i - no_ptr->Length;
            }
            tuples.Reverse();
            return(tuples);
        }
Пример #5
0
 public bool TryGetProbability(ref NativeOffset no, out double probability) => throw new NotImplementedException();