unsafe private double GetWordProbability(ref NativeOffset no)
{
if ((no.Length == 1) && !StringsHelper.IsLettersOrDash(no.BasePtr[no.StartIndex]))
{
return(1);
}
return(_Model.TryGetProbability(ref no, out var prob) ? prob : 0);
}
public bool TryGetProbability(ref NativeOffset no, out double probability)
{
if (_Set.TryGetValue(ref no, out var existsValue))
{
#if DEBUG
var len = StringsHelper.GetLength(existsValue);
Debug.Assert(len == no.Length);
#endif
probability = ToProbability(existsValue, no.Length + 1); //--- len + 1 );
return(true);
}
probability = default(double);
return(false);
}
public int GetHashCode(ref NativeOffset no)
{
char *ptr = no.BasePtr + no.StartIndex;
char *endPtr = ptr + no.Length;
int n1 = 5381;
int n2 = 5381;
for ( ; ;)
{
var n3 = (int)(*(ushort *)ptr);
n1 = ((n1 << 5) + n1 ^ n3);
n2 = ((n2 << 5) + n2 ^ n3);
ptr++;
if (ptr == endPtr)
{
break;
}
}
return(n1 + n2 * 1566083941);
}
unsafe public List <TermProbability_Offset> Run(char *text, int length)
{
var probs = stackalloc double [length + 1]; probs[0] = 1;
var offsets = stackalloc NativeOffset[length + 1];
var no = new NativeOffset()
{
BasePtr = text
};
for (var i = 0; i <= length; i++)
{
for (var j = 0; j < i; j++)
{
no.StartIndex = j;
no.Length = i - j;
var term_prob = probs[i - no.Length] * GetWordProbability(ref no);
if (probs[i] <= term_prob)
{
probs [i] = term_prob;
offsets[i] = no;
}
}
}
var tuples = new List <TermProbability_Offset>(length >> 2);
for (var i = length; 0 < i;)
{
var no_ptr = &offsets[i];
tuples.Add(new TermProbability_Offset()
{
StartIndex = no_ptr->StartIndex,
Length = no_ptr->Length,
Probability = probs[i],
});
i = i - no_ptr->Length;
}
tuples.Reverse();
return(tuples);
}
public bool TryGetProbability(ref NativeOffset no, out double probability) => throw new NotImplementedException();