public List <string> Split(string s, int maxWordLength, Func <double, double, double, double, double, double> method, bool verbose)
{
try
{
if (verbose)
{
Trace.Indent();
}
if (verbose)
{
Trace.WriteLine("Getting FPL...");
}
var fpl = FrequencyPerLength();
if (verbose)
{
Trace.Write("OK");
}
int n = s.Length;
List <string> ans = new List <string>();
int[] pre = new int[n + 1], cnt = new int[n + 1];
double[] dp = new double[n + 1];
dp[0] = 0;
for (int i = 1; i <= n; i++)
{
dp[i] = double.NegativeInfinity;
}
pre[0] = 1;//crutial to make 0-pre[0]<0 when tracing back
cnt[0] = 0;
if (verbose)
{
Trace.WriteLine("DPing...");
}
int percentage = -1;
for (int i = 0; i < n; i++)
{
Parallel.For(1, Math.Min(n - i, maxWordLength) + 1, (l) =>
{
double probLog = method(l, Count(s.Substring(i, l)), Entropy(s.Substring(i, l)), fpl[l].mean, fpl[l].stderr);//count,entropy
var v = dp[i] + probLog;
if (v > dp[i + l])
{
dp[i + l] = v;
cnt[i + l] = cnt[i] + 1;
pre[i + l] = l;
}
});
if (i > 0 && (i + 1) * 100L / n > percentage)
{
if (verbose)
{
Trace.WriteLine($"DPing... {++percentage}% Ex: {s.Substring(i - pre[i], pre[i])} scored {dp[i]} avg {(double)i / cnt[i]} words");
}
}
}
if (verbose)
{
Trace.WriteLine("Tracing back...");
}
List <int> idxs = new List <int>();
for (int i = n; i >= 0; i -= pre[i])
{
idxs.Add(i);
}
if (verbose)
{
Trace.WriteLine("Picking words...");
}
percentage = -1;
for (int i = idxs.Count - 1; i > 0; i--)
{
string _s = s.Substring(idxs[i], idxs[i - 1] - idxs[i]);
WordIdentified?.Invoke(_s);
ans.Add(_s);
if ((idxs.Count - i + 1) * 100L / idxs.Count > percentage)
{
if (verbose)
{
Trace.WriteLine($"Picking words... {++percentage}% Ex: {_s}");
}
}
}
if (verbose)
{
Trace.Write(" => OK");
}
SplittedWords = ans;
IsBuilt = true;
return(ans);
}
finally { if (verbose)
{
Trace.Unindent();
}
}
}
