public IEnumerable <MorphInfo> Parse(IEnumerable <string> words, bool lemmatize = true)
{
foreach (var batch in _batchify(words, _maxBatchSize))
{
var srcMas = batch.ToArray();
_vectorize(srcMas,
out int maxLength,
out List <int[]> indexes,
out List <int> values,
out int[] seqLens
);
var result = _net.Classify(maxLength, srcMas.Length, indexes, values, seqLens);
for (int i = 0; i < srcMas.Length; i++)
{
yield return(new MorphInfo(
srcMas[i],
Enumerable.Range(0, K)
.Select(j => new Tag(
_config.ClsDic[result.ResultIndexes[i, j]],
result.ResultProbs[i, j],
lemma: _withLemmatization
? _getLemma(srcMas[i], result.Lemmas, i, j, result.ResultIndexes[i, j])
: null,
classIndex: result.ResultIndexes[i, j]
)
)
.ToArray(),
result.GramProbs.ToDictionary(
kp => kp.Key,
kp => new GramCategory(
Enumerable.Range(0, _config[kp.Key].NnClassesCount)
.Select(j => new Gram(
_config[kp.Key, j],
result.GramProbs[kp.Key][i, j++]
))
.OrderByDescending(x => x.Power)
.ToArray()
)
)
));
}
}
}
public IEnumerable <(string srcWord, IEnumerable <Tag> tags, Dictionary <string, GramCategory> gramDic)> Parse(IEnumerable <string> words)
{
foreach (var batch in _batchify(words, _maxBatchSize))
{
var srcMas = batch.ToArray();
_vectorizeWords(srcMas,
out int maxLength,
out List <int[]> indexes,
out List <int> values,
out int[] seqLens
);
var result = _net.Classify(maxLength, srcMas.Length, indexes, values, seqLens);
for (int i = 0; i < srcMas.Length; i++)
{
var srcword = srcMas[i];
var tags = Enumerable.Range(0, K)
.Select(j => new Tag(
_tagHelper[result.ResultIndexes[i, j]],
result.ResultProbs[i, j],
lemma: _withLemmatization
? _getLemma(srcMas[i], result.Lemmas, i, j, result.ResultIndexes[i, j])
: null,
id: result.ResultIndexes[i, j]
)
);
var gramDic = result.GramProbs.ToDictionary(
kp => kp.Key,
kp => new GramCategory(
Enumerable.Range(0, _config[kp.Key].NnClassesCount)
.Select(j => new Gram(
_config[kp.Key, j],
result.GramProbs[kp.Key][i, j++]
))
.OrderByDescending(x => x.Power)
.ToArray()
));
yield return(srcWord : srcword, tags : tags, gramDic : gramDic);
}
}
}