public static long SetSentence(StreamReader reader, long wordCount, long?[] sentence,
ref ulong nextRandom, ref long sentenceLength, ref string [] lineThatGotCutOff, WordCollection wordCollection, float thresholdForOccurrenceOfWords)
{
string line;
var loopEnd = false;
if (lineThatGotCutOff != null && lineThatGotCutOff.Any())
{
loopEnd = HandleWords(reader, ref wordCount, sentence, ref nextRandom, ref sentenceLength, lineThatGotCutOff, wordCollection, thresholdForOccurrenceOfWords);
lineThatGotCutOff = null;
}
while (!loopEnd && (line = reader.ReadLine()) != null)
{
var words = WordCollection.ParseWords(line).Select(WordCollection.Clean).ToArray();
if (words.Length > sentence.Length)
{
continue;
}
if (sentenceLength > sentence.Length - words.Length)
{
lineThatGotCutOff = words;
break;
}
loopEnd = HandleWords(reader, ref wordCount, sentence, ref nextRandom, ref sentenceLength, words, wordCollection, thresholdForOccurrenceOfWords);
}
return(wordCount);
}
public int SetSentence(StreamReader reader, int wordCount, int?[] sentence, Random random, ref int sentenceLength,
ref string[] lineThatGotCutOff)
{
string line;
var loopEnd = false;
if (lineThatGotCutOff != null && lineThatGotCutOff.Any())
{
loopEnd = HandleWords(reader, ref wordCount, sentence, random, ref sentenceLength, lineThatGotCutOff);
lineThatGotCutOff = null;
}
while (!loopEnd && (line = reader.ReadLine()) != null)
{
var words = WordCollection.ParseWords(line).Select(WordCollection.Clean).ToArray();
if (words.Length > sentence.Length)
{
continue;
}
if (sentenceLength > sentence.Length - words.Length)
{
lineThatGotCutOff = words;
break;
}
loopEnd = HandleWords(reader, ref wordCount, sentence, random, ref sentenceLength, words);
}
return(wordCount);
}