private IList <DocumentPosition> GetPhraseMatches(IList <Token> tokens, HashSet <DocumentPosition>[] dictionaries)
{
var resultList = new List <DocumentPosition>();
var suffixesCount = tokens.Count;
lock (this.syncObj)
{
foreach (var currentPosition in dictionaries[0])
{
var currentOffset = tokens[0].DistanceToNext;
for (var j = 1; j < suffixesCount; j++)
{
var expectedNextResult = new DocumentPosition
{
Document = currentPosition.Document,
RowNumber = currentPosition.RowNumber,
ColNumber = currentPosition.ColNumber + currentOffset
};
if (!dictionaries[j].Contains(expectedNextResult))
{
break;
}
currentOffset += tokens[j].DistanceToNext;
if (j == suffixesCount - 1)
{
resultList.Add(currentPosition);
}
}
}
}
return(resultList);
}
private void AddToken(Token token, int rowNumber, string document)
{
var startColNumber = token.Position;
var term = token.Term;
var length = term.Length;
var prefixPosition = new DocumentPosition
{
ColNumber = startColNumber,
Document = document,
RowNumber = rowNumber
};
this.AddTerm(term, prefixPosition);
for (var i = 1; i < term.Length; i++)
{
var suffixPosition = new DocumentPosition
{
ColNumber = startColNumber + i,
Document = document,
RowNumber = rowNumber
};
var suffix = term.Substring(i, length - i);
var prefix = term.Substring(0, i);
this.AddTerm(suffix, suffixPosition);
this.AddTerm(prefix, prefixPosition);
}
}
private void AddTerm(string term, DocumentPosition result)
{
var hashCode = StringHelper.GetHashCode(term);
lock (this.syncObj)
{
if (!this.dictionary.ContainsKey(hashCode))
{
this.dictionary.Add(hashCode, new HashSet <DocumentPosition> {
result
});
}
else
{
this.dictionary[hashCode].Add(result);
}
}
}
private void AddToken(Token token, int rowNumber, string document)
{
var startColNumber = token.Position;
var term = token.Term;
var length = term.Length;
for (var i = 0; i < term.Length; i++)
{
var documentPosition = new DocumentPosition
{
ColNumber = startColNumber + i,
Document = document,
RowNumber = rowNumber
};
var suffix = term.Substring(i, length - i);
lock (this.syncObj)
{
if (!this.suffixArray.TryGetValue(suffix, out var list, this.matchComparer))
{
this.suffixArray.TryAdd(suffix, new List <DocumentPosition> {
documentPosition
}, this.matchComparer);
}
