// Token: 0x06003F1B RID: 16155 RVA: 0x0012041C File Offset: 0x0011E61C
private static WinRTSpellerInterop.TextRange?GetSpellCheckCleanSubstitutionToken(SpellChecker spellChecker, string documentText, WinRTSpellerInterop.SpellerSegment lastToken, WinRTSpellerInterop.SpellerSegment missingFragment)
{
if (string.IsNullOrWhiteSpace((missingFragment != null) ? missingFragment.Text : null) || string.IsNullOrWhiteSpace((lastToken != null) ? lastToken.Text : null) || string.IsNullOrWhiteSpace(documentText))
{
return(null);
}
int num = Math.Min(missingFragment.TextRange.Length, 4);
List <SpellChecker.SpellingError> list = (spellChecker != null) ? spellChecker.ComprehensiveCheck(lastToken.Text, true) : null;
if (list == null || list.Count != 0)
{
for (int i = 1; i <= num; i++)
{
string text = documentText.Substring(lastToken.TextRange.Start, lastToken.TextRange.Length + i).TrimEnd(new char[0]);
if (spellChecker != null)
{
List <SpellChecker.SpellingError> list2 = spellChecker.ComprehensiveCheck(text, true);
if (((list2 != null) ? new int?(list2.Count) : null) == 0)
{
return(new WinRTSpellerInterop.TextRange?(new WinRTSpellerInterop.TextRange(lastToken.TextRange.Start, text.Length)));
}
}
}
}
return(null);
}
private void EnumerateSuggestions()
{
List<string> result = new List<string>();
_isClean = true;
if (_spellChecker == null)
{
_suggestions = result.AsReadOnly();
return;
}
List<SpellChecker.SpellingError> spellingErrors = null;
using (new SpellerCOMActionTraceLogger(Owner, SpellerCOMActionTraceLogger.Actions.ComprehensiveCheck))
{
spellingErrors = Text != null ? _spellChecker.ComprehensiveCheck(Text) : null;
}
if (spellingErrors == null)
{
_suggestions = result.AsReadOnly();
return;
}
foreach (var spellingError in spellingErrors)
{
result.AddRange(spellingError.Suggestions);
if (spellingError.CorrectiveAction != SpellChecker.CorrectiveAction.None)
{
_isClean = false;
}
}
_suggestions = result.AsReadOnly();
}
// Token: 0x06003F1A RID: 16154 RVA: 0x00120230 File Offset: 0x0011E430
public static IReadOnlyList <WinRTSpellerInterop.SpellerSegment> ComprehensiveGetTokens(this WordsSegmenter segmenter, string text, SpellChecker spellChecker, WinRTSpellerInterop owner)
{
IReadOnlyList <WordSegment> readOnlyList = ((segmenter != null) ? segmenter.GetTokens(text) : null) ?? new List <WordSegment>().AsReadOnly();
List <WinRTSpellerInterop.SpellerSegment> list = new List <WinRTSpellerInterop.SpellerSegment>();
if (readOnlyList.Count == 0)
{
return(list.AsReadOnly());
}
int num = 0;
for (int i = 0; i < readOnlyList.Count; i++)
{
int startPosition = (int)readOnlyList[i].SourceTextSegment.StartPosition;
int length = (int)readOnlyList[i].SourceTextSegment.Length;
if (spellChecker != null && startPosition > num)
{
WinRTSpellerInterop.SpellerSegment missingFragment = new WinRTSpellerInterop.SpellerSegment(text, new WinRTSpellerInterop.TextRange(num, startPosition - num), spellChecker, owner);
if (list.Count > 0)
{
WinRTSpellerInterop.TextRange?spellCheckCleanSubstitutionToken = WinRTSpellerInteropExtensions.GetSpellCheckCleanSubstitutionToken(spellChecker, text, list[list.Count - 1], missingFragment);
if (spellCheckCleanSubstitutionToken != null)
{
list[list.Count - 1] = new WinRTSpellerInterop.SpellerSegment(text, spellCheckCleanSubstitutionToken.Value, spellChecker, owner);
}
}
}
list.Add(new WinRTSpellerInterop.SpellerSegment(text, new WinRTSpellerInterop.TextRange(startPosition, length), spellChecker, owner));
num = startPosition + length;
}
if (readOnlyList.Count > 0)
{
bool flag;
if (spellChecker == null)
{
flag = true;
}
else
{
List <SpellChecker.SpellingError> list2 = spellChecker.ComprehensiveCheck(readOnlyList[readOnlyList.Count - 1].Text, true);
flag = (((list2 != null) ? new int?(list2.Count) : null) != 0);
}
if (flag && num < text.Length)
{
WinRTSpellerInterop.SpellerSegment missingFragment2 = new WinRTSpellerInterop.SpellerSegment(text, new WinRTSpellerInterop.TextRange(num, text.Length - num), spellChecker, owner);
if (list.Count > 0)
{
WinRTSpellerInterop.TextRange?spellCheckCleanSubstitutionToken2 = WinRTSpellerInteropExtensions.GetSpellCheckCleanSubstitutionToken(spellChecker, text, list[list.Count - 1], missingFragment2);
if (spellCheckCleanSubstitutionToken2 != null)
{
list[list.Count - 1] = new WinRTSpellerInterop.SpellerSegment(text, spellCheckCleanSubstitutionToken2.Value, spellChecker, owner);
}
}
}
}
return(list.AsReadOnly());
}
/// <summary>
/// Checks through combinations of <paramref name="lastToken"/> + substrings(<paramref name="missingFragment"/>) and
/// returns the first spellcheck-clean result.
/// </summary>
/// <param name="spellChecker">Spell-checker</param>
/// <param name="documentText">Overall document text within which the text-ranges are computed</param>
/// <param name="lastToken">Previous token immediately preceding <paramref name="missingFragment"/></param>
/// <param name="missingFragment">The missing-fragment identified immediately after <paramref name="lastToken"/></param>
/// <returns></returns>
/// <remarks>
/// See note about MAXLEN in <see cref="ComprehensiveGetTokens(WordsSegmenter, string, SpellChecker, WinRTSpellerInterop)"/>
/// which explains the rationale behind the value of the constant AlternateFormsMaximumCount.
/// </remarks>
private static WinRTSpellerInterop.TextRange?GetSpellCheckCleanSubstitutionToken(
SpellChecker spellChecker,
string documentText,
SpellerSegment lastToken,
SpellerSegment missingFragment)
{
const int AlternateFormsMaximumCount = 4;
if (string.IsNullOrWhiteSpace(missingFragment?.Text) ||
string.IsNullOrWhiteSpace(lastToken?.Text) ||
string.IsNullOrWhiteSpace(documentText))
{
return(null);
}
int altFormsCount = Math.Min(missingFragment.TextRange.Length, AlternateFormsMaximumCount);
var spellingErrors = spellChecker?.ComprehensiveCheck(lastToken.Text);
if (spellingErrors?.Count != 0)
{
// One of the substring-permutations of the missingFragment - when concatenated with 'lastToken' - could be a viable
// replacement for 'lastToken'
for (int i = 1; i <= altFormsCount; i++)
{
var altForm = documentText.Substring(lastToken.TextRange.Start, lastToken.TextRange.Length + i).TrimEnd();
if (spellChecker?.ComprehensiveCheck(altForm)?.Count == 0)
{
// Use this altForm in place lastToken
return(new WinRTSpellerInterop.TextRange(
lastToken.TextRange.Start,
altForm.Length));
}
}
}
return(null);
}
/// <summary>
/// Tokenizes <paramref name="text"/> using <paramref name="segmenter"/>, and then identifies fixes-up
/// the tokens to account for any missed text "in-between" those tokens.
/// </summary>
/// <param name="segmenter">Word-breaker instance</param>
/// <param name="text">The text being tokenized</param>
/// <param name="spellChecker">The spell-checker instance used to augment the tokenizing process</param>
/// <param name="owner">Calling <see cref="WinRTSpellerInterop"/> instance</param>
/// <returns></returns>
/// <remarks>
/// Windows.Data.Text.WordsSegmenter tends to drop punctuation characters like period ('.')
/// when tokenizing text. Though this behavior is compatible with a vast majority of text-processing
/// scenarios (like word-counting), it is not ideal for spell-checking.
///
/// In this method, the following <paramref name="spellChecker"/> augmented heuristic is applied to update the token-list generated by
/// <paramref name="segmenter"/>.
///
/// - Identify if any text 'missingFragment' has been dropped by the <paramref name="segmenter"/>
/// - If the token immediately preceding 'missingFragment', previousToken, has a spelling error, then attempt to
/// create new candiate tokens in the following order:
///
/// previousToken + missingFragment[0..0]
/// previousToken + missingFragment[0..1]
/// previousToken + missingFragment[0..2]
/// ...
/// ...
/// previousToken + missingFragment[0..LEN-1], where LEN = LEN(missingFragment)
///
/// - Select the first candidate token that is free of spelling errors, and replace 'previousToken' with it.
/// - For performance reasons, we choose a constant MAXLEN = 4 such that when LEN > MAXLEN, only MAXLEN
/// tokens are considered.
/// - MAXLEN = 4 is a somewhat arbitrary choice, though it seems more than sufficient to address common
/// problems this heuristic is intended to help with.
///
/// - Typical word-breaking problems that have been observed empirically involve only one missed character,
/// for which MAXLEN=1 would be sufficient. MAXLEN=4 is chosen as a sufficiently-large tradeoff between
/// correctness and performance.
///
/// - Also see https://github.com/dotnet/wpf/pull/2753#issuecomment-602120768 for a discussion related to this.
/// </remarks>
public static IReadOnlyList <SpellerSegment> ComprehensiveGetTokens(
this WordsSegmenter segmenter,
string text,
SpellChecker spellChecker,
WinRTSpellerInterop owner)
{
IReadOnlyList <WordSegment> tokens = segmenter?.GetTokens(text) ?? Array.Empty <WordSegment>();
if (tokens.Count == 0)
{
return(Array.Empty <SpellerSegment>());
}
var allTokens = new List <SpellerSegment>();
int predictedNextTokenStartPosition = 0;
for (int i = 0; i < tokens.Count; i++)
{
int nextTokenStartPosition = (int)tokens[i].SourceTextSegment.StartPosition;
int nextTokenLength = (int)tokens[i].SourceTextSegment.Length;
if (spellChecker != null)
{
if (nextTokenStartPosition > predictedNextTokenStartPosition)
{
// There is a "gap" between the last recorded token and the current token.
// Identify the missing token and add it as a "supplementary word segment" - but only if the token
// turns out to be a substantial one (i.e., if the string is non-blank/non-empty).
var missingFragment =
new SpellerSegment(
text,
new WinRTSpellerInterop.TextRange(
predictedNextTokenStartPosition,
nextTokenStartPosition - predictedNextTokenStartPosition),
spellChecker,
owner);
if (allTokens.Count > 0)
{
var substToken = GetSpellCheckCleanSubstitutionToken(spellChecker, text, allTokens[allTokens.Count - 1], missingFragment);
if (substToken != null)
{
allTokens[allTokens.Count - 1] = new SpellerSegment(text, substToken.Value, spellChecker, owner);
}
}
}
}
allTokens.Add(
new SpellerSegment(
text,
new WinRTSpellerInterop.TextRange(
nextTokenStartPosition,
nextTokenLength),
spellChecker,
owner));
predictedNextTokenStartPosition = nextTokenStartPosition + nextTokenLength;
}
if (tokens.Count > 0 &&
spellChecker?.ComprehensiveCheck(tokens[tokens.Count - 1].Text)?.Count != 0 &&
predictedNextTokenStartPosition < text.Length)
{
// There is a token possibly missing at the end of the string
var missingFragment =
new SpellerSegment(
text,
new WinRTSpellerInterop.TextRange(
predictedNextTokenStartPosition,
text.Length - predictedNextTokenStartPosition),
spellChecker,
owner);
if (allTokens.Count > 0)
{
var substToken = GetSpellCheckCleanSubstitutionToken(spellChecker, text, allTokens[allTokens.Count - 1], missingFragment);
if (substToken != null)
{
allTokens[allTokens.Count - 1] = new SpellerSegment(text, substToken.Value, spellChecker, owner);
}
}
}
return(allTokens.AsReadOnly());
}
