private bool AddWordCapitalized(string word, FlagSet flags, MorphSet morphs, CapitalizationType capType)
{
// add inner capitalized forms to handle the following allcap forms:
// Mixed caps: OpenOffice.org -> OPENOFFICE.ORG
// Allcaps with suffixes: CIA's -> CIA'S
if (
(
capType == CapitalizationType.Huh ||
capType == CapitalizationType.HuhInit ||
(capType == CapitalizationType.All && flags.HasItems)
)
&&
!flags.Contains(Affix.ForbiddenWord)
)
{
flags = Builder.Dedup(FlagSet.Union(flags, SpecialFlags.OnlyUpcaseFlag));
var textInfo = Affix.Culture.TextInfo;
var initCapBuilder = StringBuilderPool.Get(word);
if (initCapBuilder.Length > 0)
{
initCapBuilder[0] = textInfo.ToUpper(initCapBuilder[0]);
for (var i = 1; i < initCapBuilder.Length; i++)
{
initCapBuilder[i] = textInfo.ToLower(initCapBuilder[i]);
}
}
return(AddWord(StringBuilderPool.GetStringAndReturn(initCapBuilder), flags, morphs, true));
}
return(false);
}
internal List <Affix <SuffixEntry> > GetMatchingAffixes(string word, FlagSet groupFlagFilter = null)
{
if (string.IsNullOrEmpty(word))
{
return(new List <Affix <SuffixEntry> >(0));
}
var results = new List <Affix <SuffixEntry> >();
if (AffixesByIndexedByKey.TryGetValue(word[word.Length - 1], out AffixEntryGroupCollection <SuffixEntry> indexedGroups))
{
foreach (var group in indexedGroups)
{
if (groupFlagFilter == null || groupFlagFilter.Contains(group.AFlag))
{
foreach (var entry in group.Entries)
{
if (HunspellTextFunctions.IsReverseSubset(entry.Key, word))
{
results.Add(Affix <SuffixEntry> .Create(entry, group));
}
}
}
}
}
if (AffixesWithDots.HasItems)
{
results.AddRange(GetMatchingWithDotAffixes(word, HunspellTextFunctions.IsReverseSubset));
}
return(results);
}
public static bool ContainsAny(FlagSet a, FlagSet b)
{
if (a != null && !a.IsEmpty && b != null && !b.IsEmpty)
{
if (a.Count == 1)
{
return(b.Contains(a[0]));
}
if (b.Count == 1)
{
return(a.Contains(b[0]));
}
if (a.Count > b.Count)
{
ReferenceHelpers.Swap(ref a, ref b);
}
foreach (var item in a)
{
if (b.Contains(item))
{
return(true);
}
}
}
return(false);
}
public bool Equals(FlagSet other) =>
!ReferenceEquals(other, null)
&&
(
ReferenceEquals(this, other) ||
ArrayComparer <FlagValue> .Default.Equals(other.items, items)
);
internal static FlagSet Union(FlagSet set, FlagValue value)
{
var valueIndex = Array.BinarySearch(set.items, value);
if (valueIndex >= 0)
{
return(set);
}
valueIndex = ~valueIndex; // locate the best insertion point
var newItems = new FlagValue[set.items.Length + 1];
if (valueIndex >= set.items.Length)
{
Array.Copy(set.items, newItems, set.items.Length);
newItems[set.items.Length] = value;
}
else
{
Array.Copy(set.items, newItems, valueIndex);
Array.Copy(set.items, valueIndex, newItems, valueIndex + 1, set.items.Length - valueIndex);
newItems[valueIndex] = value;
}
return(new FlagSet(newItems));
}
private PrefixCollection(
Dictionary <FlagValue, AffixEntryGroup <PrefixEntry> > affixesByFlag,
Dictionary <char, AffixEntryGroupCollection <PrefixEntry> > affixesByIndexedByKey,
AffixEntryGroupCollection <PrefixEntry> affixesWithDots,
AffixEntryGroupCollection <PrefixEntry> affixesWithEmptyKeys,
FlagSet contClasses)
: base(affixesByFlag, affixesByIndexedByKey, affixesWithDots, affixesWithEmptyKeys, contClasses)
{
}
public PrefixEntry(
string strip,
string affixText,
CharacterConditionGroup conditions,
MorphSet morph,
FlagSet contClass)
: base(strip, affixText, conditions, morph, contClass)
{
}
public FlagSet Dedup(FlagSet values)
{
#if DEBUG
if (values == null)
{
throw new ArgumentNullException(nameof(values));
}
#endif
return(FlagSetDeduper.GetEqualOrAdd(values));
}
public SuffixEntry(
string strip,
string affixText,
CharacterConditionGroup conditions,
MorphSet morph,
FlagSet contClass)
: base(strip, affixText, conditions, morph, contClass)
{
Key = affixText.GetReversed();
}
protected AffixEntry(
string strip,
string affixText,
CharacterConditionGroup conditions,
MorphSet morph,
FlagSet contClass)
{
Strip = strip ?? string.Empty;
Append = affixText ?? string.Empty;
Conditions = conditions ?? CharacterConditionGroup.Empty;
MorphCode = morph ?? MorphSet.Empty;
ContClass = contClass ?? FlagSet.Empty;
}
internal AffixCollection(
Dictionary <FlagValue, AffixEntryGroup <TEntry> > affixesByFlag,
Dictionary <char, AffixEntryGroupCollection <TEntry> > affixesByIndexedByKey,
AffixEntryGroupCollection <TEntry> affixesWithDots,
AffixEntryGroupCollection <TEntry> affixesWithEmptyKeys,
FlagSet contClasses)
{
AffixesByFlag = affixesByFlag;
AffixesByIndexedByKey = affixesByIndexedByKey;
AffixesWithDots = affixesWithDots;
AffixesWithEmptyKeys = affixesWithEmptyKeys;
ContClasses = contClasses;
HasAffixes = affixesByFlag.Count != 0;
}
private WordList ToImmutable(bool destructive)
{
var affix = Affix ?? new AffixConfig.Builder().MoveToImmutable();
var nGramRestrictedFlags = Dedup(FlagSet.Create(
new[]
{
affix.ForbiddenWord,
affix.NoSuggest,
affix.NoNgramSuggest,
affix.OnlyInCompound,
SpecialFlags.OnlyUpcaseFlag
}
.Where(f => f.HasValue)));
var result = new WordList(affix)
{
NGramRestrictedFlags = nGramRestrictedFlags,
};
if (destructive)
{
result.EntriesByRoot = EntriesByRoot ?? new Dictionary <string, WordEntrySet>();
EntriesByRoot = null;
}
else
{
result.EntriesByRoot = EntriesByRoot == null
? new Dictionary <string, WordEntrySet>()
: new Dictionary <string, WordEntrySet>(EntriesByRoot);
}
var nGramRestrictedEntries = new HashSet <WordEntry>();
foreach (var rootSet in result.EntriesByRoot)
{
foreach (var entry in rootSet.Value)
{
if (nGramRestrictedFlags.ContainsAny(entry.Flags))
{
nGramRestrictedEntries.Add(entry);
}
}
}
result.NGramRestrictedEntries = nGramRestrictedEntries;
return(result);
}
private AffixCollection(
Dictionary <FlagValue, AffixEntryGroup <TEntry> > affixesByFlag,
Dictionary <char, AffixEntryWithDetailCollection <TEntry> > affixesByIndexedByKey,
AffixEntryWithDetailCollection <TEntry> affixesWithDots,
AffixEntryWithDetailCollection <TEntry> affixesWithEmptyKeys,
FlagSet contClasses)
{
this.affixesByFlag = affixesByFlag;
this.affixesByIndexedByKey = affixesByIndexedByKey;
this.affixesWithDots = affixesWithDots;
AffixesWithEmptyKeys = affixesWithEmptyKeys;
ContClasses = contClasses;
HasAffixes = affixesByFlag.Count != 0;
IsEmpty = !HasAffixes;
}
public static TEntry Create <TEntry>(
string strip,
string affixText,
CharacterConditionGroup conditions,
MorphSet morph,
FlagSet contClass)
where TEntry : AffixEntry, new()
=>
new TEntry
{
Strip = strip,
Append = affixText,
Conditions = conditions,
MorphCode = morph ?? MorphSet.Empty,
ContClass = contClass ?? FlagSet.Empty
};
internal List <AffixEntryGroup <TEntry> > GetByFlags(FlagSet flags)
{
#if DEBUG
if (flags == null)
{
throw new ArgumentNullException(nameof(flags));
}
#endif
var results = new List <AffixEntryGroup <TEntry> >(flags.Count);
foreach (var flag in flags)
{
if (AffixesByFlag.TryGetValue(flag, out AffixEntryGroup <TEntry> result))
{
results.Add(result);
}
}
return(results);
}
internal List <AffixEntryGroup <TEntry> > GetAffixesWithEmptyKeysAndFlag(FlagSet flags)
{
#if DEBUG
if (flags == null)
{
throw new ArgumentNullException(nameof(flags));
}
#endif
var results = new List <AffixEntryGroup <TEntry> >(flags.Count);
foreach (var group in AffixesWithEmptyKeys)
{
if (flags.Contains(group.AFlag))
{
results.Add(group);
}
}
return(results);
}
public FlagSet Dedup(FlagSet values) => FlagSetDeduper.GetEqualOrAdd(values);
public FlagSet TakeArrayForFlagSet(FlagValue[] values) => Dedup(FlagSet.TakeArray(values));
private AffixConfig ToImmutable(bool destructive)
{
var culture = CultureInfo.ReadOnly(Culture ?? CultureInfo.InvariantCulture);
var config = new AffixConfig
{
Options = Options,
FlagMode = FlagMode,
KeyString = Dedup(KeyString ?? DefaultKeyString),
TryString = Dedup(TryString ?? string.Empty),
Language = Dedup(Language ?? string.Empty),
Culture = culture,
IsHungarian = string.Equals(culture?.TwoLetterISOLanguageName, "HU", StringComparison.OrdinalIgnoreCase),
StringComparer = new CulturedStringComparer(culture),
CompoundFlag = CompoundFlag,
CompoundBegin = CompoundBegin,
CompoundEnd = CompoundEnd,
CompoundMiddle = CompoundMiddle,
CompoundWordMax = CompoundWordMax,
CompoundMin = CompoundMin ?? DefaultCompoundMinLength,
CompoundRoot = CompoundRoot,
CompoundPermitFlag = CompoundPermitFlag,
CompoundForbidFlag = CompoundForbidFlag,
NoSuggest = NoSuggest,
NoNgramSuggest = NoNgramSuggest,
ForbiddenWord = ForbiddenWord ?? SpecialFlags.ForbiddenWord,
LemmaPresent = LemmaPresent,
Circumfix = Circumfix,
OnlyInCompound = OnlyInCompound,
NeedAffix = NeedAffix,
MaxNgramSuggestions = MaxNgramSuggestions ?? DefaultMaxNgramSuggestions,
MaxDifferency = MaxDifferency,
MaxCompoundSuggestions = MaxCompoundSuggestions ?? DefaultMaxCompoundSuggestions,
KeepCase = KeepCase,
ForceUpperCase = ForceUpperCase,
Warn = Warn,
SubStandard = SubStandard,
CompoundSyllableNum = CompoundSyllableNum,
Encoding = Encoding,
CompoundMaxSyllable = CompoundMaxSyllable,
CompoundVowels = CompoundVowels ?? CharacterSet.Empty,
WordChars = WordChars ?? CharacterSet.Empty,
IgnoredChars = IgnoredChars ?? CharacterSet.Empty,
Version = Dedup(Version),
BreakPoints = BreakSet.Create(BreakPoints)
};
if (destructive)
{
config.Replacements = SingleReplacementSet.TakeList(ReferenceHelpers.Steal(ref Replacements));
config.CompoundRules = CompoundRuleSet.TakeList(ReferenceHelpers.Steal(ref CompoundRules));
config.CompoundPatterns = PatternSet.TakeList(ReferenceHelpers.Steal(ref CompoundPatterns));
config.RelatedCharacterMap = MapTable.TakeList(ReferenceHelpers.Steal(ref RelatedCharacterMap));
config.Phone = PhoneTable.TakeList(ReferenceHelpers.Steal(ref Phone));
config.InputConversions = MultiReplacementTable.TakeDictionary(ReferenceHelpers.Steal(ref InputConversions));
config.OutputConversions = MultiReplacementTable.TakeDictionary(ReferenceHelpers.Steal(ref OutputConversions));
config.Warnings = WarningList.TakeList(ReferenceHelpers.Steal(ref Warnings));
config.aliasF = AliasF ?? new List <FlagSet>(0);
AliasF = null;
config.aliasM = AliasM ?? new List <MorphSet>(0);
AliasM = null;
}
else
{
config.Replacements = SingleReplacementSet.Create(Replacements);
config.CompoundRules = CompoundRuleSet.Create(CompoundRules);
config.CompoundPatterns = PatternSet.Create(CompoundPatterns);
config.RelatedCharacterMap = MapTable.Create(RelatedCharacterMap);
config.Phone = PhoneTable.Create(Phone);
config.InputConversions = MultiReplacementTable.Create(InputConversions);
config.OutputConversions = MultiReplacementTable.Create(OutputConversions);
config.Warnings = WarningList.Create(Warnings);
config.aliasF = AliasF == null ? new List <FlagSet>(0) : AliasF.ToList();
config.aliasM = AliasM == null ? new List <MorphSet>(0) : AliasM.ToList();
}
config.Prefixes = AffixCollection <PrefixEntry> .Create(Prefixes);
config.Suffixes = AffixCollection <SuffixEntry> .Create(Suffixes);
config.ContClasses = FlagSet.Union(config.Prefixes.ContClasses, config.Suffixes.ContClasses);
return(config);
}
internal static FlagSet ParseFlags(StringSlice text, FlagMode mode) => FlagSet.TakeArray(ParseFlagsInOrder(text, mode));
public static FlagSet ParseFlags(string text, FlagMode mode) =>
string.IsNullOrEmpty(text)
? FlagSet.Empty
: FlagSet.TakeArray(ParseFlagsInOrder(text, mode));
public static AffixCollection <TEntry> Create(List <AffixEntryGroup.Builder <TEntry> > builders)
{
if (builders == null || builders.Count == 0)
{
return(Empty);
}
var affixesByFlag = new Dictionary <FlagValue, AffixEntryGroup <TEntry> >(builders.Count);
var affixesByIndexedByKeyBuilders = new Dictionary <char, List <AffixEntryWithDetail <TEntry> > >();
var affixesWithDots = new List <AffixEntryWithDetail <TEntry> >();
var affixesWithEmptyKeys = new List <AffixEntryWithDetail <TEntry> >();
var contClasses = new HashSet <FlagValue>();
foreach (var builder in builders)
{
var group = builder.ToGroup();
affixesByFlag.Add(group.AFlag, group);
foreach (var entry in group.Entries)
{
var key = entry.Key;
contClasses.UnionWith(entry.ContClass);
var entryWithDetail = new AffixEntryWithDetail <TEntry>(group, entry);
if (string.IsNullOrEmpty(key))
{
affixesWithEmptyKeys.Add(entryWithDetail);
}
else
{
if (key.Contains('.'))
{
affixesWithDots.Add(entryWithDetail);
}
else
{
var indexedKey = key[0];
if (!affixesByIndexedByKeyBuilders.TryGetValue(indexedKey, out List <AffixEntryWithDetail <TEntry> > keyedAffixes))
{
keyedAffixes = new List <AffixEntryWithDetail <TEntry> >();
affixesByIndexedByKeyBuilders.Add(indexedKey, keyedAffixes);
}
keyedAffixes.Add(entryWithDetail);
}
}
}
}
var affixesByIndexedByKey = new Dictionary <char, AffixEntryWithDetailCollection <TEntry> >(
affixesByIndexedByKeyBuilders.Count);
foreach (var keyedBuilder in affixesByIndexedByKeyBuilders)
{
affixesByIndexedByKey.Add(keyedBuilder.Key, AffixEntryWithDetailCollection <TEntry> .TakeList(keyedBuilder.Value));
}
return(new AffixCollection <TEntry>
(
affixesByFlag,
affixesByIndexedByKey,
AffixEntryWithDetailCollection <TEntry> .TakeList(affixesWithDots),
AffixEntryWithDetailCollection <TEntry> .TakeList(affixesWithEmptyKeys),
FlagSet.Create(contClasses)
));
}
private bool AddWord(string word, FlagSet flags, MorphSet morphs) => AddWord(word, flags, morphs, false) || AddWordCapitalized(word, flags, morphs, CapitalizationTypeEx.GetCapitalizationType(word, Affix));
public static FlagSet Union(FlagSet a, FlagSet b) => Create(Enumerable.Concat(a, b));
private bool AddWord(string word, FlagSet flags, MorphSet morphs, bool onlyUpperCase)
{
if (Affix.IgnoredChars.HasItems)
{
word = word.RemoveChars(Affix.IgnoredChars);
}
if (Affix.ComplexPrefixes)
{
word = word.Reverse();
if (morphs.HasItems && !Affix.IsAliasM)
{
var newMorphs = new string[morphs.Count];
for (int i = 0; i < morphs.Count; i++)
{
newMorphs[i] = morphs[morphs.Count - i - 1].Reverse();
}
morphs = MorphSet.TakeArray(newMorphs);
}
}
WordEntryOptions options;
if (morphs.HasItems)
{
if (Affix.IsAliasM)
{
options = WordEntryOptions.AliasM;
var morphBuilder = new List <string>();
foreach (var originalValue in morphs)
{
if (IntEx.TryParseInvariant(originalValue, out int morphNumber) && Affix.TryGetAliasM(morphNumber, out MorphSet aliasedMorph))
{
morphBuilder.AddRange(aliasedMorph);
}
else
{
morphBuilder.Add(originalValue);
}
}
morphs = MorphSet.Create(morphBuilder);
}
else
{
options = WordEntryOptions.None;
}
if (morphs.AnyStartsWith(MorphologicalTags.Phon))
{
options |= WordEntryOptions.Phon;
}
}
else
{
options = WordEntryOptions.None;
}
bool saveEntryList = false;
word = Builder.Dedup(word);
if (!Builder.EntriesByRoot.TryGetValue(word, out WordEntrySet entryList))
{
saveEntryList = true;
entryList = WordEntrySet.Empty;
}
var upperCaseHomonym = false;
for (var i = 0; i < entryList.Count; i++)
{
var existingEntry = entryList[i];
if (!onlyUpperCase)
{
if (existingEntry.ContainsFlag(SpecialFlags.OnlyUpcaseFlag))
{
existingEntry = new WordEntry(
existingEntry.Word,
flags,
existingEntry.Morphs,
existingEntry.Options);
entryList.DestructiveReplace(i, existingEntry);
return(false);
}
}
else
{
upperCaseHomonym = true;
}
}
if (!upperCaseHomonym)
{
saveEntryList = true;
entryList = WordEntrySet.CopyWithItemAdded(entryList, new WordEntry(
word,
flags,
Builder.Dedup(morphs),
options));
}
if (saveEntryList)
{
Builder.EntriesByRoot[word] = entryList;
}
return(false);
}
public bool ContainsAny(FlagSet values) => ContainsAny(this, values);
public FlagSet Dedup(FlagSet value) =>
value == null
? value
: value.Count == 0
? FlagSet.Empty
: FlagSetDeduper.GetEqualOrAdd(value);
private WordList ToImmutable(bool destructive)
{
var affix = Affix ?? new AffixConfig.Builder().MoveToImmutable();
var nGramRestrictedFlags = Dedup(FlagSet.Create(
new[]
{
affix.ForbiddenWord,
affix.NoSuggest,
affix.NoNgramSuggest,
affix.OnlyInCompound,
SpecialFlags.OnlyUpcaseFlag
}
.Where(f => f.HasValue)));
var result = new WordList(affix)
{
NGramRestrictedFlags = nGramRestrictedFlags,
};
if (EntryDetailsByRoot == null)
{
result.EntriesByRoot = new Dictionary <string, WordEntryDetail[]>();
}
else
{
result.EntriesByRoot = new Dictionary <string, WordEntryDetail[]>(EntryDetailsByRoot.Count);
foreach (var pair in EntryDetailsByRoot)
{
result.EntriesByRoot.Add(pair.Key, pair.Value.ToArray());
}
if (destructive)
{
EntryDetailsByRoot = null;
}
}
result.AllReplacements = affix.Replacements;
if (PhoneticReplacements != null && PhoneticReplacements.Count != 0)
{
// store ph: field of a morphological description in reptable
if (result.AllReplacements.IsEmpty)
{
result.AllReplacements = SingleReplacementSet.Create(PhoneticReplacements);
}
else
{
result.AllReplacements = SingleReplacementSet.Create(result.AllReplacements.Concat(PhoneticReplacements));
}
}
result.NGramRestrictedDetails = new Dictionary <string, WordEntryDetail[]>();
var details = new List <WordEntryDetail>();
foreach (var rootSet in result.EntriesByRoot)
{
details.Clear();
foreach (var entry in rootSet.Value)
{
if (nGramRestrictedFlags.ContainsAny(entry.Flags))
{
details.Add(entry);
}
}
if (details.Count != 0)
{
result.NGramRestrictedDetails.Add(rootSet.Key, details.ToArray());
}
}
return(result);
}
internal static TResult Create <TResult>(List <AffixEntryGroup <TEntry> .Builder> builders, Constructor <TResult> constructor)
where TResult : AffixCollection <TEntry>
{
var affixesByFlag = new Dictionary <FlagValue, AffixEntryGroup <TEntry> >(builders.Count);
var affixesByIndexedByKeyBuilders = new Dictionary <char, Dictionary <FlagValue, AffixEntryGroup <TEntry> .Builder> >();
var affixesWithEmptyKeys = new List <AffixEntryGroup <TEntry> >();
var affixesWithDots = new List <AffixEntryGroup <TEntry> >();
var contClasses = new HashSet <FlagValue>();
if (builders != null)
{
foreach (var builder in builders)
{
var group = builder.ToGroup();
affixesByFlag.Add(group.AFlag, group);
var entriesWithNoKey = new List <TEntry>();
var entriesWithDots = new List <TEntry>();
foreach (var entry in group.Entries)
{
var key = entry.Key;
contClasses.UnionWith(entry.ContClass);
if (string.IsNullOrEmpty(key))
{
entriesWithNoKey.Add(entry);
}
else if (key.Contains('.'))
{
entriesWithDots.Add(entry);
}
else
{
var indexedKey = key[0];
if (!affixesByIndexedByKeyBuilders.TryGetValue(indexedKey, out var keyedAffixes))
{
keyedAffixes = new Dictionary <FlagValue, AffixEntryGroup <TEntry> .Builder>();
affixesByIndexedByKeyBuilders.Add(indexedKey, keyedAffixes);
}
if (!keyedAffixes.TryGetValue(group.AFlag, out var groupBuilder))
{
groupBuilder = new AffixEntryGroup <TEntry> .Builder
{
AFlag = group.AFlag,
Options = group.Options,
Entries = new List <TEntry>()
};
keyedAffixes.Add(group.AFlag, groupBuilder);
}
groupBuilder.Entries.Add(entry);
}
}
if (entriesWithNoKey.Count > 0)
{
affixesWithEmptyKeys.Add(new AffixEntryGroup <TEntry>(group.AFlag, group.Options, AffixEntryCollection <TEntry> .Create(entriesWithNoKey)));
}
if (entriesWithDots.Count > 0)
{
affixesWithDots.Add(new AffixEntryGroup <TEntry>(group.AFlag, group.Options, AffixEntryCollection <TEntry> .Create(entriesWithDots)));
}
}
}
var affixesByIndexedByKey = new Dictionary <char, AffixEntryGroupCollection <TEntry> >(affixesByIndexedByKeyBuilders.Count);
foreach (var keyedBuilder in affixesByIndexedByKeyBuilders)
{
var indexedAffixGroup = new AffixEntryGroup <TEntry> [keyedBuilder.Value.Count];
var writeIndex = 0;
foreach (var builderPair in keyedBuilder.Value)
{
indexedAffixGroup[writeIndex++] = builderPair.Value.ToGroup();
}
affixesByIndexedByKey.Add(keyedBuilder.Key, AffixEntryGroupCollection <TEntry> .TakeArray(indexedAffixGroup));
}
return(constructor
(
affixesByFlag,
affixesByIndexedByKey,
AffixEntryGroupCollection <TEntry> .Create(affixesWithDots),
AffixEntryGroupCollection <TEntry> .Create(affixesWithEmptyKeys),
FlagSet.Create(contClasses)
));
}
public FlagSet Dedup(FlagSet value) => value == null ? null : FlagSetDeduper.GetEqualOrAdd(value);