internal WordVocabularySource(ApplicationModel model) : base(model, predictorWidth: 4) { var words = model.Environment.GetOrderedSeedWords(); _tokens = StringTokens.Create(words); _tileFilter = new WordTileFilter(this, _tokens, CultureInfo.CurrentUICulture); _headItems.Add(new HeadStartItem(this)); _tailItems.Add(null); // Create the slot for the tail item. SetSelectedIndex(0); HeadItems = new ReadOnlyObservableCollection <ITile>(_headItems); TailItems = new ReadOnlyObservableCollection <ITile>(_tailItems); PopulateVocabularyList(); InitializeUtterance(); SetRunOnSuggestions(); var spellingSource = new SpellingVocabularySource(model, this); _spellingSource = spellingSource; ParanoidAssertValid(); }
private SuggestedWordListsCreator(WordVocabularySource source, StringTokens tokens, ScoredTokenPredictionMaker maker, Func <int, bool> isTokenVisible, bool startOnFirstWord, int lowerBound, int upperBound, int maxListCount, int maxListItemCount) { _source = source; _tokens = tokens; _maker = maker; _isTokenVisible = isTokenVisible; _startOnFirstWord = startOnFirstWord; _lowerBound = lowerBound; _upperBound = upperBound; _maxListCount = maxListCount; _maxListItemCount = maxListItemCount; var settings = source.Model.Environment.Settings; _allowDuplicateStems = settings.AllowDuplicateStems; _findFollowOnPredictions = settings.FindFollowOnPredictions; _combineCorePredictions = settings.CombineCorePredictions; _findCorePredictionPrefixes = settings.FindCorePredictionPrefixes; _findCorePredictionSuffixes = settings.FindCorePredictionSuffixes; _nascents = new List <NascentWordPredictionList>(_maxListCount); _capitalizer = source.Model.Environment.TryCapitalizeFirstWord; }
private static IDictionary <string, object> ToJsonDictionary(StringTokens tokens, TokenPredictorDatabase database) { var json = new SortedDictionary <string, object>(); foreach (var pair in database) { var key = tokens.GetString(pair.Key); var value = ToJsonDictionary(tokens, pair.Value); json.Add(key, value); } return(json); }
private static IDictionary <string, object> ToJsonDictionary(StringTokens tokens, TokenPredictorInfo info) { var saved = new SortedDictionary <string, object> { { "#", info.Count } }; var children = info.TryGetChildren(); if (children != null) { var value = ToJsonDictionary(tokens, children); saved.Add("~", value); } return(saved); }
static void BreakOnProblemCase(StringTokens tokens, int[] context, params string[] words) { if (words.Length <= context.Length) { var contextLim = context.Length; var wordsLim = words.Length; while (0 < wordsLim && tokens.TryGetToken(words[wordsLim - 1], out var token) && context[contextLim - 1] == token) { wordsLim--; contextLim--; } if (wordsLim == 0) { Debugger.Break(); } } }
internal static void PrototypeTreePredictions(ScoredTokenPredictionMaker maker, StringTokens tokens, int maxCount) { string ToText(TokenPredictorInfo info) { var text = tokens[info.Token]; var nullIndex = text.IndexOf('\0'); if (nullIndex != -1) { text = text.Substring(0, nullIndex); } return(text); } var roots = maker.GetContextRoots(); Debug.WriteLine(string.Empty); Debug.WriteLine("Prototype tree prediction:"); var enumerable = TreePredictableItem <TokenPredictorInfo> .FindOrderedItems(roots.SortedEnumerable); using (var enumerator = enumerable.GetEnumerator()) { for (var count = 0; count < maxCount && enumerator.MoveNext(); count++) { var text = string.Empty; foreach (var info in enumerator.Current) { text += $"{ToText(info)} "; } Debug.WriteLine(text); } } Debug.WriteLine(string.Empty); }
internal static SortedList <int, IReadOnlyList <ITile> > CreateSuggestionLists(WordVocabularySource source, StringTokens tokens, TokenPredictor tokenPredictor, int[] context, ITokenTileFilter filter, bool isFirstWord, int lowerBound, int upperBound, int maxListCount, int maxListItemCount) { BreakOnProblemCase(tokens, context, "these", "are", "the", "voyages"); var maker = tokenPredictor.CreatePredictionMaker(source, filter.IsTokenVisible, context); var creator = new SuggestedWordListsCreator(source, tokens, maker, filter.IsTokenVisible, isFirstWord, lowerBound, upperBound, maxListCount, maxListItemCount); var list = creator.Run(); return(list); }
public WordTileFilter(WordVocabularySource source, StringTokens tokens, CultureInfo culture) { _source = source; _tokens = tokens; _compare = culture.CompareInfo; }
internal IDictionary <string, object> ToJsonDictionary(StringTokens tokens) { var json = ToJsonDictionary(tokens, _database); return(json); }
/// <summary> /// Create a default set of English word tokens. /// </summary> /// <returns>A StringTokens object.</returns> public static StringTokens Create(IEnumerable <string> seed) { var value = new StringTokens(seed); return(value); }