/// <summary>
        /// Is character a Variation Selector? [https://en.wikipedia.org/wiki/Variation_Selectors_(Unicode_block)]
        /// Note that these are included in IsCombiner as well.
        /// </summary>
        //public static bool IsVariationSelector(this char c) => c.IsInRange(0xfe00, 0xfe0f);

        /// <summary>
        /// Helper to determine whether a character is a numeral.
        /// This includes numeral characters that are not classified as such in Unicode,
        /// such as Chinese numbers.
        /// This is meant for FactoredSegmenter, which uses this to prevent numeral characters
        /// from being merged in SentencePiece.
        public static bool IsNumeral(this char c)
        {
            // @BUGBUG: currently known failures:
            //  - Arabic fractions: ٠٫٢٥

            // Chinese numeral letters are not classified as digits in Unicode
            if (ScriptHelpers.ChineseDigits.Contains(c))
            {
                return(true);
            }
            else
            {
                return(Unicode.GetUnicodeMajorDesignation(c) == 'N');
            }
        }
        /// <summary>
        /// Get script designators for each character in a line.
        /// This function handles surrogate pairs and combining marks.   --@TODO: ...not yet, actually
        /// The function can optionally operate on a substring.
        /// </summary>
        //public static Unicode.Script[] GetScripts(string line, int startIndex = 0, int length = int.MaxValue)
        //{
        //    if (length == int.MaxValue)
        //        length = line.Length;
        //    var scripts = new Unicode.Script[length];
        //    for (var i = 0; i < length; i++)
        //    {
        //        // @TODO: Handle surrogates
        //        char c = line[startIndex + i];
        //        if (c.IsCombiner() && i > 0)
        //            scripts[i] = scripts[i - 1];
        //        else
        //            scripts[i] = Unicode.GetScript(c);
        //    }
        //    return scripts;
        //}

        /// <summary>
        /// Simplistic word-boundary detector.
        /// This function attempts to detect word boundaries that can be detected in a language-independent
        /// fashion from the surface form, and without additional knowledge sources.
        /// I.e. it looks for a change in script and some changes in Unicode character designation.
        /// This does not detect word breaks in continuous scripts, which require additional knowledge sources.
        /// This function handles these special cases:
        ///  - some known allowed punctuation between characters, such as ' in words and . in numbers
        ///    @TODO: This rule may not apply to all scripts.
        ///  - surrogate pairs    --@TODO
        ///  - combiners inherit the script of the character to the left
        ///  - combiners are classified as the char type (major designation) they are "typically"
        ///    applied to (not depending on actual char).
        ///    This is needed so that combiners that end up as single SentencePieces are classifyable.
        ///    Any error this causes must be learned by the model.
        ///  - designation changes only are a boundary if a letter or a number is on either side,
        ///    but e.g. not a punctuation symbol next to a space or math symbol
        ///  - (special rule: Hiragana is not split from Kanji. Currently this rule is disabled.)
        /// Each space gives rise to two boundaries (one on each side).
        /// It returns a cut list. An empty string is not cut.
        /// </summary>
        public static IList <int> DetectUnambiguousWordBreaks(string line) // @TODO: Better name for this?
        {
            // First, determines the major Unicode designation and script for each character, but with modifications,
            // for purpose of simple word breaking:
            //  - allowed punctuation marks inside words are flipped to 'L'
            //  - allowed punctuation marks inside numbers are flipped to 'N'
            //  - unambiguous CJK number letters are flipped to 'N'
            //  - combining marks carry over both designation and script from their main character
            var scripts      = new Unicode.Script[line.Length];
            var designations = new char[line.Length];

            for (var i = 0; i < line.Length; i++)
            {
                var c = line[i];
                // @TODO: handle surrogate pairs
                var m = Unicode.GetUnicodeMajorDesignation(c);
                var s = Unicode.GetScript(c);
                // special case: consider unambiguous CJK number symbols as numerals
                if (c.IsNumeral())
                {
                    designations[i] = 'N';
                }
                // special case: combining marks carry over main character's script, and are classified as their most likely use (for consistency)
                else if (m == 'M')
                {
                    m = c.GetCombinerTypicalMajorDesignation();
                    if (i > 0)
                    {
                        s = scripts[i - 1];
                    }
                }
                designations[i] = m;
                scripts[i]      = s;
                // special case: allowed punctuation inside a word  --@TODO: Likely script dependent, maybe language dependent
                if (i - 2 >= 0 && designations[i] == 'L' && designations[i - 2] == 'L' && IsValidPuncInsideWord(line[i - 1]))
                {
                    designations[i - 1] = 'L';
                }
                // special case: allowed punctuation inside a number  --@TODO: Likely script dependent, maybe language-locale dependent
                else if (i - 2 >= 0 && designations[i] == 'N' && designations[i - 2] == 'N' && IsValidPuncInsideNumber(line[i - 1]))
                {
                    designations[i - 1] = 'N';
                }
                // @TODO: double-check handling of space characters: non-breaking space; optional hyphen
            }

            // This function operates on a string, so we can handle the case of Unicode.Script 1 - Common - Unicode.Script 2
            // This presently breaks this as (Unicode.Script 1 - Common, Unicode.Script 2).
            // Without further knowledge, we can only make an arbitrary hard choice here.
            // This is used by FactoredSegmenter, where that is OK because characters in Common are
            // typically broken off anyways.

            if (line.Length == 0) // graceful exit in case of empty input
            {
                return new List <int> {
                           0, 0
                }
            }
            ;                                 // empty input is not cut

            var cutList = new List <int>(200)
            {
                0
            };                                      // (0=line start, which the resulting cut list must include)
            var lastNonCommonScript = scripts[0];

            //if (lastNonCommonScript == Unicode.Script.Hiragana)
            //    lastNonCommonScript = Unicode.Script.Han; // no boundary between Kanji and Hiragana
            for (var pos = 1; pos < line.Length; pos++)
            {
                // detect change in character designation
                //  - break at number boundaries
                //     - add number factor
                //     - can numbers be part of words that need to be kept together for determining word-level factors?
                //  - break at word boundaries
                //     - letter/non-letter transitions
                //     - don't break apostrophes and hyphens with letters on both sides
                //     - break at script boundaries
                bool atDesignationChange = (designations[pos - 1] != designations[pos] &&
                                            (designations[pos - 1] == 'N' || designations[pos] == 'N' ||
                                             designations[pos - 1] == 'L' || designations[pos] == 'L'));

                // detect script change
                var thisScript = scripts[pos];
                //if (thisScript == Unicode.Script.Hiragana) // the jury is still out whether we should do this or not
                //    thisScript = Unicode.Script.Han;
                bool atScriptChange = lastNonCommonScript != thisScript && thisScript != Unicode.Script.Common;
                // Note: If there is a script change across Common, we choose one arbitrarily.
                if (thisScript != Unicode.Script.Common || atDesignationChange) // condition 'atDesignationChange' is for back compat only; maybe not needed
                {
                    lastNonCommonScript = thisScript;
                }

                // add cut point if one was found
                if (atDesignationChange || atScriptChange)
                {
                    cutList.Add(pos);
                }
            }
            cutList.Add(line.Length);
            return(cutList);
        }