private TokenHashes[] ComputeTokenHashes(IList <Core.Tokenization.Token> sourceObjects)
        {
            TokenHashes[] result = new TokenHashes[sourceObjects.Count];
            for (int p = 0; p < sourceObjects.Count; ++p)
            {
                Core.Tokenization.Token       t  = sourceObjects[p];
                Core.Tokenization.SimpleToken st = t as Core.Tokenization.SimpleToken;

                TokenHashes th = new TokenHashes();

                th.TextHash = t.Text == null ? 0 : t.Text.GetHashCode();
                th.CaseInsensitiveTextHash = t.Text == null ? 0 : t.Text.ToLowerInvariant().GetHashCode();
                th.StemHash = (st == null || st.Stem == null)
                                        ? 0
                                        : st.Stem.ToLowerInvariant().GetHashCode();

                result[p] = th;
            }
            return(result);
        }
        public static double GetTokenSimilarity(Core.Tokenization.Token a, Core.Tokenization.Token b,
                                                bool useStringEditDistance,
                                                Core.Tokenization.BuiltinRecognizers disabledAutoSubstitutions)
        {
            /*
             * identical form - 1.0
             * identical case-insensitive - 0.9
             * stem-identical or same placeable type - 0.85
             * identical type - 0.4
             * */

            // TODO consider normalization of token texts, or let the tokenizer store the text in normalized form.

            Core.Tokenization.TagToken ta = a as Core.Tokenization.TagToken;
            Core.Tokenization.TagToken tb = b as Core.Tokenization.TagToken;

            bool aIsTag = (ta != null);
            bool bIsTag = (tb != null);

            if (aIsTag != bIsTag ||
                a.IsWhitespace != b.IsWhitespace ||
                a.IsPunctuation != b.IsPunctuation)
            {
                // comparing a tag with a non-tag results in no-change-allowed similarity (<0)
                // same for whitespace, punctuation
                return(-1.0d);
            }

            if (aIsTag && bIsTag)
            {
                System.Diagnostics.Debug.Assert(ta.Tag != null && tb.Tag != null);
                if (ta.Tag.Type == tb.Tag.Type)
                {
                    // assignable
                    return(0.95d);
                }
                else if ((ta.Tag.Type == Sdl.LanguagePlatform.Core.TagType.Standalone &&
                          tb.Tag.Type == Sdl.LanguagePlatform.Core.TagType.TextPlaceholder)
                         ||
                         (ta.Tag.Type == Sdl.LanguagePlatform.Core.TagType.TextPlaceholder &&
                          tb.Tag.Type == Sdl.LanguagePlatform.Core.TagType.Standalone))
                {
                    // one placeholder, one text placeholder
                    return(0.85d);
                }
                else
                {
                    // not assignable
                    return(-1.0d);
                }
            }

            double malus = 0.0d;
            double sim   = 0.0d;

            if (a.IsPlaceable && b.IsPlaceable)
            {
                return(GetPlaceableSimilarity(a, b, disabledAutoSubstitutions));
            }

            if (a.Text == null || b.Text == null)
            {
                System.Diagnostics.Debug.Assert(false, "Expected non-null token text. Let Oli know if this assertion fires and provide test data.");
                return(0.0d);
            }

            if (a.IsWord != b.IsWord)
            {
                // tokens of different types - reduce similarity accordingly
                // NOTE only checks whether both are words or non-words
                malus = 0.1d;
            }

            if (a.Text.Equals(b.Text, StringComparison.Ordinal))
            {
                sim = 1.0d;
            }
            else if (a.IsWhitespace || a.IsPunctuation)
            {
                // slightly less than the SegmentEditDistanceComputer's move threshold, as
                //  we don't want to move them around
                sim = 0.94d;
            }
            else if (a.Text.Equals(b.Text, StringComparison.OrdinalIgnoreCase))
            {
                // we want to detect moves for such tokens, so:
                sim = 0.95d;                 // the SegmentEditDistanceComputer's move threshold
            }
            else if (a is Core.Tokenization.SimpleToken && b is Core.Tokenization.SimpleToken)
            {
                Core.Tokenization.SimpleToken ast = a as Core.Tokenization.SimpleToken;
                Core.Tokenization.SimpleToken bst = b as Core.Tokenization.SimpleToken;

                if (ast != null && bst != null &&
                    ast.Stem != null && bst.Stem != null &&
                    ast.Stem.Equals(bst.Stem, StringComparison.OrdinalIgnoreCase))
                {
                    sim = 0.85d;
                }
                else
                {
                    sim = useStringEditDistance
                                                ? 0.95d * GetThreshold(GetStringSimilarity(a.Text, b.Text))
                                                : 0.0d;
                }
            }
            else
            {
                // strings are not identical or identical w/ ignore case
                sim = useStringEditDistance
                                        ? 0.95d * GetThreshold(GetStringSimilarity(a.Text, b.Text))
                                        : 0.0d;
            }

            return(Math.Max(0.0d, sim - malus));
        }