Exemplo n.º 1
0
        public override bool Equals(object other)
        {
            if (other == null)
            {
                return(false);
            }
            if (!this.GetType().Equals(other.GetType()))
            {
                return(false);
            }
            CollationSettings o = (CollationSettings)other;

            if (options != o.options)
            {
                return(false);
            }
            if ((options & AlternateMask) != 0 && variableTop != o.variableTop)
            {
                return(false);
            }
            if (!ArrayEqualityComparer <int> .OneDimensional.Equals(reorderCodes, o.reorderCodes))
            {
                return(false);
            }
            return(true);
        }
Exemplo n.º 2
0
        public override bool Equals(object other)
        {
            if (other == null)
            {
                return(false);
            }
            if (!this.GetType().Equals(other.GetType()))
            {
                return(false);
            }
            CollationSettings o = (CollationSettings)other;

            if (options != o.options)
            {
                return(false);
            }
            if ((options & ALTERNATE_MASK) != 0 && variableTop != o.variableTop)
            {
                return(false);
            }
            if (!Arrays.Equals(reorderCodes, o.reorderCodes))
            {
                return(false);
            }
            return(true);
        }
Exemplo n.º 3
0
        public override object Clone()
        {
            CollationSettings newSettings = (CollationSettings)base.Clone();

            // Note: The reorderTable, reorderRanges, and reorderCodes need not be cloned
            // because, in Java, they only get replaced but not modified.
            newSettings.fastLatinPrimaries = (char[])fastLatinPrimaries.Clone();
            return(newSettings);
        }
Exemplo n.º 4
0
 public void CopyReorderingFrom(CollationSettings other)
 {
     if (!other.HasReordering)
     {
         ResetReordering();
         return;
     }
     minHighNoReorder = other.minHighNoReorder;
     reorderTable     = other.reorderTable;
     reorderRanges    = other.reorderRanges;
     reorderCodes     = other.reorderCodes;
 }
Exemplo n.º 5
0
        /**
         * Writes the sort key bytes for minLevel up to the iterator data's strength. Optionally writes
         * the case level. Stops writing levels when callback.needToWrite(level) returns false.
         * Separates levels with the LEVEL_SEPARATOR_BYTE but does not write a TERMINATOR_BYTE.
         */
        public static void WriteSortKeyUpToQuaternary(CollationIterator iter, bool[] compressibleBytes,
                                                      CollationSettings settings, SortKeyByteSink sink, int minLevel, LevelCallback callback,
                                                      bool preflight)
        {
            int options = settings.Options;
            // Set of levels to process and write.
            int levels = levelMasks[(int)CollationSettings.GetStrength(options)];

            if ((options & CollationSettings.CASE_LEVEL) != 0)
            {
                levels |= Collation.CASE_LEVEL_FLAG;
            }
            // Minus the levels below minLevel.
            levels &= ~((1 << minLevel) - 1);
            if (levels == 0)
            {
                return;
            }

            long variableTop;

            if ((options & CollationSettings.ALTERNATE_MASK) == 0)
            {
                variableTop = 0;
            }
            else
            {
                // +1 so that we can use "<" and primary ignorables test out early.
                variableTop = settings.VariableTop + 1;
            }

            int tertiaryMask = CollationSettings.GetTertiaryMask(options);

            byte[]       p234         = new byte[3];
            SortKeyLevel cases        = GetSortKeyLevel(levels, Collation.CASE_LEVEL_FLAG);
            SortKeyLevel secondaries  = GetSortKeyLevel(levels, Collation.SECONDARY_LEVEL_FLAG);
            SortKeyLevel tertiaries   = GetSortKeyLevel(levels, Collation.TERTIARY_LEVEL_FLAG);
            SortKeyLevel quaternaries = GetSortKeyLevel(levels, Collation.QUATERNARY_LEVEL_FLAG);

            long prevReorderedPrimary = 0;  // 0==no compression
            int  commonCases          = 0;
            int  commonSecondaries    = 0;
            int  commonTertiaries     = 0;
            int  commonQuaternaries   = 0;

            int prevSecondary   = 0;
            int secSegmentStart = 0;

            for (; ;)
            {
                // No need to keep all CEs in the buffer when we write a sort key.
                iter.ClearCEsIfNoneRemaining();
                long ce = iter.NextCE();
                long p  = ce.TripleShift(32);
                if (p < variableTop && p > Collation.MERGE_SEPARATOR_PRIMARY)
                {
                    // Variable CE, shift it to quaternary level.
                    // Ignore all following primary ignorables, and shift further variable CEs.
                    if (commonQuaternaries != 0)
                    {
                        --commonQuaternaries;
                        while (commonQuaternaries >= QUAT_COMMON_MAX_COUNT)
                        {
                            quaternaries.AppendByte(QUAT_COMMON_MIDDLE);
                            commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
                        }
                        // Shifted primary weights are lower than the common weight.
                        quaternaries.AppendByte(QUAT_COMMON_LOW + commonQuaternaries);
                        commonQuaternaries = 0;
                    }
                    do
                    {
                        if ((levels & Collation.QUATERNARY_LEVEL_FLAG) != 0)
                        {
                            if (settings.HasReordering)
                            {
                                p = settings.Reorder(p);
                            }
                            if (((int)p.TripleShift(24)) >= QUAT_SHIFTED_LIMIT_BYTE)
                            {
                                // Prevent shifted primary lead bytes from
                                // overlapping with the common compression range.
                                quaternaries.AppendByte(QUAT_SHIFTED_LIMIT_BYTE);
                            }
                            quaternaries.AppendWeight32(p);
                        }
                        do
                        {
                            ce = iter.NextCE();
                            p  = ce.TripleShift(32);
                        } while (p == 0);
                    } while (p < variableTop && p > Collation.MERGE_SEPARATOR_PRIMARY);
                }
                // ce could be primary ignorable, or NO_CE, or the merge separator,
                // or a regular primary CE, but it is not variable.
                // If ce==NO_CE, then write nothing for the primary level but
                // terminate compression on all levels and then exit the loop.
                if (p > Collation.NO_CE_PRIMARY && (levels & Collation.PRIMARY_LEVEL_FLAG) != 0)
                {
                    // Test the un-reordered primary for compressibility.
                    bool isCompressible = compressibleBytes[(int)p.TripleShift(24)];
                    if (settings.HasReordering)
                    {
                        p = settings.Reorder(p);
                    }
                    int p1 = (int)p.TripleShift(24);
                    if (!isCompressible || p1 != ((int)prevReorderedPrimary.TripleShift(24)))
                    {
                        if (prevReorderedPrimary != 0)
                        {
                            if (p < prevReorderedPrimary)
                            {
                                // No primary compression terminator
                                // at the end of the level or merged segment.
                                if (p1 > Collation.MERGE_SEPARATOR_BYTE)
                                {
                                    sink.Append(Collation.PRIMARY_COMPRESSION_LOW_BYTE);
                                }
                            }
                            else
                            {
                                sink.Append(Collation.PRIMARY_COMPRESSION_HIGH_BYTE);
                            }
                        }
                        sink.Append(p1);
                        if (isCompressible)
                        {
                            prevReorderedPrimary = p;
                        }
                        else
                        {
                            prevReorderedPrimary = 0;
                        }
                    }
                    byte p2 = (byte)(p.TripleShift(16));
                    if (p2 != 0)
                    {
                        p234[0] = p2;
                        p234[1] = (byte)(p.TripleShift(8));
                        p234[2] = (byte)p;
                        sink.Append(p234, (p234[1] == 0) ? 1 : (p234[2] == 0) ? 2 : 3);
                    }
                    // Optimization for internalNextSortKeyPart():
                    // When the primary level overflows we can stop because we need not
                    // calculate (preflight) the whole sort key length.
                    if (!preflight && sink.Overflowed)
                    {
                        // not used in Java -- if (!sink.IsOk()) {
                        // Java porting note: U_MEMORY_ALLOCATION_ERROR is set here in
                        // C implementation. IsOk() in Java always returns true, so this
                        // is a dead code.
                        return;
                    }
                }

                int lower32 = (int)ce;
                if (lower32 == 0)
                {
                    continue;
                } // completely ignorable, no secondary/case/tertiary/quaternary

                if ((levels & Collation.SECONDARY_LEVEL_FLAG) != 0)
                {
                    int s = lower32.TripleShift(16);  // 16 bits
                    if (s == 0)
                    {
                        // secondary ignorable
                    }
                    else if (s == Collation.COMMON_WEIGHT16 &&
                             ((options & CollationSettings.BACKWARD_SECONDARY) == 0 ||
                              p != Collation.MERGE_SEPARATOR_PRIMARY))
                    {
                        // s is a common secondary weight, and
                        // backwards-secondary is off or the ce is not the merge separator.
                        ++commonSecondaries;
                    }
                    else if ((options & CollationSettings.BACKWARD_SECONDARY) == 0)
                    {
                        if (commonSecondaries != 0)
                        {
                            --commonSecondaries;
                            while (commonSecondaries >= SEC_COMMON_MAX_COUNT)
                            {
                                secondaries.AppendByte(SEC_COMMON_MIDDLE);
                                commonSecondaries -= SEC_COMMON_MAX_COUNT;
                            }
                            int b;
                            if (s < Collation.COMMON_WEIGHT16)
                            {
                                b = SEC_COMMON_LOW + commonSecondaries;
                            }
                            else
                            {
                                b = SEC_COMMON_HIGH - commonSecondaries;
                            }
                            secondaries.AppendByte(b);
                            commonSecondaries = 0;
                        }
                        secondaries.AppendWeight16(s);
                    }
                    else
                    {
                        if (commonSecondaries != 0)
                        {
                            --commonSecondaries;
                            // Append reverse weights. The level will be re-reversed later.
                            int remainder = commonSecondaries % SEC_COMMON_MAX_COUNT;
                            int b;
                            if (prevSecondary < Collation.COMMON_WEIGHT16)
                            {
                                b = SEC_COMMON_LOW + remainder;
                            }
                            else
                            {
                                b = SEC_COMMON_HIGH - remainder;
                            }
                            secondaries.AppendByte(b);
                            commonSecondaries -= remainder;
                            // commonSecondaries is now a multiple of SEC_COMMON_MAX_COUNT.
                            while (commonSecondaries > 0)
                            { // same as >= SEC_COMMON_MAX_COUNT
                                secondaries.AppendByte(SEC_COMMON_MIDDLE);
                                commonSecondaries -= SEC_COMMON_MAX_COUNT;
                            }
                            // commonSecondaries == 0
                        }
                        if (0 < p && p <= Collation.MERGE_SEPARATOR_PRIMARY)
                        {
                            // The backwards secondary level compares secondary weights backwards
                            // within segments separated by the merge separator (U+FFFE).
                            byte[] secs = secondaries.Data();
                            int    last = secondaries.Length - 1;
                            while (secSegmentStart < last)
                            {
                                byte b = secs[secSegmentStart];
                                secs[secSegmentStart++] = secs[last];
                                secs[last--]            = b;
                            }
                            secondaries.AppendByte(p == Collation.NO_CE_PRIMARY ?
                                                   Collation.LEVEL_SEPARATOR_BYTE : Collation.MERGE_SEPARATOR_BYTE);
                            prevSecondary   = 0;
                            secSegmentStart = secondaries.Length;
                        }
                        else
                        {
                            secondaries.AppendReverseWeight16(s);
                            prevSecondary = s;
                        }
                    }
                }

                if ((levels & Collation.CASE_LEVEL_FLAG) != 0)
                {
                    if ((CollationSettings.GetStrength(options) == (int)CollationStrength.Primary) ? p == 0
                            : (lower32.TripleShift(16)) == 0)
                    {
                        // Primary+caseLevel: Ignore case level weights of primary ignorables.
                        // Otherwise: Ignore case level weights of secondary ignorables.
                        // For details see the comments in the CollationCompare class.
                    }
                    else
                    {
                        int c = (lower32.TripleShift(8)) & 0xff; // case bits & tertiary lead byte
                        Debug.Assert((c & 0xc0) != 0xc0);
                        if ((c & 0xc0) == 0 && c > Collation.LEVEL_SEPARATOR_BYTE)
                        {
                            ++commonCases;
                        }
                        else
                        {
                            if ((options & CollationSettings.UPPER_FIRST) == 0)
                            {
                                // lowerFirst: Compress common weights to nibbles 1..7..13, mixed=14,
                                // upper=15.
                                // If there are only common (=lowest) weights in the whole level,
                                // then we need not write anything.
                                // Level length differences are handled already on the next-higher level.
                                if (commonCases != 0 &&
                                    (c > Collation.LEVEL_SEPARATOR_BYTE || !cases.IsEmpty))
                                {
                                    --commonCases;
                                    while (commonCases >= CASE_LOWER_FIRST_COMMON_MAX_COUNT)
                                    {
                                        cases.AppendByte(CASE_LOWER_FIRST_COMMON_MIDDLE << 4);
                                        commonCases -= CASE_LOWER_FIRST_COMMON_MAX_COUNT;
                                    }
                                    int b;
                                    if (c <= Collation.LEVEL_SEPARATOR_BYTE)
                                    {
                                        b = CASE_LOWER_FIRST_COMMON_LOW + commonCases;
                                    }
                                    else
                                    {
                                        b = CASE_LOWER_FIRST_COMMON_HIGH - commonCases;
                                    }
                                    cases.AppendByte(b << 4);
                                    commonCases = 0;
                                }
                                if (c > Collation.LEVEL_SEPARATOR_BYTE)
                                {
                                    c = (CASE_LOWER_FIRST_COMMON_HIGH + (c.TripleShift(6))) << 4; // 14 or 15
                                }
                            }
                            else
                            {
                                // upperFirst: Compress common weights to nibbles 3..15, mixed=2,
                                // upper=1.
                                // The compressed common case weights only go up from the "low" value
                                // because with upperFirst the common weight is the highest one.
                                if (commonCases != 0)
                                {
                                    --commonCases;
                                    while (commonCases >= CASE_UPPER_FIRST_COMMON_MAX_COUNT)
                                    {
                                        cases.AppendByte(CASE_UPPER_FIRST_COMMON_LOW << 4);
                                        commonCases -= CASE_UPPER_FIRST_COMMON_MAX_COUNT;
                                    }
                                    cases.AppendByte((CASE_UPPER_FIRST_COMMON_LOW + commonCases) << 4);
                                    commonCases = 0;
                                }
                                if (c > Collation.LEVEL_SEPARATOR_BYTE)
                                {
                                    c = (CASE_UPPER_FIRST_COMMON_LOW - (c.TripleShift(6))) << 4; // 2 or 1
                                }
                            }
                            // c is a separator byte 01,
                            // or a left-shifted nibble 0x10, 0x20, ... 0xf0.
                            cases.AppendByte(c);
                        }
                    }
                }

                if ((levels & Collation.TERTIARY_LEVEL_FLAG) != 0)
                {
                    int t = lower32 & tertiaryMask;
                    Debug.Assert((lower32 & 0xc000) != 0xc000);
                    if (t == Collation.COMMON_WEIGHT16)
                    {
                        ++commonTertiaries;
                    }
                    else if ((tertiaryMask & 0x8000) == 0)
                    {
                        // Tertiary weights without case bits.
                        // Move lead bytes 06..3F to C6..FF for a large common-weight range.
                        if (commonTertiaries != 0)
                        {
                            --commonTertiaries;
                            while (commonTertiaries >= TER_ONLY_COMMON_MAX_COUNT)
                            {
                                tertiaries.AppendByte(TER_ONLY_COMMON_MIDDLE);
                                commonTertiaries -= TER_ONLY_COMMON_MAX_COUNT;
                            }
                            int b;
                            if (t < Collation.COMMON_WEIGHT16)
                            {
                                b = TER_ONLY_COMMON_LOW + commonTertiaries;
                            }
                            else
                            {
                                b = TER_ONLY_COMMON_HIGH - commonTertiaries;
                            }
                            tertiaries.AppendByte(b);
                            commonTertiaries = 0;
                        }
                        if (t > Collation.COMMON_WEIGHT16)
                        {
                            t += 0xc000;
                        }
                        tertiaries.AppendWeight16(t);
                    }
                    else if ((options & CollationSettings.UPPER_FIRST) == 0)
                    {
                        // Tertiary weights with caseFirst=lowerFirst.
                        // Move lead bytes 06..BF to 46..FF for the common-weight range.
                        if (commonTertiaries != 0)
                        {
                            --commonTertiaries;
                            while (commonTertiaries >= TER_LOWER_FIRST_COMMON_MAX_COUNT)
                            {
                                tertiaries.AppendByte(TER_LOWER_FIRST_COMMON_MIDDLE);
                                commonTertiaries -= TER_LOWER_FIRST_COMMON_MAX_COUNT;
                            }
                            int b;
                            if (t < Collation.COMMON_WEIGHT16)
                            {
                                b = TER_LOWER_FIRST_COMMON_LOW + commonTertiaries;
                            }
                            else
                            {
                                b = TER_LOWER_FIRST_COMMON_HIGH - commonTertiaries;
                            }
                            tertiaries.AppendByte(b);
                            commonTertiaries = 0;
                        }
                        if (t > Collation.COMMON_WEIGHT16)
                        {
                            t += 0x4000;
                        }
                        tertiaries.AppendWeight16(t);
                    }
                    else
                    {
                        // Tertiary weights with caseFirst=upperFirst.
                        // Do not change the artificial uppercase weight of a tertiary CE (0.0.ut),
                        // to keep tertiary CEs well-formed.
                        // Their case+tertiary weights must be greater than those of
                        // primary and secondary CEs.
                        //
                        // Separator         01 -> 01      (unchanged)
                        // Lowercase     02..04 -> 82..84  (includes uncased)
                        // Common weight     05 -> 85..C5  (common-weight compression range)
                        // Lowercase     06..3F -> C6..FF
                        // Mixed case    42..7F -> 42..7F
                        // Uppercase     82..BF -> 02..3F
                        // Tertiary CE   86..BF -> C6..FF
                        if (t <= Collation.NO_CE_WEIGHT16)
                        {
                            // Keep separators unchanged.
                        }
                        else if ((lower32.TripleShift(16)) != 0)
                        {
                            // Invert case bits of primary & secondary CEs.
                            t ^= 0xc000;
                            if (t < (TER_UPPER_FIRST_COMMON_HIGH << 8))
                            {
                                t -= 0x4000;
                            }
                        }
                        else
                        {
                            // Keep uppercase bits of tertiary CEs.
                            Debug.Assert(0x8600 <= t && t <= 0xbfff);
                            t += 0x4000;
                        }
                        if (commonTertiaries != 0)
                        {
                            --commonTertiaries;
                            while (commonTertiaries >= TER_UPPER_FIRST_COMMON_MAX_COUNT)
                            {
                                tertiaries.AppendByte(TER_UPPER_FIRST_COMMON_MIDDLE);
                                commonTertiaries -= TER_UPPER_FIRST_COMMON_MAX_COUNT;
                            }
                            int b;
                            if (t < (TER_UPPER_FIRST_COMMON_LOW << 8))
                            {
                                b = TER_UPPER_FIRST_COMMON_LOW + commonTertiaries;
                            }
                            else
                            {
                                b = TER_UPPER_FIRST_COMMON_HIGH - commonTertiaries;
                            }
                            tertiaries.AppendByte(b);
                            commonTertiaries = 0;
                        }
                        tertiaries.AppendWeight16(t);
                    }
                }

                if ((levels & Collation.QUATERNARY_LEVEL_FLAG) != 0)
                {
                    int q = lower32 & 0xffff;
                    if ((q & 0xc0) == 0 && q > Collation.NO_CE_WEIGHT16)
                    {
                        ++commonQuaternaries;
                    }
                    else if (q == Collation.NO_CE_WEIGHT16 &&
                             (options & CollationSettings.ALTERNATE_MASK) == 0 &&
                             quaternaries.IsEmpty)
                    {
                        // If alternate=non-ignorable and there are only common quaternary weights,
                        // then we need not write anything.
                        // The only weights greater than the merge separator and less than the common
                        // weight
                        // are shifted primary weights, which are not generated for
                        // alternate=non-ignorable.
                        // There are also exactly as many quaternary weights as tertiary weights,
                        // so level length differences are handled already on tertiary level.
                        // Any above-common quaternary weight will compare greater regardless.
                        quaternaries.AppendByte(Collation.LEVEL_SEPARATOR_BYTE);
                    }
                    else
                    {
                        if (q == Collation.NO_CE_WEIGHT16)
                        {
                            q = Collation.LEVEL_SEPARATOR_BYTE;
                        }
                        else
                        {
                            q = 0xfc + ((q.TripleShift(6)) & 3);
                        }
                        if (commonQuaternaries != 0)
                        {
                            --commonQuaternaries;
                            while (commonQuaternaries >= QUAT_COMMON_MAX_COUNT)
                            {
                                quaternaries.AppendByte(QUAT_COMMON_MIDDLE);
                                commonQuaternaries -= QUAT_COMMON_MAX_COUNT;
                            }
                            int b;
                            if (q < QUAT_COMMON_LOW)
                            {
                                b = QUAT_COMMON_LOW + commonQuaternaries;
                            }
                            else
                            {
                                b = QUAT_COMMON_HIGH - commonQuaternaries;
                            }
                            quaternaries.AppendByte(b);
                            commonQuaternaries = 0;
                        }
                        quaternaries.AppendByte(q);
                    }
                }

                if ((lower32.TripleShift(24)) == Collation.LEVEL_SEPARATOR_BYTE)
                {
                    break;
                } // ce == NO_CE
            }

            // Append the beyond-primary levels.
            // not used in Java -- boolean ok = true;
            if ((levels & Collation.SECONDARY_LEVEL_FLAG) != 0)
            {
                if (!callback.NeedToWrite(Collation.SECONDARY_LEVEL))
                {
                    return;
                }
                // not used in Java -- ok &= secondaries.isOk();
                sink.Append(Collation.LEVEL_SEPARATOR_BYTE);
                secondaries.AppendTo(sink);
            }

            if ((levels & Collation.CASE_LEVEL_FLAG) != 0)
            {
                if (!callback.NeedToWrite(Collation.CASE_LEVEL))
                {
                    return;
                }
                // not used in Java -- ok &= cases.isOk();
                sink.Append(Collation.LEVEL_SEPARATOR_BYTE);
                // Write pairs of nibbles as bytes, except separator bytes as themselves.
                int  length = cases.Length - 1; // Ignore the trailing NO_CE.
                byte b      = 0;
                for (int i = 0; i < length; ++i)
                {
                    byte c = cases.GetAt(i);
                    Debug.Assert((c & 0xf) == 0 && c != 0);
                    if (b == 0)
                    {
                        b = c;
                    }
                    else
                    {
                        sink.Append(b | ((c >> 4) & 0xf));
                        b = 0;
                    }
                }
                if (b != 0)
                {
                    sink.Append(b);
                }
            }

            if ((levels & Collation.TERTIARY_LEVEL_FLAG) != 0)
            {
                if (!callback.NeedToWrite(Collation.TERTIARY_LEVEL))
                {
                    return;
                }
                // not used in Java -- ok &= tertiaries.isOk();
                sink.Append(Collation.LEVEL_SEPARATOR_BYTE);
                tertiaries.AppendTo(sink);
            }

            if ((levels & Collation.QUATERNARY_LEVEL_FLAG) != 0)
            {
                if (!callback.NeedToWrite(Collation.QUATERNARY_LEVEL))
                {
                    return;
                }
                // not used in Java -- ok &= quaternaries.isOk();
                sink.Append(Collation.LEVEL_SEPARATOR_BYTE);
                quaternaries.AppendTo(sink);
            }

            // not used in Java -- if (!ok || !sink.IsOk()) {
            // Java porting note: U_MEMORY_ALLOCATION_ERROR is set here in
            // C implementation. IsOk() in Java always returns true, so this
            // is a dead code.
        }
Exemplo n.º 6
0
        public static int CompareUTF16(char[] table, char[] primaries, int options,
                                       ICharSequence left, ICharSequence right, int startIndex)
        {
            // This is a modified copy of CollationCompare.compareUpToQuaternary(),
            // optimized for common Latin text.
            // Keep them in sync!

            int variableTop = options >> 16; // see getOptions()

            options &= 0xffff;               // needed for CollationSettings.getStrength() to work

            // Check for supported characters, fetch mini CEs, and compare primaries.
            int leftIndex = startIndex, rightIndex = startIndex;

            // Single mini CE or a pair.
            // The current mini CE is in the lower 16 bits, the next one is in the upper 16 bits.
            // If there is only one, then it is in the lower bits, and the upper bits are 0.
            int leftPair = 0, rightPair = 0;

            for (; ;)
            {
                // We fetch CEs until we get a non-ignorable primary or reach the end.
                while (leftPair == 0)
                {
                    if (leftIndex == left.Length)
                    {
                        leftPair = EOS;
                        break;
                    }
                    int c = left[leftIndex++];
                    if (c <= LatinMax)
                    {
                        leftPair = primaries[c];
                        if (leftPair != 0)
                        {
                            break;
                        }
                        if (c <= 0x39 && c >= 0x30 && (options & CollationSettings.Numeric) != 0)
                        {
                            return(BailOutResult);
                        }
                        leftPair = table[c];
                    }
                    else if (PUNCT_START <= c && c < PUNCT_LIMIT)
                    {
                        leftPair = table[c - PUNCT_START + LatinLimit];
                    }
                    else
                    {
                        leftPair = Lookup(table, c);
                    }
                    if (leftPair >= MIN_SHORT)
                    {
                        leftPair &= SHORT_PRIMARY_MASK;
                        break;
                    }
                    else if (leftPair > variableTop)
                    {
                        leftPair &= LONG_PRIMARY_MASK;
                        break;
                    }
                    else
                    {
                        long pairAndInc = NextPair(table, c, leftPair, left, leftIndex);
                        if (pairAndInc < 0)
                        {
                            ++leftIndex;
                            pairAndInc = ~pairAndInc;
                        }
                        leftPair = (int)pairAndInc;
                        if (leftPair == BAIL_OUT)
                        {
                            return(BailOutResult);
                        }
                        leftPair = GetPrimaries(variableTop, leftPair);
                    }
                }

                while (rightPair == 0)
                {
                    if (rightIndex == right.Length)
                    {
                        rightPair = EOS;
                        break;
                    }
                    int c = right[rightIndex++];
                    if (c <= LatinMax)
                    {
                        rightPair = primaries[c];
                        if (rightPair != 0)
                        {
                            break;
                        }
                        if (c <= 0x39 && c >= 0x30 && (options & CollationSettings.Numeric) != 0)
                        {
                            return(BailOutResult);
                        }
                        rightPair = table[c];
                    }
                    else if (PUNCT_START <= c && c < PUNCT_LIMIT)
                    {
                        rightPair = table[c - PUNCT_START + LatinLimit];
                    }
                    else
                    {
                        rightPair = Lookup(table, c);
                    }
                    if (rightPair >= MIN_SHORT)
                    {
                        rightPair &= SHORT_PRIMARY_MASK;
                        break;
                    }
                    else if (rightPair > variableTop)
                    {
                        rightPair &= LONG_PRIMARY_MASK;
                        break;
                    }
                    else
                    {
                        long pairAndInc = NextPair(table, c, rightPair, right, rightIndex);
                        if (pairAndInc < 0)
                        {
                            ++rightIndex;
                            pairAndInc = ~pairAndInc;
                        }
                        rightPair = (int)pairAndInc;
                        if (rightPair == BAIL_OUT)
                        {
                            return(BailOutResult);
                        }
                        rightPair = GetPrimaries(variableTop, rightPair);
                    }
                }

                if (leftPair == rightPair)
                {
                    if (leftPair == EOS)
                    {
                        break;
                    }
                    leftPair = rightPair = 0;
                    continue;
                }
                int leftPrimary  = leftPair & 0xffff;
                int rightPrimary = rightPair & 0xffff;
                if (leftPrimary != rightPrimary)
                {
                    // Return the primary difference.
                    return((leftPrimary < rightPrimary) ? Collation.Less : Collation.Greater);
                }
                if (leftPair == EOS)
                {
                    break;
                }
                //leftPair >>>= 16;
                //rightPair >>>= 16;
                leftPair  = leftPair.TripleShift(16);
                rightPair = rightPair.TripleShift(16);
            }
            // In the following, we need to re-fetch each character because we did not buffer the CEs,
            // but we know that the string is well-formed and
            // only contains supported characters and mappings.

            // We might skip the secondary level but continue with the case level
            // which is turned on separately.
            if (CollationSettings.GetStrength(options) >= CollationStrength.Secondary)
            {
                leftIndex = rightIndex = startIndex;
                leftPair  = rightPair = 0;
                for (; ;)
                {
                    while (leftPair == 0)
                    {
                        if (leftIndex == left.Length)
                        {
                            leftPair = EOS;
                            break;
                        }
                        int c = left[leftIndex++];
                        if (c <= LatinMax)
                        {
                            leftPair = table[c];
                        }
                        else if (PUNCT_START <= c && c < PUNCT_LIMIT)
                        {
                            leftPair = table[c - PUNCT_START + LatinLimit];
                        }
                        else
                        {
                            leftPair = Lookup(table, c);
                        }
                        if (leftPair >= MIN_SHORT)
                        {
                            leftPair = GetSecondariesFromOneShortCE(leftPair);
                            break;
                        }
                        else if (leftPair > variableTop)
                        {
                            leftPair = COMMON_SEC_PLUS_OFFSET;
                            break;
                        }
                        else
                        {
                            long pairAndInc = NextPair(table, c, leftPair, left, leftIndex);
                            if (pairAndInc < 0)
                            {
                                ++leftIndex;
                                pairAndInc = ~pairAndInc;
                            }
                            leftPair = GetSecondaries(variableTop, (int)pairAndInc);
                        }
                    }

                    while (rightPair == 0)
                    {
                        if (rightIndex == right.Length)
                        {
                            rightPair = EOS;
                            break;
                        }
                        int c = right[rightIndex++];
                        if (c <= LatinMax)
                        {
                            rightPair = table[c];
                        }
                        else if (PUNCT_START <= c && c < PUNCT_LIMIT)
                        {
                            rightPair = table[c - PUNCT_START + LatinLimit];
                        }
                        else
                        {
                            rightPair = Lookup(table, c);
                        }
                        if (rightPair >= MIN_SHORT)
                        {
                            rightPair = GetSecondariesFromOneShortCE(rightPair);
                            break;
                        }
                        else if (rightPair > variableTop)
                        {
                            rightPair = COMMON_SEC_PLUS_OFFSET;
                            break;
                        }
                        else
                        {
                            long pairAndInc = NextPair(table, c, rightPair, right, rightIndex);
                            if (pairAndInc < 0)
                            {
                                ++rightIndex;
                                pairAndInc = ~pairAndInc;
                            }
                            rightPair = GetSecondaries(variableTop, (int)pairAndInc);
                        }
                    }

                    if (leftPair == rightPair)
                    {
                        if (leftPair == EOS)
                        {
                            break;
                        }
                        leftPair = rightPair = 0;
                        continue;
                    }
                    int leftSecondary  = leftPair & 0xffff;
                    int rightSecondary = rightPair & 0xffff;
                    if (leftSecondary != rightSecondary)
                    {
                        if ((options & CollationSettings.BackwardSecondary) != 0)
                        {
                            // Full support for backwards secondary requires backwards contraction matching
                            // and moving backwards between merge separators.
                            return(BailOutResult);
                        }
                        return((leftSecondary < rightSecondary) ? Collation.Less : Collation.Greater);
                    }
                    if (leftPair == EOS)
                    {
                        break;
                    }
                    //leftPair >>>= 16;
                    //rightPair >>>= 16;
                    leftPair  = leftPair.TripleShift(16);
                    rightPair = rightPair.TripleShift(16);
                }
            }

            if ((options & CollationSettings.CaseLevel) != 0)
            {
                bool strengthIsPrimary = CollationSettings.GetStrength(options) == CollationStrength.Primary;
                leftIndex = rightIndex = startIndex;
                leftPair  = rightPair = 0;
                for (; ;)
                {
                    while (leftPair == 0)
                    {
                        if (leftIndex == left.Length)
                        {
                            leftPair = EOS;
                            break;
                        }
                        int c = left[leftIndex++];
                        leftPair = (c <= LatinMax) ? table[c] : Lookup(table, c);
                        if (leftPair < MIN_LONG)
                        {
                            long pairAndInc = NextPair(table, c, leftPair, left, leftIndex);
                            if (pairAndInc < 0)
                            {
                                ++leftIndex;
                                pairAndInc = ~pairAndInc;
                            }
                            leftPair = (int)pairAndInc;
                        }
                        leftPair = GetCases(variableTop, strengthIsPrimary, leftPair);
                    }

                    while (rightPair == 0)
                    {
                        if (rightIndex == right.Length)
                        {
                            rightPair = EOS;
                            break;
                        }
                        int c = right[rightIndex++];
                        rightPair = (c <= LatinMax) ? table[c] : Lookup(table, c);
                        if (rightPair < MIN_LONG)
                        {
                            long pairAndInc = NextPair(table, c, rightPair, right, rightIndex);
                            if (pairAndInc < 0)
                            {
                                ++rightIndex;
                                pairAndInc = ~pairAndInc;
                            }
                            rightPair = (int)pairAndInc;
                        }
                        rightPair = GetCases(variableTop, strengthIsPrimary, rightPair);
                    }

                    if (leftPair == rightPair)
                    {
                        if (leftPair == EOS)
                        {
                            break;
                        }
                        leftPair = rightPair = 0;
                        continue;
                    }
                    int leftCase  = leftPair & 0xffff;
                    int rightCase = rightPair & 0xffff;
                    if (leftCase != rightCase)
                    {
                        if ((options & CollationSettings.UpperFirst) == 0)
                        {
                            return((leftCase < rightCase) ? Collation.Less : Collation.Greater);
                        }
                        else
                        {
                            return((leftCase < rightCase) ? Collation.Greater : Collation.Less);
                        }
                    }
                    if (leftPair == EOS)
                    {
                        break;
                    }
                    //leftPair >>>= 16;
                    //rightPair >>>= 16;
                    leftPair  = leftPair.TripleShift(16);
                    rightPair = rightPair.TripleShift(16);
                }
            }
            if (CollationSettings.GetStrength(options) <= CollationStrength.Secondary)
            {
                return(Collation.Equal);
            }

            // Remove the case bits from the tertiary weight when caseLevel is on or caseFirst is off.
            bool withCaseBits = CollationSettings.IsTertiaryWithCaseBits(options);

            leftIndex = rightIndex = startIndex;
            leftPair  = rightPair = 0;
            for (; ;)
            {
                while (leftPair == 0)
                {
                    if (leftIndex == left.Length)
                    {
                        leftPair = EOS;
                        break;
                    }
                    int c = left[leftIndex++];
                    leftPair = (c <= LatinMax) ? table[c] : Lookup(table, c);
                    if (leftPair < MIN_LONG)
                    {
                        long pairAndInc = NextPair(table, c, leftPair, left, leftIndex);
                        if (pairAndInc < 0)
                        {
                            ++leftIndex;
                            pairAndInc = ~pairAndInc;
                        }
                        leftPair = (int)pairAndInc;
                    }
                    leftPair = GetTertiaries(variableTop, withCaseBits, leftPair);
                }

                while (rightPair == 0)
                {
                    if (rightIndex == right.Length)
                    {
                        rightPair = EOS;
                        break;
                    }
                    int c = right[rightIndex++];
                    rightPair = (c <= LatinMax) ? table[c] : Lookup(table, c);
                    if (rightPair < MIN_LONG)
                    {
                        long pairAndInc = NextPair(table, c, rightPair, right, rightIndex);
                        if (pairAndInc < 0)
                        {
                            ++rightIndex;
                            pairAndInc = ~pairAndInc;
                        }
                        rightPair = (int)pairAndInc;
                    }
                    rightPair = GetTertiaries(variableTop, withCaseBits, rightPair);
                }

                if (leftPair == rightPair)
                {
                    if (leftPair == EOS)
                    {
                        break;
                    }
                    leftPair = rightPair = 0;
                    continue;
                }
                int leftTertiary  = leftPair & 0xffff;
                int rightTertiary = rightPair & 0xffff;
                if (leftTertiary != rightTertiary)
                {
                    if (CollationSettings.SortsTertiaryUpperCaseFirst(options))
                    {
                        // Pass through EOS and MERGE_WEIGHT
                        // and keep real tertiary weights larger than the MERGE_WEIGHT.
                        // Tertiary CEs (secondary ignorables) are not supported in fast Latin.
                        if (leftTertiary > MERGE_WEIGHT)
                        {
                            leftTertiary ^= CASE_MASK;
                        }
                        if (rightTertiary > MERGE_WEIGHT)
                        {
                            rightTertiary ^= CASE_MASK;
                        }
                    }
                    return((leftTertiary < rightTertiary) ? Collation.Less : Collation.Greater);
                }
                if (leftPair == EOS)
                {
                    break;
                }
                //leftPair >>>= 16;
                //rightPair >>>= 16;
                leftPair  = leftPair.TripleShift(16);
                rightPair = rightPair.TripleShift(16);
            }
            if (CollationSettings.GetStrength(options) <= CollationStrength.Tertiary)
            {
                return(Collation.Equal);
            }

            leftIndex = rightIndex = startIndex;
            leftPair  = rightPair = 0;
            for (; ;)
            {
                while (leftPair == 0)
                {
                    if (leftIndex == left.Length)
                    {
                        leftPair = EOS;
                        break;
                    }
                    int c = left[leftIndex++];
                    leftPair = (c <= LatinMax) ? table[c] : Lookup(table, c);
                    if (leftPair < MIN_LONG)
                    {
                        long pairAndInc = NextPair(table, c, leftPair, left, leftIndex);
                        if (pairAndInc < 0)
                        {
                            ++leftIndex;
                            pairAndInc = ~pairAndInc;
                        }
                        leftPair = (int)pairAndInc;
                    }
                    leftPair = GetQuaternaries(variableTop, leftPair);
                }

                while (rightPair == 0)
                {
                    if (rightIndex == right.Length)
                    {
                        rightPair = EOS;
                        break;
                    }
                    int c = right[rightIndex++];
                    rightPair = (c <= LatinMax) ? table[c] : Lookup(table, c);
                    if (rightPair < MIN_LONG)
                    {
                        long pairAndInc = NextPair(table, c, rightPair, right, rightIndex);
                        if (pairAndInc < 0)
                        {
                            ++rightIndex;
                            pairAndInc = ~pairAndInc;
                        }
                        rightPair = (int)pairAndInc;
                    }
                    rightPair = GetQuaternaries(variableTop, rightPair);
                }

                if (leftPair == rightPair)
                {
                    if (leftPair == EOS)
                    {
                        break;
                    }
                    leftPair = rightPair = 0;
                    continue;
                }
                int leftQuaternary  = leftPair & 0xffff;
                int rightQuaternary = rightPair & 0xffff;
                if (leftQuaternary != rightQuaternary)
                {
                    return((leftQuaternary < rightQuaternary) ? Collation.Less : Collation.Greater);
                }
                if (leftPair == EOS)
                {
                    break;
                }
                //leftPair >>>= 16;
                //rightPair >>>= 16;
                leftPair  = leftPair.TripleShift(16);
                rightPair = rightPair.TripleShift(16);
            }
            return(Collation.Equal);
        }
Exemplo n.º 7
0
        /// <summary>
        /// Computes the options value for the compare functions
        /// and writes the precomputed primary weights.
        /// Returns -1 if the Latin fastpath is not supported for the data and settings.
        /// The capacity must be <see cref="LatinLimit"/>.
        /// </summary>
        public static int GetOptions(CollationData data, CollationSettings settings,
                                     char[] primaries)
        {
            char[] header = data.fastLatinTableHeader;
            if (header == null)
            {
                return(-1);
            }
            Debug.Assert((header[0] >> 8) == Version);
            if (primaries.Length != LatinLimit)
            {
                Debug.Assert(false);
                return(-1);
            }

            int miniVarTop;

            if ((settings.Options & CollationSettings.AlternateMask) == 0)
            {
                // No mini primaries are variable, set a variableTop just below the
                // lowest long mini primary.
                miniVarTop = MIN_LONG - 1;
            }
            else
            {
                int headerLength = header[0] & 0xff;
                int i            = 1 + settings.MaxVariable;
                if (i >= headerLength)
                {
                    return(-1);  // variableTop >= digits, should not occur
                }
                miniVarTop = header[i];
            }

            bool digitsAreReordered = false;

            if (settings.HasReordering)
            {
                long prevStart        = 0;
                long beforeDigitStart = 0;
                long digitStart       = 0;
                long afterDigitStart  = 0;
                for (int group = ReorderCodes.First;
                     group < ReorderCodes.First + CollationData.MAX_NUM_SPECIAL_REORDER_CODES;
                     ++group)
                {
                    long start = data.GetFirstPrimaryForGroup(group);
                    start = settings.Reorder(start);
                    if (group == ReorderCodes.Digit)
                    {
                        beforeDigitStart = prevStart;
                        digitStart       = start;
                    }
                    else if (start != 0)
                    {
                        if (start < prevStart)
                        {
                            // The permutation affects the groups up to Latin.
                            return(-1);
                        }
                        // In the future, there might be a special group between digits & Latin.
                        if (digitStart != 0 && afterDigitStart == 0 && prevStart == beforeDigitStart)
                        {
                            afterDigitStart = start;
                        }
                        prevStart = start;
                    }
                }
                long latinStart = data.GetFirstPrimaryForGroup(UScript.Latin);
                latinStart = settings.Reorder(latinStart);
                if (latinStart < prevStart)
                {
                    return(-1);
                }
                if (afterDigitStart == 0)
                {
                    afterDigitStart = latinStart;
                }
                if (!(beforeDigitStart < digitStart && digitStart < afterDigitStart))
                {
                    digitsAreReordered = true;
                }
            }

            char[] table = data.FastLatinTable;  // skip the header
            for (int c = 0; c < LatinLimit; ++c)
            {
                int p = table[c];
                if (p >= MIN_SHORT)
                {
                    p &= SHORT_PRIMARY_MASK;
                }
                else if (p > miniVarTop)
                {
                    p &= LONG_PRIMARY_MASK;
                }
                else
                {
                    p = 0;
                }
                primaries[c] = (char)p;
            }
            if (digitsAreReordered || (settings.Options & CollationSettings.Numeric) != 0)
            {
                // Bail out for digits.
                for (int c = 0x30; c <= 0x39; ++c)
                {
                    primaries[c] = (char)0;
                }
            }

            // Shift the miniVarTop above other options.
            return((miniVarTop << 16) | settings.Options);
        }
Exemplo n.º 8
0
        public static int CompareUpToQuaternary(CollationIterator left, CollationIterator right,
                                                CollationSettings settings)
        {
            int  options = settings.Options;
            long variableTop;

            if ((options & CollationSettings.ALTERNATE_MASK) == 0)
            {
                variableTop = 0;
            }
            else
            {
                // +1 so that we can use "<" and primary ignorables test out early.
                variableTop = settings.VariableTop + 1;
            }
            bool anyVariable = false;

            // Fetch CEs, compare primaries, store secondary & tertiary weights.
            for (; ;)
            {
                // We fetch CEs until we get a non-ignorable primary or reach the end.
                long leftPrimary;
                do
                {
                    long ce = left.NextCE();
                    leftPrimary = ce.TripleShift(32);
                    if (leftPrimary < variableTop && leftPrimary > Collation.MERGE_SEPARATOR_PRIMARY)
                    {
                        // Variable CE, shift it to quaternary level.
                        // Ignore all following primary ignorables, and shift further variable CEs.
                        anyVariable = true;
                        do
                        {
                            // Store only the primary of the variable CE.
                            left.SetCurrentCE(ce & unchecked ((long)0xffffffff00000000L));
                            for (; ;)
                            {
                                ce          = left.NextCE();
                                leftPrimary = ce.TripleShift(32);
                                if (leftPrimary == 0)
                                {
                                    left.SetCurrentCE(0);
                                }
                                else
                                {
                                    break;
                                }
                            }
                        } while (leftPrimary < variableTop && leftPrimary > Collation.MERGE_SEPARATOR_PRIMARY);
                    }
                } while (leftPrimary == 0);

                long rightPrimary;
                do
                {
                    long ce = right.NextCE();
                    rightPrimary = ce.TripleShift(32);
                    if (rightPrimary < variableTop && rightPrimary > Collation.MERGE_SEPARATOR_PRIMARY)
                    {
                        // Variable CE, shift it to quaternary level.
                        // Ignore all following primary ignorables, and shift further variable CEs.
                        anyVariable = true;
                        do
                        {
                            // Store only the primary of the variable CE.
                            right.SetCurrentCE(ce & unchecked ((long)0xffffffff00000000L));
                            for (; ;)
                            {
                                ce           = right.NextCE();
                                rightPrimary = ce.TripleShift(32);
                                if (rightPrimary == 0)
                                {
                                    right.SetCurrentCE(0);
                                }
                                else
                                {
                                    break;
                                }
                            }
                        } while (rightPrimary < variableTop && rightPrimary > Collation.MERGE_SEPARATOR_PRIMARY);
                    }
                } while (rightPrimary == 0);

                if (leftPrimary != rightPrimary)
                {
                    // Return the primary difference, with script reordering.
                    if (settings.HasReordering)
                    {
                        leftPrimary  = settings.Reorder(leftPrimary);
                        rightPrimary = settings.Reorder(rightPrimary);
                    }
                    return((leftPrimary < rightPrimary) ? Collation.LESS : Collation.GREATER);
                }
                if (leftPrimary == Collation.NO_CE_PRIMARY)
                {
                    break;
                }
            }

            // Compare the buffered secondary & tertiary weights.
            // We might skip the secondary level but continue with the case level
            // which is turned on separately.
            if (CollationSettings.GetStrength(options) >= CollationStrength.Secondary)
            {
                if ((options & CollationSettings.BACKWARD_SECONDARY) == 0)
                {
                    int leftIndex2  = 0;
                    int rightIndex2 = 0;
                    for (; ;)
                    {
                        int leftSecondary;
                        do
                        {
                            leftSecondary = ((int)left.GetCE(leftIndex2++)).TripleShift(16);
                        } while (leftSecondary == 0);

                        int rightSecondary;
                        do
                        {
                            rightSecondary = ((int)right.GetCE(rightIndex2++)).TripleShift(16);
                        } while (rightSecondary == 0);

                        if (leftSecondary != rightSecondary)
                        {
                            return((leftSecondary < rightSecondary) ? Collation.LESS : Collation.GREATER);
                        }
                        if (leftSecondary == Collation.NO_CE_WEIGHT16)
                        {
                            break;
                        }
                    }
                }
                else
                {
                    // The backwards secondary level compares secondary weights backwards
                    // within segments separated by the merge separator (U+FFFE, weight 02).
                    int leftStart  = 0;
                    int rightStart = 0;
                    for (; ;)
                    {
                        // Find the merge separator or the NO_CE terminator.
                        long p;
                        int  leftLimit = leftStart;
                        while ((p = left.GetCE(leftLimit).TripleShift(32)) > Collation.MERGE_SEPARATOR_PRIMARY ||
                               p == 0)
                        {
                            ++leftLimit;
                        }
                        int rightLimit = rightStart;
                        while ((p = right.GetCE(rightLimit).TripleShift(32)) > Collation.MERGE_SEPARATOR_PRIMARY ||
                               p == 0)
                        {
                            ++rightLimit;
                        }

                        // Compare the segments.
                        int leftIndex3  = leftLimit;
                        int rightIndex3 = rightLimit;
                        for (; ;)
                        {
                            int leftSecondary = 0;
                            while (leftSecondary == 0 && leftIndex3 > leftStart)
                            {
                                leftSecondary = ((int)left.GetCE(--leftIndex3)).TripleShift(16);
                            }

                            int rightSecondary = 0;
                            while (rightSecondary == 0 && rightIndex3 > rightStart)
                            {
                                rightSecondary = ((int)right.GetCE(--rightIndex3)).TripleShift(16);
                            }

                            if (leftSecondary != rightSecondary)
                            {
                                return((leftSecondary < rightSecondary) ? Collation.LESS : Collation.GREATER);
                            }
                            if (leftSecondary == 0)
                            {
                                break;
                            }
                        }

                        // Did we reach the end of either string?
                        // Both strings have the same number of merge separators,
                        // or else there would have been a primary-level difference.
                        Debug.Assert(left.GetCE(leftLimit) == right.GetCE(rightLimit));
                        if (p == Collation.NO_CE_PRIMARY)
                        {
                            break;
                        }
                        // Skip both merge separators and continue.
                        leftStart  = leftLimit + 1;
                        rightStart = rightLimit + 1;
                    }
                }
            }

            if ((options & CollationSettings.CASE_LEVEL) != 0)
            {
                CollationStrength strength = CollationSettings.GetStrength(options);
                int leftIndex4             = 0;
                int rightIndex4            = 0;
                for (; ;)
                {
                    int leftCase, leftLower32, rightCase;
                    if (strength == CollationStrength.Primary)
                    {
                        // Primary+caseLevel: Ignore case level weights of primary ignorables.
                        // Otherwise we would get a-umlaut > a
                        // which is not desirable for accent-insensitive sorting.
                        // Check for (lower 32 bits) == 0 as well because variable CEs are stored
                        // with only primary weights.
                        long ce;
                        do
                        {
                            ce       = left.GetCE(leftIndex4++);
                            leftCase = (int)ce;
                        } while ((ce.TripleShift(32)) == 0 || leftCase == 0);
                        leftLower32 = leftCase;
                        leftCase   &= 0xc000;

                        do
                        {
                            ce        = right.GetCE(rightIndex4++);
                            rightCase = (int)ce;
                        } while ((ce.TripleShift(32)) == 0 || rightCase == 0);
                        rightCase &= 0xc000;
                    }
                    else
                    {
                        // Secondary+caseLevel: By analogy with the above,
                        // ignore case level weights of secondary ignorables.
                        //
                        // Note: A tertiary CE has uppercase case bits (0.0.ut)
                        // to keep tertiary+caseFirst well-formed.
                        //
                        // Tertiary+caseLevel: Also ignore case level weights of secondary ignorables.
                        // Otherwise a tertiary CE's uppercase would be no greater than
                        // a primary/secondary CE's uppercase.
                        // (See UCA well-formedness condition 2.)
                        // We could construct a special case weight higher than uppercase,
                        // but it's simpler to always ignore case weights of secondary ignorables,
                        // turning 0.0.ut into 0.0.0.t.
                        // (See LDML Collation, Case Parameters.)
                        do
                        {
                            leftCase = (int)left.GetCE(leftIndex4++);
                        } while ((leftCase & 0xffff0000) == 0);
                        leftLower32 = leftCase;
                        leftCase   &= 0xc000;

                        do
                        {
                            rightCase = (int)right.GetCE(rightIndex4++);
                        } while ((rightCase & 0xffff0000) == 0);
                        rightCase &= 0xc000;
                    }

                    // No need to handle NO_CE and MERGE_SEPARATOR specially:
                    // There is one case weight for each previous-level weight,
                    // so level length differences were handled there.
                    if (leftCase != rightCase)
                    {
                        if ((options & CollationSettings.UPPER_FIRST) == 0)
                        {
                            return((leftCase < rightCase) ? Collation.LESS : Collation.GREATER);
                        }
                        else
                        {
                            return((leftCase < rightCase) ? Collation.GREATER : Collation.LESS);
                        }
                    }
                    if ((leftLower32.TripleShift(16)) == Collation.NO_CE_WEIGHT16)
                    {
                        break;
                    }
                }
            }
            if (CollationSettings.GetStrength(options) <= CollationStrength.Secondary)
            {
                return(Collation.EQUAL);
            }

            int tertiaryMask = CollationSettings.GetTertiaryMask(options);

            int leftIndex       = 0;
            int rightIndex      = 0;
            int anyQuaternaries = 0;

            for (; ;)
            {
                int leftLower32, leftTertiary;
                do
                {
                    leftLower32      = (int)left.GetCE(leftIndex++);
                    anyQuaternaries |= leftLower32;
                    Debug.Assert((leftLower32 & Collation.ONLY_TERTIARY_MASK) != 0 || (leftLower32 & 0xc0c0) == 0);
                    leftTertiary = leftLower32 & tertiaryMask;
                } while (leftTertiary == 0);

                int rightLower32, rightTertiary;
                do
                {
                    rightLower32     = (int)right.GetCE(rightIndex++);
                    anyQuaternaries |= rightLower32;
                    Debug.Assert((rightLower32 & Collation.ONLY_TERTIARY_MASK) != 0 || (rightLower32 & 0xc0c0) == 0);
                    rightTertiary = rightLower32 & tertiaryMask;
                } while (rightTertiary == 0);

                if (leftTertiary != rightTertiary)
                {
                    if (CollationSettings.SortsTertiaryUpperCaseFirst(options))
                    {
                        // Pass through NO_CE and keep real tertiary weights larger than that.
                        // Do not change the artificial uppercase weight of a tertiary CE (0.0.ut),
                        // to keep tertiary CEs well-formed.
                        // Their case+tertiary weights must be greater than those of
                        // primary and secondary CEs.
                        if (leftTertiary > Collation.NO_CE_WEIGHT16)
                        {
                            if ((leftLower32 & 0xffff0000) != 0)
                            {
                                leftTertiary ^= 0xc000;
                            }
                            else
                            {
                                leftTertiary += 0x4000;
                            }
                        }
                        if (rightTertiary > Collation.NO_CE_WEIGHT16)
                        {
                            if ((rightLower32 & 0xffff0000) != 0)
                            {
                                rightTertiary ^= 0xc000;
                            }
                            else
                            {
                                rightTertiary += 0x4000;
                            }
                        }
                    }
                    return((leftTertiary < rightTertiary) ? Collation.LESS : Collation.GREATER);
                }
                if (leftTertiary == Collation.NO_CE_WEIGHT16)
                {
                    break;
                }
            }
            if (CollationSettings.GetStrength(options) <= CollationStrength.Tertiary)
            {
                return(Collation.EQUAL);
            }

            if (!anyVariable && (anyQuaternaries & 0xc0) == 0)
            {
                // If there are no "variable" CEs and no non-zero quaternary weights,
                // then there are no quaternary differences.
                return(Collation.EQUAL);
            }

            leftIndex  = 0;
            rightIndex = 0;
            for (; ;)
            {
                long leftQuaternary;
                do
                {
                    long ce = left.GetCE(leftIndex++);
                    leftQuaternary = ce & 0xffff;
                    if (leftQuaternary <= Collation.NO_CE_WEIGHT16)
                    {
                        // Variable primary or completely ignorable or NO_CE.
                        leftQuaternary = ce.TripleShift(32);
                    }
                    else
                    {
                        // Regular CE, not tertiary ignorable.
                        // Preserve the quaternary weight in bits 7..6.
                        leftQuaternary |= 0xffffff3fL;
                    }
                } while (leftQuaternary == 0);

                long rightQuaternary;
                do
                {
                    long ce = right.GetCE(rightIndex++);
                    rightQuaternary = ce & 0xffff;
                    if (rightQuaternary <= Collation.NO_CE_WEIGHT16)
                    {
                        // Variable primary or completely ignorable or NO_CE.
                        rightQuaternary = ce.TripleShift(32);
                    }
                    else
                    {
                        // Regular CE, not tertiary ignorable.
                        // Preserve the quaternary weight in bits 7..6.
                        rightQuaternary |= 0xffffff3fL;
                    }
                } while (rightQuaternary == 0);

                if (leftQuaternary != rightQuaternary)
                {
                    // Return the difference, with script reordering.
                    if (settings.HasReordering)
                    {
                        leftQuaternary  = settings.Reorder(leftQuaternary);
                        rightQuaternary = settings.Reorder(rightQuaternary);
                    }
                    return((leftQuaternary < rightQuaternary) ? Collation.LESS : Collation.GREATER);
                }
                if (leftQuaternary == Collation.NO_CE_PRIMARY)
                {
                    break;
                }
            }
            return(Collation.EQUAL);
        }
Exemplo n.º 9
0
 internal void Parse(string ruleString, CollationSettings outSettings)
 {
     settings = outSettings;
     Parse(ruleString);
 }
Exemplo n.º 10
0
        internal static void Read(CollationTailoring @base, ByteBuffer inBytes,
                                  CollationTailoring tailoring)
        {
            tailoring.Version = ICUBinary.ReadHeader(inBytes, DATA_FORMAT, IS_ACCEPTABLE);
            if (@base != null && @base.GetUCAVersion() != tailoring.GetUCAVersion())
            {
                throw new ICUException("Tailoring UCA version differs from base data UCA version");
            }

            int inLength = inBytes.Remaining;

            if (inLength < 8)
            {
                throw new ICUException("not enough bytes");
            }
            int indexesLength = inBytes.GetInt32();  // inIndexes[IX_INDEXES_LENGTH]

            if (indexesLength < 2 || inLength < indexesLength * 4)
            {
                throw new ICUException("not enough indexes");
            }
            int[] inIndexes = new int[IX_TOTAL_SIZE + 1];
            inIndexes[0] = indexesLength;
            for (int i = 1; i < indexesLength && i < inIndexes.Length; ++i)
            {
                inIndexes[i] = inBytes.GetInt32();
            }
            for (int i = indexesLength; i < inIndexes.Length; ++i)
            {
                inIndexes[i] = -1;
            }
            if (indexesLength > inIndexes.Length)
            {
                ICUBinary.SkipBytes(inBytes, (indexesLength - inIndexes.Length) * 4);
            }

            // Assume that the tailoring data is in initial state,
            // with null pointers and 0 lengths.

            // Set pointers to non-empty data parts.
            // Do this in order of their byte offsets. (Should help porting to Java.)

            int index;  // one of the indexes[] slots
            int offset; // byte offset for the index part
            int length; // number of bytes in the index part

            if (indexesLength > IX_TOTAL_SIZE)
            {
                length = inIndexes[IX_TOTAL_SIZE];
            }
            else if (indexesLength > IX_REORDER_CODES_OFFSET)
            {
                length = inIndexes[indexesLength - 1];
            }
            else
            {
                length = 0;  // only indexes, and inLength was already checked for them
            }
            if (inLength < length)
            {
                throw new ICUException("not enough bytes");
            }

            CollationData baseData = @base == null ? null : @base.Data;

            int[] reorderCodes;
            int   reorderCodesLength;

            index  = IX_REORDER_CODES_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 4)
            {
                if (baseData == null)
                {
                    // We assume for collation settings that
                    // the base data does not have a reordering.
                    throw new ICUException("Collation base data must not reorder scripts");
                }
                reorderCodesLength = length / 4;
                reorderCodes       = ICUBinary.GetInts(inBytes, reorderCodesLength, length & 3);

                // The reorderRanges (if any) are the trailing reorderCodes entries.
                // Split the array at the boundary.
                // Script or reorder codes do not exceed 16-bit values.
                // Range limits are stored in the upper 16 bits, and are never 0.
                int reorderRangesLength = 0;
                while (reorderRangesLength < reorderCodesLength &&
                       (reorderCodes[reorderCodesLength - reorderRangesLength - 1] & 0xffff0000) != 0)
                {
                    ++reorderRangesLength;
                }
                Debug.Assert(reorderRangesLength < reorderCodesLength);
                reorderCodesLength -= reorderRangesLength;
            }
            else
            {
                reorderCodes       = new int[0];
                reorderCodesLength = 0;
                ICUBinary.SkipBytes(inBytes, length);
            }

            // There should be a reorder table only if there are reorder codes.
            // However, when there are reorder codes the reorder table may be omitted to reduce
            // the data size.
            byte[] reorderTable = null;
            index  = IX_REORDER_TABLE_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 256)
            {
                if (reorderCodesLength == 0)
                {
                    throw new ICUException("Reordering table without reordering codes");
                }
                reorderTable = new byte[256];
                inBytes.Get(reorderTable);
                length -= 256;
            }
            else
            {
                // If we have reorder codes, then build the reorderTable at the end,
                // when the CollationData is otherwise complete.
            }
            ICUBinary.SkipBytes(inBytes, length);

            if (baseData != null && baseData.numericPrimary != (inIndexes[IX_OPTIONS] & 0xff000000L))
            {
                throw new ICUException("Tailoring numeric primary weight differs from base data");
            }
            CollationData data = null;  // Remains null if there are no mappings.

            index  = IX_TRIE_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 8)
            {
                tailoring.EnsureOwnedData();
                data                = tailoring.OwnedData;
                data.Base           = baseData;
                data.numericPrimary = inIndexes[IX_OPTIONS] & 0xff000000L;
                data.trie           = tailoring.Trie = Trie2_32.CreateFromSerialized(inBytes);
                int trieLength = data.trie.GetSerializedLength();
                if (trieLength > length)
                {
                    throw new ICUException("Not enough bytes for the mappings trie");  // No mappings.
                }
                length -= trieLength;
            }
            else if (baseData != null)
            {
                // Use the base data. Only the settings are tailored.
                tailoring.Data = baseData;
            }
            else
            {
                throw new ICUException("Missing collation data mappings");  // No mappings.
            }
            ICUBinary.SkipBytes(inBytes, length);

            index  = IX_RESERVED8_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            ICUBinary.SkipBytes(inBytes, length);

            index  = IX_CES_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 8)
            {
                if (data == null)
                {
                    throw new ICUException("Tailored ces without tailored trie");
                }
                data.ces = ICUBinary.GetLongs(inBytes, length / 8, length & 7);
            }
            else
            {
                ICUBinary.SkipBytes(inBytes, length);
            }

            index  = IX_RESERVED10_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            ICUBinary.SkipBytes(inBytes, length);

            index  = IX_CE32S_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 4)
            {
                if (data == null)
                {
                    throw new ICUException("Tailored ce32s without tailored trie");
                }
                data.ce32s = ICUBinary.GetInts(inBytes, length / 4, length & 3);
            }
            else
            {
                ICUBinary.SkipBytes(inBytes, length);
            }

            int jamoCE32sStart = inIndexes[IX_JAMO_CE32S_START];

            if (jamoCE32sStart >= 0)
            {
                if (data == null || data.ce32s == null)
                {
                    throw new ICUException("JamoCE32sStart index into non-existent ce32s[]");
                }
                data.jamoCE32s = new int[CollationData.JAMO_CE32S_LENGTH];
                // ICU4N specific - added extension method to IList<T> to handle "copy to"
                data.ce32s.CopyTo(jamoCE32sStart, data.jamoCE32s, 0, CollationData.JAMO_CE32S_LENGTH);
            }
            else if (data == null)
            {
                // Nothing to do.
            }
            else if (baseData != null)
            {
                data.jamoCE32s = baseData.jamoCE32s;
            }
            else
            {
                throw new ICUException("Missing Jamo CE32s for Hangul processing");
            }

            index  = IX_ROOT_ELEMENTS_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 4)
            {
                int rootElementsLength = length / 4;
                if (data == null)
                {
                    throw new ICUException("Root elements but no mappings");
                }
                if (rootElementsLength <= CollationRootElements.IX_SEC_TER_BOUNDARIES)
                {
                    throw new ICUException("Root elements array too short");
                }
                data.rootElements = new long[rootElementsLength];
                for (int i = 0; i < rootElementsLength; ++i)
                {
                    data.rootElements[i] = inBytes.GetInt32() & 0xffffffffL;  // unsigned int -> long
                }
                long commonSecTer = data.rootElements[CollationRootElements.IX_COMMON_SEC_AND_TER_CE];
                if (commonSecTer != Collation.COMMON_SEC_AND_TER_CE)
                {
                    throw new ICUException("Common sec/ter weights in base data differ from the hardcoded value");
                }
                long secTerBoundaries = data.rootElements[CollationRootElements.IX_SEC_TER_BOUNDARIES];
                if ((secTerBoundaries.TripleShift(24)) < CollationKeys.SEC_COMMON_HIGH)
                {
                    // [fixed last secondary common byte] is too low,
                    // and secondary weights would collide with compressed common secondaries.
                    throw new ICUException("[fixed last secondary common byte] is too low");
                }
                length &= 3;
            }
            ICUBinary.SkipBytes(inBytes, length);

            index  = IX_CONTEXTS_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 2)
            {
                if (data == null)
                {
                    throw new ICUException("Tailored contexts without tailored trie");
                }
                data.contexts = ICUBinary.GetString(inBytes, length / 2, length & 1);
            }
            else
            {
                ICUBinary.SkipBytes(inBytes, length);
            }

            index  = IX_UNSAFE_BWD_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 2)
            {
                if (data == null)
                {
                    throw new ICUException("Unsafe-backward-set but no mappings");
                }
                if (baseData == null)
                {
                    // Create the unsafe-backward set for the root collator.
                    // Include all non-zero combining marks and trail surrogates.
                    // We do this at load time, rather than at build time,
                    // to simplify Unicode version bootstrapping:
                    // The root data builder only needs the new FractionalUCA.txt data,
                    // but it need not be built with a version of ICU already updated to
                    // the corresponding new Unicode Character Database.
                    //
                    // The following is an optimized version of
                    // new UnicodeSet("[[:^lccc=0:][\\udc00-\\udfff]]").
                    // It is faster and requires fewer code dependencies.
                    tailoring.UnsafeBackwardSet = new UnicodeSet(0xdc00, 0xdfff);  // trail surrogates
                    data.nfcImpl.AddLcccChars(tailoring.UnsafeBackwardSet);
                }
                else
                {
                    // Clone the root collator's set contents.
                    tailoring.UnsafeBackwardSet = baseData.unsafeBackwardSet.CloneAsThawed();
                }
                // Add the ranges from the data file to the unsafe-backward set.
                USerializedSet sset       = new USerializedSet();
                char[]         unsafeData = ICUBinary.GetChars(inBytes, length / 2, length & 1);
                length = 0;
                sset.GetSet(unsafeData, 0);
                int   count = sset.CountRanges();
                int[] range = new int[2];
                for (int i = 0; i < count; ++i)
                {
                    sset.GetRange(i, range);
                    tailoring.UnsafeBackwardSet.Add(range[0], range[1]);
                }
                // Mark each lead surrogate as "unsafe"
                // if any of its 1024 associated supplementary code points is "unsafe".
                int c = 0x10000;
                for (int lead = 0xd800; lead < 0xdc00; ++lead, c += 0x400)
                {
                    if (!tailoring.UnsafeBackwardSet.ContainsNone(c, c + 0x3ff))
                    {
                        tailoring.UnsafeBackwardSet.Add(lead);
                    }
                }
                tailoring.UnsafeBackwardSet.Freeze();
                data.unsafeBackwardSet = tailoring.UnsafeBackwardSet;
            }
            else if (data == null)
            {
                // Nothing to do.
            }
            else if (baseData != null)
            {
                // No tailoring-specific data: Alias the root collator's set.
                data.unsafeBackwardSet = baseData.unsafeBackwardSet;
            }
            else
            {
                throw new ICUException("Missing unsafe-backward-set");
            }
            ICUBinary.SkipBytes(inBytes, length);

            // If the fast Latin format version is different,
            // or the version is set to 0 for "no fast Latin table",
            // then just always use the normal string comparison path.
            index  = IX_FAST_LATIN_TABLE_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (data != null)
            {
                data.fastLatinTable       = null;
                data.fastLatinTableHeader = null;
                if (((inIndexes[IX_OPTIONS] >> 16) & 0xff) == CollationFastLatin.VERSION)
                {
                    if (length >= 2)
                    {
                        char header0      = inBytes.GetChar();
                        int  headerLength = header0 & 0xff;
                        data.fastLatinTableHeader    = new char[headerLength];
                        data.fastLatinTableHeader[0] = header0;
                        for (int i = 1; i < headerLength; ++i)
                        {
                            data.fastLatinTableHeader[i] = inBytes.GetChar();
                        }
                        int tableLength = length / 2 - headerLength;
                        data.fastLatinTable = ICUBinary.GetChars(inBytes, tableLength, length & 1);
                        length = 0;
                        if ((header0 >> 8) != CollationFastLatin.VERSION)
                        {
                            throw new ICUException("Fast-Latin table version differs from version in data header");
                        }
                    }
                    else if (baseData != null)
                    {
                        data.fastLatinTable       = baseData.fastLatinTable;
                        data.fastLatinTableHeader = baseData.fastLatinTableHeader;
                    }
                }
            }
            ICUBinary.SkipBytes(inBytes, length);

            index  = IX_SCRIPTS_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 2)
            {
                if (data == null)
                {
                    throw new ICUException("Script order data but no mappings");
                }
                int        scriptsLength = length / 2;
                CharBuffer inChars       = inBytes.AsCharBuffer();
                data.numScripts = inChars.Get();
                // There must be enough entries for both arrays, including more than two range starts.
                int scriptStartsLength = scriptsLength - (1 + data.numScripts + 16);
                if (scriptStartsLength <= 2)
                {
                    throw new ICUException("Script order data too short");
                }
                inChars.Get(data.scriptsIndex = new char[data.numScripts + 16]);
                inChars.Get(data.scriptStarts = new char[scriptStartsLength]);
                if (!(data.scriptStarts[0] == 0 &&
                      data.scriptStarts[1] == ((Collation.MERGE_SEPARATOR_BYTE + 1) << 8) &&
                      data.scriptStarts[scriptStartsLength - 1] ==
                      (Collation.TRAIL_WEIGHT_BYTE << 8)))
                {
                    throw new ICUException("Script order data not valid");
                }
            }
            else if (data == null)
            {
                // Nothing to do.
            }
            else if (baseData != null)
            {
                data.numScripts   = baseData.numScripts;
                data.scriptsIndex = baseData.scriptsIndex;
                data.scriptStarts = baseData.scriptStarts;
            }
            ICUBinary.SkipBytes(inBytes, length);

            index  = IX_COMPRESSIBLE_BYTES_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            if (length >= 256)
            {
                if (data == null)
                {
                    throw new ICUException("Data for compressible primary lead bytes but no mappings");
                }
                data.compressibleBytes = new bool[256];
                for (int i = 0; i < 256; ++i)
                {
                    data.compressibleBytes[i] = inBytes.Get() != 0;
                }
                length -= 256;
            }
            else if (data == null)
            {
                // Nothing to do.
            }
            else if (baseData != null)
            {
                data.compressibleBytes = baseData.compressibleBytes;
            }
            else
            {
                throw new ICUException("Missing data for compressible primary lead bytes");
            }
            ICUBinary.SkipBytes(inBytes, length);

            index  = IX_RESERVED18_OFFSET;
            offset = inIndexes[index];
            length = inIndexes[index + 1] - offset;
            ICUBinary.SkipBytes(inBytes, length);

            CollationSettings ts = tailoring.Settings.ReadOnly;
            int options          = inIndexes[IX_OPTIONS] & 0xffff;

            char[] fastLatinPrimaries = new char[CollationFastLatin.LATIN_LIMIT];
            int    fastLatinOptions   = CollationFastLatin.GetOptions(
                tailoring.Data, ts, fastLatinPrimaries);

            if (options == ts.Options && ts.VariableTop != 0 &&
                Arrays.Equals(reorderCodes, ts.ReorderCodes) &&
                fastLatinOptions == ts.FastLatinOptions &&
                (fastLatinOptions < 0 ||
                 Arrays.Equals(fastLatinPrimaries, ts.FastLatinPrimaries)))
            {
                return;
            }

            CollationSettings settings = tailoring.Settings.CopyOnWrite();

            settings.Options = options;
            // Set variableTop from options and scripts data.
            settings.VariableTop = tailoring.Data.GetLastPrimaryForGroup(
                ReorderCodes.First + settings.MaxVariable);
            if (settings.VariableTop == 0)
            {
                throw new ICUException("The maxVariable could not be mapped to a variableTop");
            }

            if (reorderCodesLength != 0)
            {
                settings.AliasReordering(baseData, reorderCodes, reorderCodesLength, reorderTable);
            }

            settings.FastLatinOptions = CollationFastLatin.GetOptions(
                tailoring.Data, settings,
                settings.FastLatinPrimaries);
        }