/// <summary>
        /// Converts an index value for a text column into the entry value (which
        /// is based on a variety of nifty codes).
        /// </summary>
        /// <remarks>
        /// Converts an index value for a text column into the entry value (which
        /// is based on a variety of nifty codes).
        /// </remarks>
        /// <exception cref="System.IO.IOException"></exception>
        internal virtual void WriteNonNullIndexTextValue(object value, ByteUtil.ByteStream
                                                         bout, bool isAscending)
        {
            // first, convert to string
            string str = Column.ToCharSequence(value).ToString();

            // all text columns (including memos) are only indexed up to the max
            // number of chars in a VARCHAR column
            if (str.Length > MAX_TEXT_INDEX_CHAR_LENGTH)
            {
                str = Sharpen.Runtime.Substring(str, 0, MAX_TEXT_INDEX_CHAR_LENGTH);
            }
            // record pprevious entry length so we can do any post-processing
            // necessary for this entry (handling descending)
            int prevLength = bout.GetLength();

            // now, convert each character to a "code" of one or more bytes
            GeneralLegacyIndexCodes.ExtraCodesStream extraCodes = null;
            ByteUtil.ByteStream unprintableCodes = null;
            ByteUtil.ByteStream crazyCodes       = null;
            int charOffset = 0;

            for (int i = 0; i < str.Length; ++i)
            {
                char c = str[i];
                GeneralLegacyIndexCodes.CharHandler ch = GetCharHandler(c);
                int    curCharOffset = charOffset;
                byte[] bytes         = ch.GetInlineBytes();
                if (bytes != null)
                {
                    // write the "inline" codes immediately
                    bout.Write(bytes);
                    // only increment the charOffset for chars with inline codes
                    ++charOffset;
                }
                if (ch.GetEncodingType() == GeneralLegacyIndexCodes.Type.SIMPLE)
                {
                    // common case, skip further code handling
                    continue;
                }
                bytes = ch.GetExtraBytes();
                byte extraCodeModifier = ch.GetExtraByteModifier();
                if ((bytes != null) || (extraCodeModifier != 0))
                {
                    if (extraCodes == null)
                    {
                        extraCodes = new GeneralLegacyIndexCodes.ExtraCodesStream(str.Length);
                    }
                    // keep track of the extra codes for later
                    WriteExtraCodes(curCharOffset, bytes, extraCodeModifier, extraCodes);
                }
                bytes = ch.GetUnprintableBytes();
                if (bytes != null)
                {
                    if (unprintableCodes == null)
                    {
                        unprintableCodes = new ByteUtil.ByteStream();
                    }
                    // keep track of the unprintable codes for later
                    WriteUnprintableCodes(curCharOffset, bytes, unprintableCodes, extraCodes);
                }
                byte crazyFlag = ch.GetCrazyFlag();
                if (crazyFlag != 0)
                {
                    if (crazyCodes == null)
                    {
                        crazyCodes = new ByteUtil.ByteStream();
                    }
                    // keep track of the crazy flags for later
                    crazyCodes.Write(crazyFlag);
                }
            }
            // write end text flag
            bout.Write(END_TEXT);
            bool hasExtraCodes = TrimExtraCodes(extraCodes, unchecked ((byte)0), INTERNATIONAL_EXTRA_PLACEHOLDER
                                                );
            bool hasUnprintableCodes = (unprintableCodes != null);
            bool hasCrazyCodes       = (crazyCodes != null);

            if (hasExtraCodes || hasUnprintableCodes || hasCrazyCodes)
            {
                // we write all the international extra bytes first
                if (hasExtraCodes)
                {
                    extraCodes.WriteTo(bout);
                }
                if (hasCrazyCodes || hasUnprintableCodes)
                {
                    // write 2 more end flags
                    bout.Write(END_TEXT);
                    bout.Write(END_TEXT);
                    // next come the crazy flags
                    if (hasCrazyCodes)
                    {
                        WriteCrazyCodes(crazyCodes, bout);
                        // if we are writing unprintable codes after this, tack on another
                        // code
                        if (hasUnprintableCodes)
                        {
                            bout.Write(CRAZY_CODES_UNPRINT_SUFFIX);
                        }
                    }
                    // then we write all the unprintable extra bytes
                    if (hasUnprintableCodes)
                    {
                        // write another end flag
                        bout.Write(END_TEXT);
                        unprintableCodes.WriteTo(bout);
                    }
                }
            }
            // handle descending order by inverting the bytes
            if (!isAscending)
            {
                // we actually write the end byte before flipping the bytes, and write
                // another one after flipping
                bout.Write(END_EXTRA_TEXT);
                // flip the bytes that we have written thus far for this text value
                IndexData.FlipBytes(bout.GetBytes(), prevLength, (bout.GetLength() - prevLength));
            }
            // write end extra text
            bout.Write(END_EXTRA_TEXT);
        }
        /// <summary>
        /// Loads the CharHandlers for the given range of characters from the
        /// resource file with the given name.
        /// </summary>
        /// <remarks>
        /// Loads the CharHandlers for the given range of characters from the
        /// resource file with the given name.
        /// </remarks>
        internal static GeneralLegacyIndexCodes.CharHandler[] LoadCodes(string codesFilePath
                                                                        , char firstChar, char lastChar)
        {
            int numCodes = (AsUnsignedChar(lastChar) - AsUnsignedChar(firstChar)) + 1;

            GeneralLegacyIndexCodes.CharHandler[] values = new GeneralLegacyIndexCodes.CharHandler
                                                           [numCodes];
            IDictionary <string, GeneralLegacyIndexCodes.Type> prefixMap = new Dictionary <string
                                                                                           , GeneralLegacyIndexCodes.Type>();

            prefixMap.Put("S", GeneralLegacyIndexCodes.Type.SIMPLE);
            prefixMap.Put("I", GeneralLegacyIndexCodes.Type.INTERNATIONAL);
            prefixMap.Put("U", GeneralLegacyIndexCodes.Type.UNPRINTABLE);
            prefixMap.Put("P", GeneralLegacyIndexCodes.Type.UNPRINTABLE_EXT);
            prefixMap.Put("Z", GeneralLegacyIndexCodes.Type.INTERNATIONAL_EXT);
            prefixMap.Put("X", GeneralLegacyIndexCodes.Type.IGNORED);

            InputStreamReader reader = null;

            try
            {
                reader = new InputStreamReader(Database.GetResourceAsStream(codesFilePath
                                                                            ), "US-ASCII");
                int start = AsUnsignedChar(firstChar);
                int end   = AsUnsignedChar(lastChar);
                for (int i = start; i <= end; ++i)
                {
                    char c = (char)i;
                    GeneralLegacyIndexCodes.CharHandler ch = null;
                    if (char.IsHighSurrogate(c) || char.IsLowSurrogate(c))
                    {
                        // surrogate chars are not included in the codes files
                        ch = SURROGATE_CHAR_HANDLER;
                    }
                    else
                    {
                        string codeLine = reader.ReadLine();
                        ch = ParseCodes(prefixMap, codeLine);
                    }
                    values[(i - start)] = ch;
                }
            }
            catch (IOException e)
            {
                throw new RuntimeException("failed loading index codes file " + codesFilePath, e);
            }
            finally
            {
                if (reader != null)
                {
                    try
                    {
                        reader.Close();
                    }
                    catch (IOException)
                    {
                    }
                }
            }
            // ignored
            return(values);
        }