//##################################################################################################
// SAMPLE code of class documentation
//##################################################################################################

void documentationSampleTokenizer()
{
    UTSampleWriter utsw= new UTSampleWriter("DOC_SAMPLES_ALIB_LIB_STRING_TOKEN.txt");


//! [DOC_SAMPLES_ALIB_LIB_STRING_TOKEN]
    // data string to tokenize
    AString data= new AString( "test;  abc ; 1,2 , 3 ; xyz ; including;separator" );

    // create tokenizer on data with ';' as delimiter
    Tokenizer tknzr= new Tokenizer( data, ';' );

    // read tokens

    System.Console.WriteLine( tknzr.Next() ); // will print "test"
    System.Console.WriteLine( tknzr.Next() ); // will print "abc"
    System.Console.WriteLine( tknzr.Next() ); // will print "1,2 , 3"

    // tokenize actual (third) token (nested tokenizer)
    Tokenizer subTknzr= new Tokenizer( tknzr.Actual,  ',');
    System.Console.Write( subTknzr.Next().ToString() );

    while( subTknzr.HasNext() )
        System.Console.Write( "~" + subTknzr.Next().ToString() );

    System.Console.WriteLine();

    // continue with the main tokenizer
    System.Console.WriteLine( tknzr.Next().ToString() ); // will print "xyz"

    // grab the rest, as we know that the last token might include our separator character
    System.Console.WriteLine( tknzr.GetRest().ToString() ); // will print "including;separator"
//! [DOC_SAMPLES_ALIB_LIB_STRING_TOKEN]

    utsw.FlushAndResetConsole();
}
    // #############################################################################################
    // logText
    // #############################################################################################

    /** ********************************************************************************************
     *
     *  The implementation of the abstract method of parent class TextLogger. Logs messages to the
     *  application console and/or the VStudio output window.
     *
     * @param domain      The <em>Log Domain</em>.
     * @param verbosity   The verbosity. This has been checked to be active already on this
     *                    stage and is provided to be able to be logged out only.
     * @param msg         The log message
     * @param scope       Information about the scope of the <em>Log Statement</em>..
     * @param lineNumber  The line number of a multi-line message, starting with 0. For
     *                    single line messages this is -1.
     **********************************************************************************************/
    override protected void logText(  Domain         domain,     Verbosity verbosity,
                                        AString        msg,
                                        ScopeInfo      scope,      int           lineNumber)
    {
        // loop over message, print the parts between the escape sequences
        Tokenizer msgParts= new Tokenizer( msg, '\x001B' );
        Substring actual=   msgParts.Actual;
        Substring rest=     msgParts.Rest;
        int       column=   0;
        for(;;)
        {
            msgParts.Next( Whitespaces.Keep );

            // check if this is an ANSI sequence already
            if ( rest.CharAtStart() == '[' )
            {
                // read the 'm'
                int idx= rest.IndexOf( 'm' );

                if ( idx < 0 ) // unknown ANSI Code
                {
                    ALIB.WARNING( "Unknown ANSI ESC Code " );
                    textWriter.Write( actual.Buf, actual.Start, actual.Length() );
                    continue;
                }

                column+= actual.Length();

                actual.End= rest.Start + idx ;
                rest.Start+= idx + 1;

                textWriter.Write( actual.Buf, actual.Start, actual.Length() );

                continue;
            }

            if ( actual.IsNotEmpty() )
            {
                textWriter.Write( actual.Buf, actual.Start, actual.Length() );
                column+= actual.Length();
            }

            // end of loop?
            if ( !msgParts.HasNext() )
                break;

            // found an ESC sequence
            char c= rest.Consume();

            // Colors
            bool isForeGround=  true;
            if( c == 'C' || c == 'c' )
            {
                isForeGround=  c== 'c';

                c= rest.Consume();
                int colNo= c - '0';
                ALIB.ASSERT_WARNING( colNo >=0 && colNo <=9, "Unknown ESC-c code" );

                // add bg
                colNo+=  isForeGround ? 0 : 10;

                // add light
                colNo+=  (isForeGround ? !IsBackgroundLight : IsBackgroundLight )  ? 20 : 0;

                textWriter.Write( ansiCols[ colNo ] );

            }

            // Styles
            else if ( c == 's' )
            {
                // bold/italics style not supported in Windows console

                // reset all
                if ( rest.Consume() == 'a' )
                {
                    textWriter.Write( ANSI_RESET );
                }
            }

            // auto tab / end of meta
            else if ( c == 't' || c == 'A')
            {
                bool endOfMeta= c == 'A';
                c=  rest.Consume();
                int extraSpace=  c >= '0' && c <= '9' ? (int)  ( c - '0' )
                                                      : (int)  ( c - 'A' ) + 10;

                int tabStop= AutoSizes.Next( column, extraSpace );

                Util.WriteSpaces( textWriter, tabStop - column );
                column= tabStop;

                if ( endOfMeta )
                {
                    String msgPrefix;
                    switch ( verbosity )
                    {
                        case lox.Verbosity.Verbose:   msgPrefix= MsgPrefixVerbose;     break;
                        case lox.Verbosity.Info:      msgPrefix= MsgPrefixInfo;        break;
                        case lox.Verbosity.Warning:   msgPrefix= MsgPrefixWarning;     break;
                        case lox.Verbosity.Error:     msgPrefix= MsgPrefixError;       break;
                        default:                  msgPrefix= "";                   break;
                    }
                    textWriter.Write( msgPrefix );
                }
            }

            // Link (we just colorize links here)
            else if ( c == 'l' )
            {
                textWriter.Write( rest.Consume() == 'S'
                                       ?  ( IsBackgroundLight ? ANSI_LIGHT_BLUE : ANSI_LIGHT_BLUE )
                                       :  ANSI_STD_COL                             );
            }

            else
            {
                ALIB.WARNING( "Unknown ESC code" );
            }

        } // write loop


        textWriter.WriteLine( MsgSuffix );
    }
    // #############################################################################################
    // public interface
    // #############################################################################################

        /** ****************************************************************************************
         * Constructs a scope info.
         * @param name              The name of the Lox we belong to.
         *                          Will be converted to upper case.
         * @param threadDictionary  A dictionary to map thread IDs to user friendly names which is
         *                          managed outside of this class.
         ******************************************************************************************/
        public ScopeInfo( String name,  Dictionary<int, String> threadDictionary )
        {
            loxName= name.ToUpper();
            ALIB.ASSERT_ERROR( !loxName.Equals( "GLOBAL" ), "Name \"GLOBAL\" not allowed for Lox instances" );

            this.threadDictionary= threadDictionary;

            cache=  new SourceFile[cacheSize= DefaultCacheSize];
            for ( int i= 0; i< cacheSize; i++ )
                cache[i]= new SourceFile();
            actual= cache[0];

            // read trim rules from config
            // do 2 times, 0== local list, 1== global list
            List<SourcePathTrimRule> trimInfoList;
            for( int trimInfoNo= 0; trimInfoNo < 2 ; trimInfoNo++ )
            {
                // check if done... or set list
                Variable variable= new Variable();
                if ( trimInfoNo == 0 )
                {
                    trimInfoList= LocalSPTRs;
                    variable.Define(ALox.SPTR_LOX, loxName).Load();
                }
                else
                {
                    if ( GlobalSPTRsReadFromConfig )
                        continue;
                    GlobalSPTRsReadFromConfig= true;
                    trimInfoList= GlobalSPTRs;
                    variable.Define(ALox.SPTR_GLOBAL).Load();
                }

                if( variable.Priority != 0 )
                {
                    Tokenizer ruleTok = new Tokenizer();
                    for( int ruleNo= 0; ruleNo< variable.Size(); ruleNo++ )
                    {
                        try {
                            ruleTok.Set( variable.GetString( ruleNo ), ',' );
                            SourcePathTrimRule rule= new SourcePathTrimRule();
                            rule.Priority= variable.Priority;
                            rule.Path= new AString();
                            ruleTok.Next();
                            if( ! ( rule.IsPrefix= !ruleTok.Actual.StartsWith( "*" ) ) )
                                ruleTok.Actual.Consume(1);
                            rule.Path._( ruleTok.Actual );
                            if ( rule.Path.CharAtEnd() == '*' )
                                rule.Path.DeleteEnd( 1 );
                            if ( rule.Path.IsEmpty() )
                                continue;

                            if( Path.DirectorySeparatorChar == '/' )
                                rule.Path.SearchAndReplaceAll( "\\", "/"  );
                            else
                                rule.Path.SearchAndReplaceAll( "/" , "\\" );


                            rule.IncludeString = ALIB.ReadInclusion( ruleTok.Next() );

                            if ( ruleTok.HasNext () )
                                ruleTok.Next().ConsumeInteger( out rule.TrimOffset );

                            rule.Sensitivity = ALIB.ReadCase( ruleTok.Next() );

                            if ( ruleTok.HasNext () )
                                rule.TrimReplacement= ruleTok.Next().ToString();

                            trimInfoList.Add( rule );
                        }
                        catch( Exception )
                        {
                            ALIB.ERROR( "Error reading source path trim rule from configuration. Invalid String: "
                                        + variable.GetString( ruleNo ).ToString() );
                        }
                    }
                }
            }
        }
    /** ********************************************************************************************
     *
     *  The implementation of the abstract method of parent class TextLogger. Logs messages to the
     *  application console and/or the VStudio output window.
     *
     * @param domain      The <em>Log Domain</em>.
     * @param verbosity   The verbosity. This has been checked to be active already on this
     *                    stage and is provided to be able to be logged out only.
     * @param msg         The log message.
     * @param scope       Information about the scope of the <em>Log Statement</em>..
     * @param lineNumber  The line number of a multi-line message, starting with 0. For
     *                    single line messages this is -1.
     **********************************************************************************************/
    override protected void logText(  Domain     domain,     Verbosity verbosity,
                                      AString    msg,
                                      ScopeInfo  scope,      int       lineNumber)
    {
        // get actual console attributes
        ConsoleColor actualFGCol= Console.ForegroundColor;
        ConsoleColor actualBGCol= Console.BackgroundColor;

        // loop over message, print the parts between the escape sequences
        Tokenizer msgParts= new Tokenizer( msg, '\x1B' );
        Substring actual=   msgParts.Actual;
        Substring rest=     msgParts.Rest;
        int       column=   0;
        for(;;)
        {
            if ( msgParts.Next(Whitespaces.Keep).IsNotEmpty() )
            {
                #if !(ALOX_WP71 || ALOX_WP8)
                    Console.Write( msg.Buffer(), actual.Start, actual.Length() );
                #else
                    Console.Write( msg.ToString( 0, actual.Start, actual.Length() );
                #endif
                column+= actual.Length();
            }

            // end of loop?
            if ( !msgParts.HasNext() )
                break;

            // found an ESC sequence
            char c= rest.Consume();

            // Colors
            bool isForeGround=  true;
            if( c == 'C' || c == 'c' )
            {
                isForeGround=  c== 'c';

                c= rest.Consume();
                int colNo= c - '0';
                ALIB.ASSERT_WARNING( colNo >=0 && colNo <=9, "Unknown ESC-c code" );


                // set color
                if( colNo >= 0 && colNo <= 8 || colNo == 8)
                {
                    ConsoleColor[] cols=  (isForeGround ? !IsBackgroundLight : IsBackgroundLight )  ? lightColors : darkColors;
                    if ( isForeGround )
                        Console.ForegroundColor= cols[ colNo ];
                    else
                        Console.BackgroundColor= cols[ colNo ];
                }
                else if ( colNo == 9 )
                {
                    if ( isForeGround )
                        Console.ForegroundColor= actualFGCol;
                    else
                        Console.BackgroundColor= actualBGCol;
                }

                else
                {
                    ALIB.WARNING( "Unknown ESC- code" );
                }

            }

            // Styles
            else if ( c == 's' )
            {
                // bold/italics style not supported in Windows console

                // reset all
                if ( rest.Consume() == 'a' )
                {
                    Console.ForegroundColor= actualFGCol;
                    Console.BackgroundColor= actualBGCol;
                }
            }

            // auto tab / end of meta
            else if ( c == 't' || c == 'A')
            {
                bool endOfMeta= c == 'A';
                c=  rest.Consume();
                int extraSpace=  c >= '0' && c <= '9' ? (int)  ( c - '0' )
                                                      : (int)  ( c - 'A' ) + 10;

                int tabStop= AutoSizes.Next( column, extraSpace );

                Util.WriteSpaces( Console.Out, tabStop - column );
                column= tabStop;

                if ( endOfMeta )
                {
                    switch ( verbosity )
                    {
                        case Verbosity.Verbose:   Console.ForegroundColor= MsgColorVerbose;     break;
                        case Verbosity.Info:      Console.ForegroundColor= MsgColorInfo;        break;
                        case Verbosity.Warning:   Console.ForegroundColor= MsgColorWarning;     break;
                        case Verbosity.Error:     Console.ForegroundColor= MsgColorError;       break;
                        default:                  break;
                    }
                }
            }

            // Link (we just colorize links here)
            else if ( c == 'l' )
            {
                if ( rest.Consume() == 'S' )
                    Console.ForegroundColor=  IsBackgroundLight ? ConsoleColor.DarkBlue : ConsoleColor.Blue;
                else
                    Console.ForegroundColor=  actualFGCol;
            }

            else
            {
                ALIB.WARNING( "Unknown ESC code" );
            }

        } // write loop

        // reset colors
        Console.ForegroundColor= actualFGCol;
        Console.BackgroundColor= actualBGCol;

        // write NL
        #if !(ALOX_WP71 || ALOX_WP8)
            Console.WriteLine();
        #else
            Console.WriteLine();
        #endif

    }
public void Tokenize()
{
    AString astr=  new AString();
    AString res= new AString();

    // tokenizing empty string
    astr.Clear()._( "" );
    res.Clear();
    {
        Tokenizer tknzr= new Tokenizer( astr, ',' );   UT_EQ( true,  tknzr.HasNext() );
        tknzr.Next().CopyTo( res, true );              UT_EQ( "", res );
    }

    // tokenizing no delim
    astr.Clear()._( "abc" );
    res.Clear();
    {
        Tokenizer tknzr= new Tokenizer( astr, ',' );                UT_EQ( true,  tknzr.HasNext() );
        tknzr.Next().CopyTo( res, true );     UT_EQ( "abc", res );
    }

    // tokenizing
    {
        tokenizerTest( "abc",       res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "abc@",      res );

        tokenizerTest( "a,bc",      res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "a@bc@",     res );

        tokenizerTest( "a,bc",      res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "a@bc@",     res );

        tokenizerTest( ",",         res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "@@",        res );

        tokenizerTest( ",,",        res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "@@@",       res );

        tokenizerTest( "a,b,c,,",   res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "a@b@c@@@",  res );

        tokenizerTest( "a,b,c",     res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "a@b@c@",    res );

        tokenizerTest( ",a,b,c",    res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "@a@b@c@",   res );

        tokenizerTest( "123567",    res, ',', '@',   Whitespaces.Trim, 2,   2 );
        UT_EQ ( "3@",        res );

        tokenizerTest( "123567",    res, ',', '@',   Whitespaces.Trim, 2,   1 );
        UT_EQ ( "@",         res );

        tokenizerTest( "  abc , def , ghe ",    res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "abc@def@ghe@",          res );

        tokenizerTest( "abc , def,ghe,",        res, ',', '@',   Whitespaces.Trim, -1,   -1 );
        UT_EQ ( "abc@def@ghe@@",         res );


        tokenizerTest( "  abc , def , ghe ",    res, ',', '@',   Whitespaces.Keep, -1,   -1 );
        UT_EQ ( "  abc @ def @ ghe @",   res );

        tokenizerTest( "abc , def,ghe,",        res, ',', '@',   Whitespaces.Keep, -1,   -1 );
        UT_EQ ( "abc @ def@ghe@@",       res );
    }

    // tokenizing with different delimiters
    {
        astr.Clear()._( "1,5;3@4" );
        Tokenizer tknzr= new Tokenizer(astr, ',');
        tknzr.Next()                       .CopyTo( res );    UT_EQ ( "1",   res );  UT_EQ( true,  tknzr.HasNext() );
        tknzr.Next( Whitespaces.Trim, ';' ).CopyTo( res );    UT_EQ ( "5",   res );  UT_EQ( true,  tknzr.HasNext() );
        tknzr.Next( Whitespaces.Trim, '@' ).CopyTo( res );    UT_EQ ( "3",   res );  UT_EQ( true,  tknzr.HasNext() );
        tknzr.Next( Whitespaces.Trim, '-' ).CopyTo( res );    UT_EQ ( "4",   res );  UT_EQ( false, tknzr.HasNext() );
    }

    // tokenizing with different delimiters
    {
        astr.Clear()._( "abc, 5;\t3;;; 4  " );
        Tokenizer tknzr= new Tokenizer(astr,',');
        tknzr.Next()                       .CopyTo( res );    UT_EQ ( "abc", res );  UT_EQ( true,  tknzr.HasNext() );
        tknzr.Next( Whitespaces.Trim, ';' ).CopyTo( res );    UT_EQ ( "5",   res );  UT_EQ( true,  tknzr.HasNext() );
        tknzr.Next()                       .CopyTo( res );    UT_EQ ( "3",   res );  UT_EQ( true,  tknzr.HasNext() );
        tknzr.Next()                       .CopyTo( res );    UT_EQ ( "",    res );  UT_EQ( true,  tknzr.HasNext() );
        tknzr.GetRest()                    .CopyTo( res );    UT_EQ ( "; 4", res );  UT_EQ( false, tknzr.HasNext() );
    }

    // sub-tokens
    {
        astr.Clear()._( "1,2;3 , 4;5,;," );
        Tokenizer tknzr= new Tokenizer(astr, ';');

        Tokenizer tknzr2= new Tokenizer( tknzr.Next(), ',');
        tknzr2.Next().CopyTo( res ); UT_EQ ( "1", res );  UT_TRUE(  tknzr2.HasNext() );
        tknzr2.Next().CopyTo( res ); UT_EQ ( "2", res );  UT_TRUE( !tknzr2.HasNext() );
        UT_TRUE( tknzr.HasNext() );

        tknzr2.Set( tknzr.Next(), ',');
        tknzr2.Next().CopyTo( res ); UT_EQ ( "3", res );  UT_TRUE(  tknzr2.HasNext() );
        tknzr2.Next().CopyTo( res ); UT_EQ ( "4", res );  UT_TRUE( !tknzr2.HasNext() );
        UT_TRUE( tknzr.HasNext() );

        tknzr2.Set( tknzr.Next(), ',');
        tknzr2.Next().CopyTo( res ); UT_EQ ( "5", res );  UT_TRUE(  tknzr2.HasNext() );
        tknzr2.Next().CopyTo( res ); UT_EQ ( "",  res );  UT_TRUE( !tknzr2.HasNext() );
        UT_TRUE( tknzr.HasNext() );

        tknzr2.Set( tknzr.Next(), ',');
        tknzr2.Next().CopyTo( res ); UT_EQ ( "", res );  UT_TRUE(  tknzr2.HasNext() );
        tknzr2.Next().CopyTo( res ); UT_EQ ( "", res );  UT_TRUE( !tknzr2.HasNext() );
        UT_TRUE( !tknzr.HasNext() );
    }
}
//--------------------------------------------------------------------------------------------------
//--- Test Tokenizer
//--------------------------------------------------------------------------------------------------
void tokenizerTest( string inputString, AString res, char delim, char newDelim,
                    Whitespaces trim, int inpStart= -1, int inpEnd= -1  )
{
    Substring inp= new Substring( inputString );
    if ( inpStart >= 0 ) inp.Start= inpStart;
    if ( inpEnd   >= 0 ) inp.End=   inpEnd;
    res.Clear();

    Tokenizer tknzr= new Tokenizer( inp, delim );

    while( tknzr.HasNext() )
    {
        tknzr.Next(trim).CopyTo( res, true );
        res._( newDelim );
    }

}