//################################################################################################## // SAMPLE code of class documentation //################################################################################################## void documentationSampleTokenizer() { UTSampleWriter utsw= new UTSampleWriter("DOC_SAMPLES_ALIB_LIB_STRING_TOKEN.txt"); //! [DOC_SAMPLES_ALIB_LIB_STRING_TOKEN] // data string to tokenize AString data= new AString( "test; abc ; 1,2 , 3 ; xyz ; including;separator" ); // create tokenizer on data with ';' as delimiter Tokenizer tknzr= new Tokenizer( data, ';' ); // read tokens System.Console.WriteLine( tknzr.Next() ); // will print "test" System.Console.WriteLine( tknzr.Next() ); // will print "abc" System.Console.WriteLine( tknzr.Next() ); // will print "1,2 , 3" // tokenize actual (third) token (nested tokenizer) Tokenizer subTknzr= new Tokenizer( tknzr.Actual, ','); System.Console.Write( subTknzr.Next().ToString() ); while( subTknzr.HasNext() ) System.Console.Write( "~" + subTknzr.Next().ToString() ); System.Console.WriteLine(); // continue with the main tokenizer System.Console.WriteLine( tknzr.Next().ToString() ); // will print "xyz" // grab the rest, as we know that the last token might include our separator character System.Console.WriteLine( tknzr.GetRest().ToString() ); // will print "including;separator" //! [DOC_SAMPLES_ALIB_LIB_STRING_TOKEN] utsw.FlushAndResetConsole(); }
// ############################################################################################# // logText // ############################################################################################# /** ******************************************************************************************** * * The implementation of the abstract method of parent class TextLogger. Logs messages to the * application console and/or the VStudio output window. * * @param domain The <em>Log Domain</em>. * @param verbosity The verbosity. This has been checked to be active already on this * stage and is provided to be able to be logged out only. * @param msg The log message * @param scope Information about the scope of the <em>Log Statement</em>.. * @param lineNumber The line number of a multi-line message, starting with 0. For * single line messages this is -1. **********************************************************************************************/ override protected void logText( Domain domain, Verbosity verbosity, AString msg, ScopeInfo scope, int lineNumber) { // loop over message, print the parts between the escape sequences Tokenizer msgParts= new Tokenizer( msg, '\x001B' ); Substring actual= msgParts.Actual; Substring rest= msgParts.Rest; int column= 0; for(;;) { msgParts.Next( Whitespaces.Keep ); // check if this is an ANSI sequence already if ( rest.CharAtStart() == '[' ) { // read the 'm' int idx= rest.IndexOf( 'm' ); if ( idx < 0 ) // unknown ANSI Code { ALIB.WARNING( "Unknown ANSI ESC Code " ); textWriter.Write( actual.Buf, actual.Start, actual.Length() ); continue; } column+= actual.Length(); actual.End= rest.Start + idx ; rest.Start+= idx + 1; textWriter.Write( actual.Buf, actual.Start, actual.Length() ); continue; } if ( actual.IsNotEmpty() ) { textWriter.Write( actual.Buf, actual.Start, actual.Length() ); column+= actual.Length(); } // end of loop? if ( !msgParts.HasNext() ) break; // found an ESC sequence char c= rest.Consume(); // Colors bool isForeGround= true; if( c == 'C' || c == 'c' ) { isForeGround= c== 'c'; c= rest.Consume(); int colNo= c - '0'; ALIB.ASSERT_WARNING( colNo >=0 && colNo <=9, "Unknown ESC-c code" ); // add bg colNo+= isForeGround ? 0 : 10; // add light colNo+= (isForeGround ? !IsBackgroundLight : IsBackgroundLight ) ? 20 : 0; textWriter.Write( ansiCols[ colNo ] ); } // Styles else if ( c == 's' ) { // bold/italics style not supported in Windows console // reset all if ( rest.Consume() == 'a' ) { textWriter.Write( ANSI_RESET ); } } // auto tab / end of meta else if ( c == 't' || c == 'A') { bool endOfMeta= c == 'A'; c= rest.Consume(); int extraSpace= c >= '0' && c <= '9' ? (int) ( c - '0' ) : (int) ( c - 'A' ) + 10; int tabStop= AutoSizes.Next( column, extraSpace ); Util.WriteSpaces( textWriter, tabStop - column ); column= tabStop; if ( endOfMeta ) { String msgPrefix; switch ( verbosity ) { case lox.Verbosity.Verbose: msgPrefix= MsgPrefixVerbose; break; case lox.Verbosity.Info: msgPrefix= MsgPrefixInfo; break; case lox.Verbosity.Warning: msgPrefix= MsgPrefixWarning; break; case lox.Verbosity.Error: msgPrefix= MsgPrefixError; break; default: msgPrefix= ""; break; } textWriter.Write( msgPrefix ); } } // Link (we just colorize links here) else if ( c == 'l' ) { textWriter.Write( rest.Consume() == 'S' ? ( IsBackgroundLight ? ANSI_LIGHT_BLUE : ANSI_LIGHT_BLUE ) : ANSI_STD_COL ); } else { ALIB.WARNING( "Unknown ESC code" ); } } // write loop textWriter.WriteLine( MsgSuffix ); }
/** **************************************************************************************** * Writes a list of comments to the file. Comment lines are started with '#'. * @param os The output stream to write to. * @param comments The comment lines for the section. * @param tknzr A temporary tokenizer needed internally. ******************************************************************************************/ protected void writeComments( TextWriter os, AString comments, Tokenizer tknzr ) { // is empty when trimmed? (we are lenting tokeinizers field 'actual' to test) if ( tknzr.Actual.Set( comments ).Trim().IsEmpty() ) return; // tokenize by NEWLINE character tknzr.Set( comments, '\n' ); char[] oldWhiteSpaces= tknzr.Whitespaces; tknzr.Whitespaces= " \r\t".ToCharArray(); // \n is not a whitespace while( tknzr.Next(Whitespaces.Keep).IsNotNull() ) { if ( !startsWithCommentSymbol( tknzr.Actual ) ) os.Write( DefaultCommentPrefix ); os.WriteLine( tknzr.Actual.Buf, tknzr.Actual.Start, tknzr.Actual.Length() ); } tknzr.Whitespaces= oldWhiteSpaces; }
// ############################################################################################# // public interface // ############################################################################################# /** **************************************************************************************** * Constructs a scope info. * @param name The name of the Lox we belong to. * Will be converted to upper case. * @param threadDictionary A dictionary to map thread IDs to user friendly names which is * managed outside of this class. ******************************************************************************************/ public ScopeInfo( String name, Dictionary<int, String> threadDictionary ) { loxName= name.ToUpper(); ALIB.ASSERT_ERROR( !loxName.Equals( "GLOBAL" ), "Name \"GLOBAL\" not allowed for Lox instances" ); this.threadDictionary= threadDictionary; cache= new SourceFile[cacheSize= DefaultCacheSize]; for ( int i= 0; i< cacheSize; i++ ) cache[i]= new SourceFile(); actual= cache[0]; // read trim rules from config // do 2 times, 0== local list, 1== global list List<SourcePathTrimRule> trimInfoList; for( int trimInfoNo= 0; trimInfoNo < 2 ; trimInfoNo++ ) { // check if done... or set list Variable variable= new Variable(); if ( trimInfoNo == 0 ) { trimInfoList= LocalSPTRs; variable.Define(ALox.SPTR_LOX, loxName).Load(); } else { if ( GlobalSPTRsReadFromConfig ) continue; GlobalSPTRsReadFromConfig= true; trimInfoList= GlobalSPTRs; variable.Define(ALox.SPTR_GLOBAL).Load(); } if( variable.Priority != 0 ) { Tokenizer ruleTok = new Tokenizer(); for( int ruleNo= 0; ruleNo< variable.Size(); ruleNo++ ) { try { ruleTok.Set( variable.GetString( ruleNo ), ',' ); SourcePathTrimRule rule= new SourcePathTrimRule(); rule.Priority= variable.Priority; rule.Path= new AString(); ruleTok.Next(); if( ! ( rule.IsPrefix= !ruleTok.Actual.StartsWith( "*" ) ) ) ruleTok.Actual.Consume(1); rule.Path._( ruleTok.Actual ); if ( rule.Path.CharAtEnd() == '*' ) rule.Path.DeleteEnd( 1 ); if ( rule.Path.IsEmpty() ) continue; if( Path.DirectorySeparatorChar == '/' ) rule.Path.SearchAndReplaceAll( "\\", "/" ); else rule.Path.SearchAndReplaceAll( "/" , "\\" ); rule.IncludeString = ALIB.ReadInclusion( ruleTok.Next() ); if ( ruleTok.HasNext () ) ruleTok.Next().ConsumeInteger( out rule.TrimOffset ); rule.Sensitivity = ALIB.ReadCase( ruleTok.Next() ); if ( ruleTok.HasNext () ) rule.TrimReplacement= ruleTok.Next().ToString(); trimInfoList.Add( rule ); } catch( Exception ) { ALIB.ERROR( "Error reading source path trim rule from configuration. Invalid String: " + variable.GetString( ruleNo ).ToString() ); } } } } }
/** ******************************************************************************************** * * The implementation of the abstract method of parent class TextLogger. Logs messages to the * application console and/or the VStudio output window. * * @param domain The <em>Log Domain</em>. * @param verbosity The verbosity. This has been checked to be active already on this * stage and is provided to be able to be logged out only. * @param msg The log message. * @param scope Information about the scope of the <em>Log Statement</em>.. * @param lineNumber The line number of a multi-line message, starting with 0. For * single line messages this is -1. **********************************************************************************************/ override protected void logText( Domain domain, Verbosity verbosity, AString msg, ScopeInfo scope, int lineNumber) { // get actual console attributes ConsoleColor actualFGCol= Console.ForegroundColor; ConsoleColor actualBGCol= Console.BackgroundColor; // loop over message, print the parts between the escape sequences Tokenizer msgParts= new Tokenizer( msg, '\x1B' ); Substring actual= msgParts.Actual; Substring rest= msgParts.Rest; int column= 0; for(;;) { if ( msgParts.Next(Whitespaces.Keep).IsNotEmpty() ) { #if !(ALOX_WP71 || ALOX_WP8) Console.Write( msg.Buffer(), actual.Start, actual.Length() ); #else Console.Write( msg.ToString( 0, actual.Start, actual.Length() ); #endif column+= actual.Length(); } // end of loop? if ( !msgParts.HasNext() ) break; // found an ESC sequence char c= rest.Consume(); // Colors bool isForeGround= true; if( c == 'C' || c == 'c' ) { isForeGround= c== 'c'; c= rest.Consume(); int colNo= c - '0'; ALIB.ASSERT_WARNING( colNo >=0 && colNo <=9, "Unknown ESC-c code" ); // set color if( colNo >= 0 && colNo <= 8 || colNo == 8) { ConsoleColor[] cols= (isForeGround ? !IsBackgroundLight : IsBackgroundLight ) ? lightColors : darkColors; if ( isForeGround ) Console.ForegroundColor= cols[ colNo ]; else Console.BackgroundColor= cols[ colNo ]; } else if ( colNo == 9 ) { if ( isForeGround ) Console.ForegroundColor= actualFGCol; else Console.BackgroundColor= actualBGCol; } else { ALIB.WARNING( "Unknown ESC- code" ); } } // Styles else if ( c == 's' ) { // bold/italics style not supported in Windows console // reset all if ( rest.Consume() == 'a' ) { Console.ForegroundColor= actualFGCol; Console.BackgroundColor= actualBGCol; } } // auto tab / end of meta else if ( c == 't' || c == 'A') { bool endOfMeta= c == 'A'; c= rest.Consume(); int extraSpace= c >= '0' && c <= '9' ? (int) ( c - '0' ) : (int) ( c - 'A' ) + 10; int tabStop= AutoSizes.Next( column, extraSpace ); Util.WriteSpaces( Console.Out, tabStop - column ); column= tabStop; if ( endOfMeta ) { switch ( verbosity ) { case Verbosity.Verbose: Console.ForegroundColor= MsgColorVerbose; break; case Verbosity.Info: Console.ForegroundColor= MsgColorInfo; break; case Verbosity.Warning: Console.ForegroundColor= MsgColorWarning; break; case Verbosity.Error: Console.ForegroundColor= MsgColorError; break; default: break; } } } // Link (we just colorize links here) else if ( c == 'l' ) { if ( rest.Consume() == 'S' ) Console.ForegroundColor= IsBackgroundLight ? ConsoleColor.DarkBlue : ConsoleColor.Blue; else Console.ForegroundColor= actualFGCol; } else { ALIB.WARNING( "Unknown ESC code" ); } } // write loop // reset colors Console.ForegroundColor= actualFGCol; Console.BackgroundColor= actualBGCol; // write NL #if !(ALOX_WP71 || ALOX_WP8) Console.WriteLine(); #else Console.WriteLine(); #endif }
public void Tokenize() { AString astr= new AString(); AString res= new AString(); // tokenizing empty string astr.Clear()._( "" ); res.Clear(); { Tokenizer tknzr= new Tokenizer( astr, ',' ); UT_EQ( true, tknzr.HasNext() ); tknzr.Next().CopyTo( res, true ); UT_EQ( "", res ); } // tokenizing no delim astr.Clear()._( "abc" ); res.Clear(); { Tokenizer tknzr= new Tokenizer( astr, ',' ); UT_EQ( true, tknzr.HasNext() ); tknzr.Next().CopyTo( res, true ); UT_EQ( "abc", res ); } // tokenizing { tokenizerTest( "abc", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "abc@", res ); tokenizerTest( "a,bc", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "a@bc@", res ); tokenizerTest( "a,bc", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "a@bc@", res ); tokenizerTest( ",", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "@@", res ); tokenizerTest( ",,", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "@@@", res ); tokenizerTest( "a,b,c,,", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "a@b@c@@@", res ); tokenizerTest( "a,b,c", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "a@b@c@", res ); tokenizerTest( ",a,b,c", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "@a@b@c@", res ); tokenizerTest( "123567", res, ',', '@', Whitespaces.Trim, 2, 2 ); UT_EQ ( "3@", res ); tokenizerTest( "123567", res, ',', '@', Whitespaces.Trim, 2, 1 ); UT_EQ ( "@", res ); tokenizerTest( " abc , def , ghe ", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "abc@def@ghe@", res ); tokenizerTest( "abc , def,ghe,", res, ',', '@', Whitespaces.Trim, -1, -1 ); UT_EQ ( "abc@def@ghe@@", res ); tokenizerTest( " abc , def , ghe ", res, ',', '@', Whitespaces.Keep, -1, -1 ); UT_EQ ( " abc @ def @ ghe @", res ); tokenizerTest( "abc , def,ghe,", res, ',', '@', Whitespaces.Keep, -1, -1 ); UT_EQ ( "abc @ def@ghe@@", res ); } // tokenizing with different delimiters { astr.Clear()._( "1,5;3@4" ); Tokenizer tknzr= new Tokenizer(astr, ','); tknzr.Next() .CopyTo( res ); UT_EQ ( "1", res ); UT_EQ( true, tknzr.HasNext() ); tknzr.Next( Whitespaces.Trim, ';' ).CopyTo( res ); UT_EQ ( "5", res ); UT_EQ( true, tknzr.HasNext() ); tknzr.Next( Whitespaces.Trim, '@' ).CopyTo( res ); UT_EQ ( "3", res ); UT_EQ( true, tknzr.HasNext() ); tknzr.Next( Whitespaces.Trim, '-' ).CopyTo( res ); UT_EQ ( "4", res ); UT_EQ( false, tknzr.HasNext() ); } // tokenizing with different delimiters { astr.Clear()._( "abc, 5;\t3;;; 4 " ); Tokenizer tknzr= new Tokenizer(astr,','); tknzr.Next() .CopyTo( res ); UT_EQ ( "abc", res ); UT_EQ( true, tknzr.HasNext() ); tknzr.Next( Whitespaces.Trim, ';' ).CopyTo( res ); UT_EQ ( "5", res ); UT_EQ( true, tknzr.HasNext() ); tknzr.Next() .CopyTo( res ); UT_EQ ( "3", res ); UT_EQ( true, tknzr.HasNext() ); tknzr.Next() .CopyTo( res ); UT_EQ ( "", res ); UT_EQ( true, tknzr.HasNext() ); tknzr.GetRest() .CopyTo( res ); UT_EQ ( "; 4", res ); UT_EQ( false, tknzr.HasNext() ); } // sub-tokens { astr.Clear()._( "1,2;3 , 4;5,;," ); Tokenizer tknzr= new Tokenizer(astr, ';'); Tokenizer tknzr2= new Tokenizer( tknzr.Next(), ','); tknzr2.Next().CopyTo( res ); UT_EQ ( "1", res ); UT_TRUE( tknzr2.HasNext() ); tknzr2.Next().CopyTo( res ); UT_EQ ( "2", res ); UT_TRUE( !tknzr2.HasNext() ); UT_TRUE( tknzr.HasNext() ); tknzr2.Set( tknzr.Next(), ','); tknzr2.Next().CopyTo( res ); UT_EQ ( "3", res ); UT_TRUE( tknzr2.HasNext() ); tknzr2.Next().CopyTo( res ); UT_EQ ( "4", res ); UT_TRUE( !tknzr2.HasNext() ); UT_TRUE( tknzr.HasNext() ); tknzr2.Set( tknzr.Next(), ','); tknzr2.Next().CopyTo( res ); UT_EQ ( "5", res ); UT_TRUE( tknzr2.HasNext() ); tknzr2.Next().CopyTo( res ); UT_EQ ( "", res ); UT_TRUE( !tknzr2.HasNext() ); UT_TRUE( tknzr.HasNext() ); tknzr2.Set( tknzr.Next(), ','); tknzr2.Next().CopyTo( res ); UT_EQ ( "", res ); UT_TRUE( tknzr2.HasNext() ); tknzr2.Next().CopyTo( res ); UT_EQ ( "", res ); UT_TRUE( !tknzr2.HasNext() ); UT_TRUE( !tknzr.HasNext() ); } }
//-------------------------------------------------------------------------------------------------- //--- Test Tokenizer //-------------------------------------------------------------------------------------------------- void tokenizerTest( string inputString, AString res, char delim, char newDelim, Whitespaces trim, int inpStart= -1, int inpEnd= -1 ) { Substring inp= new Substring( inputString ); if ( inpStart >= 0 ) inp.Start= inpStart; if ( inpEnd >= 0 ) inp.End= inpEnd; res.Clear(); Tokenizer tknzr= new Tokenizer( inp, delim ); while( tknzr.HasNext() ) { tknzr.Next(trim).CopyTo( res, true ); res._( newDelim ); } }
/** **************************************************************************************** * Reads a prefix string from the ALib configuration system. * This internal method is used when a new domain is created, * * @param dom The domain to set the verbosity for. ******************************************************************************************/ protected void getDomainPrefixFromConfig( Domain dom ) { Variable variable= new Variable( ALox.PREFIXES, GetName() ); if( 0 == variable.Load() ) return; Tokenizer prefixTok= new Tokenizer(); Tokenizer prefixTokInner= new Tokenizer(); Substring domainStr= new Substring(); AString domainStrBuf= new AString(); Substring prefixStr= new Substring(); for( int varNo= 0; varNo< variable.Size(); varNo++ ) { prefixTok.Set( variable.GetString( varNo ), '=' ); domainStr.Set( prefixTok.Next() ); if ( domainStr.StartsWith( "INTERNAL_DOMAINS", DomainSensitivity ) ) { domainStrBuf._()._( domainStr.Buf, domainStr.Start + 16, domainStr.Length() -16 ); while ( domainStrBuf.CharAtStart() == '/' ) domainStrBuf.DeleteStart( 1 ); domainStrBuf.InsertAt( ALox.InternalDomains, 0 ); domainStr.Set( domainStrBuf ); } prefixTokInner.Set( prefixTok.Next(), ',' ); prefixStr.Set( prefixTokInner.Next() ); if ( prefixStr.IsEmpty() ) continue; if ( prefixStr.Consume( '\"' ) ) prefixStr.ConsumeFromEnd( '\"' ); Inclusion otherPLs= Inclusion.Include; prefixTokInner.Next(); if ( prefixTokInner.Actual.IsNotEmpty() ) otherPLs= ALIB.ReadInclusion( prefixTokInner.Actual ); int searchMode= 0; if ( domainStr.Consume ( '*' ) ) searchMode+= 2; if ( domainStr.ConsumeFromEnd( '*' ) ) searchMode+= 1; if( ( searchMode == 0 && dom.FullPath.Equals ( domainStr, DomainSensitivity ) ) || ( searchMode == 1 && dom.FullPath.StartsWith ( domainStr, DomainSensitivity ) ) || ( searchMode == 2 && dom.FullPath.EndsWith ( domainStr, DomainSensitivity ) ) || ( searchMode == 3 && dom.FullPath.IndexOf ( domainStr, 0, DomainSensitivity ) >=0 ) ) { dom.PrefixLogables.Add( new Domain.PL( new AString( prefixStr ), otherPLs ) ); // log info on this intMsg._()._NC( "String \"" )._NC( prefixStr )._NC ( "\" added as prefix logable for domain \'" ) ._NC( dom.FullPath ) ._NC( "\'. (Retrieved from variable" ) ._NC( variable.Fullname )._( ".)" ); logInternal( Verbosity.Info, "PFX", intMsg ); } } }
/** **************************************************************************************** * Reads the verbosity for the given logger and domain from the ALib configuration system. * This internal method is used in two occasions: * - when a new logger is added: recursively for all existing domains (\p configStr is * given) * - when a new domain is created on the fly(\p configStr is not given) * * @param logger The logger to set the verbosity for. * @param dom The domain to set the verbosity for. * @param variable The (already read) variable to set verbosities from. ******************************************************************************************/ protected void getVerbosityFromConfig( Logger logger, Domain dom, Variable variable ) { // get logger number. It may happen that the logger is not existent in this domain tree. int loggerNo= dom.GetLoggerNo( logger ) ; if ( loggerNo < 0 ) return; Tokenizer verbosityTknzr= new Tokenizer(); Substring domainStr= new Substring(); AString domainStrBuf= new AString(); for( int varNo= 0; varNo< variable.Size(); varNo++ ) { verbosityTknzr.Set( variable.GetString( varNo ), '=' ); domainStr.Set( verbosityTknzr.Next() ); if ( domainStr.StartsWith( "INTERNAL_DOMAINS", DomainSensitivity ) ) { domainStrBuf._()._( domainStr.Buf, domainStr.Start + 16, domainStr.Length() -16 ); while ( domainStrBuf.CharAtStart() == '/' ) domainStrBuf.DeleteStart( 1 ); domainStrBuf.InsertAt( ALox.InternalDomains, 0 ); domainStr.Set( domainStrBuf ); } Substring verbosityStr= verbosityTknzr.Next(); if ( verbosityStr.IsEmpty() ) continue; int searchMode= 0; if ( domainStr.Consume ( '*' ) ) searchMode+= 2; if ( domainStr.ConsumeFromEnd( '*' ) ) searchMode+= 1; if( ( searchMode == 0 && dom.FullPath.Equals ( domainStr, DomainSensitivity ) ) || ( searchMode == 1 && dom.FullPath.StartsWith ( domainStr, DomainSensitivity ) ) || ( searchMode == 2 && dom.FullPath.EndsWith ( domainStr, DomainSensitivity ) ) || ( searchMode == 3 && dom.FullPath.IndexOf ( domainStr, 0, DomainSensitivity ) >=0 ) ) { Verbosity verbosity= ALox.ReadVerbosity( verbosityStr); dom.SetVerbosity( loggerNo, verbosity, variable.Priority ); // log info on this intMsg._()._NC( "Logger \"" )._NC( logger.GetName() ) ._NC( "\":" ).Tab(11 + maxLoggerNameLength) ._( '\'' )._NC( dom.FullPath ) ._( '\'' ).InsertChars(' ', maxDomainPathLength - dom.FullPath.Length() + 1 ) ._("= Verbosity." ); ALox.ToString( verbosity, dom.GetPriority(loggerNo), intMsg ).TrimEnd() ._('.'); logInternal( Verbosity.Info, "LGR", intMsg ); } } }