//##################################################################################################
// SAMPLE code of class documentation
//##################################################################################################
void documentationSampleTokenizer()
{
UTSampleWriter utsw= new UTSampleWriter("DOC_SAMPLES_ALIB_LIB_STRING_TOKEN.txt");
//! [DOC_SAMPLES_ALIB_LIB_STRING_TOKEN]
// data string to tokenize
AString data= new AString( "test; abc ; 1,2 , 3 ; xyz ; including;separator" );
// create tokenizer on data with ';' as delimiter
Tokenizer tknzr= new Tokenizer( data, ';' );
// read tokens
System.Console.WriteLine( tknzr.Next() ); // will print "test"
System.Console.WriteLine( tknzr.Next() ); // will print "abc"
System.Console.WriteLine( tknzr.Next() ); // will print "1,2 , 3"
// tokenize actual (third) token (nested tokenizer)
Tokenizer subTknzr= new Tokenizer( tknzr.Actual, ',');
System.Console.Write( subTknzr.Next().ToString() );
while( subTknzr.HasNext() )
System.Console.Write( "~" + subTknzr.Next().ToString() );
System.Console.WriteLine();
// continue with the main tokenizer
System.Console.WriteLine( tknzr.Next().ToString() ); // will print "xyz"
// grab the rest, as we know that the last token might include our separator character
System.Console.WriteLine( tknzr.GetRest().ToString() ); // will print "including;separator"
//! [DOC_SAMPLES_ALIB_LIB_STRING_TOKEN]
utsw.FlushAndResetConsole();
}
// #############################################################################################
// logText
// #############################################################################################
/** ********************************************************************************************
*
* The implementation of the abstract method of parent class TextLogger. Logs messages to the
* application console and/or the VStudio output window.
*
* @param domain The <em>Log Domain</em>.
* @param verbosity The verbosity. This has been checked to be active already on this
* stage and is provided to be able to be logged out only.
* @param msg The log message
* @param scope Information about the scope of the <em>Log Statement</em>..
* @param lineNumber The line number of a multi-line message, starting with 0. For
* single line messages this is -1.
**********************************************************************************************/
override protected void logText( Domain domain, Verbosity verbosity,
AString msg,
ScopeInfo scope, int lineNumber)
{
// loop over message, print the parts between the escape sequences
Tokenizer msgParts= new Tokenizer( msg, '\x001B' );
Substring actual= msgParts.Actual;
Substring rest= msgParts.Rest;
int column= 0;
for(;;)
{
msgParts.Next( Whitespaces.Keep );
// check if this is an ANSI sequence already
if ( rest.CharAtStart() == '[' )
{
// read the 'm'
int idx= rest.IndexOf( 'm' );
if ( idx < 0 ) // unknown ANSI Code
{
ALIB.WARNING( "Unknown ANSI ESC Code " );
textWriter.Write( actual.Buf, actual.Start, actual.Length() );
continue;
}
column+= actual.Length();
actual.End= rest.Start + idx ;
rest.Start+= idx + 1;
textWriter.Write( actual.Buf, actual.Start, actual.Length() );
continue;
}
if ( actual.IsNotEmpty() )
{
textWriter.Write( actual.Buf, actual.Start, actual.Length() );
column+= actual.Length();
}
// end of loop?
if ( !msgParts.HasNext() )
break;
// found an ESC sequence
char c= rest.Consume();
// Colors
bool isForeGround= true;
if( c == 'C' || c == 'c' )
{
isForeGround= c== 'c';
c= rest.Consume();
int colNo= c - '0';
ALIB.ASSERT_WARNING( colNo >=0 && colNo <=9, "Unknown ESC-c code" );
// add bg
colNo+= isForeGround ? 0 : 10;
// add light
colNo+= (isForeGround ? !IsBackgroundLight : IsBackgroundLight ) ? 20 : 0;
textWriter.Write( ansiCols[ colNo ] );
}
// Styles
else if ( c == 's' )
{
// bold/italics style not supported in Windows console
// reset all
if ( rest.Consume() == 'a' )
{
textWriter.Write( ANSI_RESET );
}
}
// auto tab / end of meta
else if ( c == 't' || c == 'A')
{
bool endOfMeta= c == 'A';
c= rest.Consume();
int extraSpace= c >= '0' && c <= '9' ? (int) ( c - '0' )
: (int) ( c - 'A' ) + 10;
int tabStop= AutoSizes.Next( column, extraSpace );
Util.WriteSpaces( textWriter, tabStop - column );
column= tabStop;
if ( endOfMeta )
{
String msgPrefix;
switch ( verbosity )
{
case lox.Verbosity.Verbose: msgPrefix= MsgPrefixVerbose; break;
case lox.Verbosity.Info: msgPrefix= MsgPrefixInfo; break;
case lox.Verbosity.Warning: msgPrefix= MsgPrefixWarning; break;
case lox.Verbosity.Error: msgPrefix= MsgPrefixError; break;
default: msgPrefix= ""; break;
}
textWriter.Write( msgPrefix );
}
}
// Link (we just colorize links here)
else if ( c == 'l' )
{
textWriter.Write( rest.Consume() == 'S'
? ( IsBackgroundLight ? ANSI_LIGHT_BLUE : ANSI_LIGHT_BLUE )
: ANSI_STD_COL );
}
else
{
ALIB.WARNING( "Unknown ESC code" );
}
} // write loop
textWriter.WriteLine( MsgSuffix );
}
// #############################################################################################
// public interface
// #############################################################################################
/** ****************************************************************************************
* Constructs a scope info.
* @param name The name of the Lox we belong to.
* Will be converted to upper case.
* @param threadDictionary A dictionary to map thread IDs to user friendly names which is
* managed outside of this class.
******************************************************************************************/
public ScopeInfo( String name, Dictionary<int, String> threadDictionary )
{
loxName= name.ToUpper();
ALIB.ASSERT_ERROR( !loxName.Equals( "GLOBAL" ), "Name \"GLOBAL\" not allowed for Lox instances" );
this.threadDictionary= threadDictionary;
cache= new SourceFile[cacheSize= DefaultCacheSize];
for ( int i= 0; i< cacheSize; i++ )
cache[i]= new SourceFile();
actual= cache[0];
// read trim rules from config
// do 2 times, 0== local list, 1== global list
List<SourcePathTrimRule> trimInfoList;
for( int trimInfoNo= 0; trimInfoNo < 2 ; trimInfoNo++ )
{
// check if done... or set list
Variable variable= new Variable();
if ( trimInfoNo == 0 )
{
trimInfoList= LocalSPTRs;
variable.Define(ALox.SPTR_LOX, loxName).Load();
}
else
{
if ( GlobalSPTRsReadFromConfig )
continue;
GlobalSPTRsReadFromConfig= true;
trimInfoList= GlobalSPTRs;
variable.Define(ALox.SPTR_GLOBAL).Load();
}
if( variable.Priority != 0 )
{
Tokenizer ruleTok = new Tokenizer();
for( int ruleNo= 0; ruleNo< variable.Size(); ruleNo++ )
{
try {
ruleTok.Set( variable.GetString( ruleNo ), ',' );
SourcePathTrimRule rule= new SourcePathTrimRule();
rule.Priority= variable.Priority;
rule.Path= new AString();
ruleTok.Next();
if( ! ( rule.IsPrefix= !ruleTok.Actual.StartsWith( "*" ) ) )
ruleTok.Actual.Consume(1);
rule.Path._( ruleTok.Actual );
if ( rule.Path.CharAtEnd() == '*' )
rule.Path.DeleteEnd( 1 );
if ( rule.Path.IsEmpty() )
continue;
if( Path.DirectorySeparatorChar == '/' )
rule.Path.SearchAndReplaceAll( "\\", "/" );
else
rule.Path.SearchAndReplaceAll( "/" , "\\" );
rule.IncludeString = ALIB.ReadInclusion( ruleTok.Next() );
if ( ruleTok.HasNext () )
ruleTok.Next().ConsumeInteger( out rule.TrimOffset );
rule.Sensitivity = ALIB.ReadCase( ruleTok.Next() );
if ( ruleTok.HasNext () )
rule.TrimReplacement= ruleTok.Next().ToString();
trimInfoList.Add( rule );
}
catch( Exception )
{
ALIB.ERROR( "Error reading source path trim rule from configuration. Invalid String: "
+ variable.GetString( ruleNo ).ToString() );
}
}
}
}
}
/** ********************************************************************************************
*
* The implementation of the abstract method of parent class TextLogger. Logs messages to the
* application console and/or the VStudio output window.
*
* @param domain The <em>Log Domain</em>.
* @param verbosity The verbosity. This has been checked to be active already on this
* stage and is provided to be able to be logged out only.
* @param msg The log message.
* @param scope Information about the scope of the <em>Log Statement</em>..
* @param lineNumber The line number of a multi-line message, starting with 0. For
* single line messages this is -1.
**********************************************************************************************/
override protected void logText( Domain domain, Verbosity verbosity,
AString msg,
ScopeInfo scope, int lineNumber)
{
// get actual console attributes
ConsoleColor actualFGCol= Console.ForegroundColor;
ConsoleColor actualBGCol= Console.BackgroundColor;
// loop over message, print the parts between the escape sequences
Tokenizer msgParts= new Tokenizer( msg, '\x1B' );
Substring actual= msgParts.Actual;
Substring rest= msgParts.Rest;
int column= 0;
for(;;)
{
if ( msgParts.Next(Whitespaces.Keep).IsNotEmpty() )
{
#if !(ALOX_WP71 || ALOX_WP8)
Console.Write( msg.Buffer(), actual.Start, actual.Length() );
#else
Console.Write( msg.ToString( 0, actual.Start, actual.Length() );
#endif
column+= actual.Length();
}
// end of loop?
if ( !msgParts.HasNext() )
break;
// found an ESC sequence
char c= rest.Consume();
// Colors
bool isForeGround= true;
if( c == 'C' || c == 'c' )
{
isForeGround= c== 'c';
c= rest.Consume();
int colNo= c - '0';
ALIB.ASSERT_WARNING( colNo >=0 && colNo <=9, "Unknown ESC-c code" );
// set color
if( colNo >= 0 && colNo <= 8 || colNo == 8)
{
ConsoleColor[] cols= (isForeGround ? !IsBackgroundLight : IsBackgroundLight ) ? lightColors : darkColors;
if ( isForeGround )
Console.ForegroundColor= cols[ colNo ];
else
Console.BackgroundColor= cols[ colNo ];
}
else if ( colNo == 9 )
{
if ( isForeGround )
Console.ForegroundColor= actualFGCol;
else
Console.BackgroundColor= actualBGCol;
}
else
{
ALIB.WARNING( "Unknown ESC- code" );
}
}
// Styles
else if ( c == 's' )
{
// bold/italics style not supported in Windows console
// reset all
if ( rest.Consume() == 'a' )
{
Console.ForegroundColor= actualFGCol;
Console.BackgroundColor= actualBGCol;
}
}
// auto tab / end of meta
else if ( c == 't' || c == 'A')
{
bool endOfMeta= c == 'A';
c= rest.Consume();
int extraSpace= c >= '0' && c <= '9' ? (int) ( c - '0' )
: (int) ( c - 'A' ) + 10;
int tabStop= AutoSizes.Next( column, extraSpace );
Util.WriteSpaces( Console.Out, tabStop - column );
column= tabStop;
if ( endOfMeta )
{
switch ( verbosity )
{
case Verbosity.Verbose: Console.ForegroundColor= MsgColorVerbose; break;
case Verbosity.Info: Console.ForegroundColor= MsgColorInfo; break;
case Verbosity.Warning: Console.ForegroundColor= MsgColorWarning; break;
case Verbosity.Error: Console.ForegroundColor= MsgColorError; break;
default: break;
}
}
}
// Link (we just colorize links here)
else if ( c == 'l' )
{
if ( rest.Consume() == 'S' )
Console.ForegroundColor= IsBackgroundLight ? ConsoleColor.DarkBlue : ConsoleColor.Blue;
else
Console.ForegroundColor= actualFGCol;
}
else
{
ALIB.WARNING( "Unknown ESC code" );
}
} // write loop
// reset colors
Console.ForegroundColor= actualFGCol;
Console.BackgroundColor= actualBGCol;
// write NL
#if !(ALOX_WP71 || ALOX_WP8)
Console.WriteLine();
#else
Console.WriteLine();
#endif
}
public void Tokenize()
{
AString astr= new AString();
AString res= new AString();
// tokenizing empty string
astr.Clear()._( "" );
res.Clear();
{
Tokenizer tknzr= new Tokenizer( astr, ',' ); UT_EQ( true, tknzr.HasNext() );
tknzr.Next().CopyTo( res, true ); UT_EQ( "", res );
}
// tokenizing no delim
astr.Clear()._( "abc" );
res.Clear();
{
Tokenizer tknzr= new Tokenizer( astr, ',' ); UT_EQ( true, tknzr.HasNext() );
tknzr.Next().CopyTo( res, true ); UT_EQ( "abc", res );
}
// tokenizing
{
tokenizerTest( "abc", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "abc@", res );
tokenizerTest( "a,bc", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "a@bc@", res );
tokenizerTest( "a,bc", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "a@bc@", res );
tokenizerTest( ",", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "@@", res );
tokenizerTest( ",,", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "@@@", res );
tokenizerTest( "a,b,c,,", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "a@b@c@@@", res );
tokenizerTest( "a,b,c", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "a@b@c@", res );
tokenizerTest( ",a,b,c", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "@a@b@c@", res );
tokenizerTest( "123567", res, ',', '@', Whitespaces.Trim, 2, 2 );
UT_EQ ( "3@", res );
tokenizerTest( "123567", res, ',', '@', Whitespaces.Trim, 2, 1 );
UT_EQ ( "@", res );
tokenizerTest( " abc , def , ghe ", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "abc@def@ghe@", res );
tokenizerTest( "abc , def,ghe,", res, ',', '@', Whitespaces.Trim, -1, -1 );
UT_EQ ( "abc@def@ghe@@", res );
tokenizerTest( " abc , def , ghe ", res, ',', '@', Whitespaces.Keep, -1, -1 );
UT_EQ ( " abc @ def @ ghe @", res );
tokenizerTest( "abc , def,ghe,", res, ',', '@', Whitespaces.Keep, -1, -1 );
UT_EQ ( "abc @ def@ghe@@", res );
}
// tokenizing with different delimiters
{
astr.Clear()._( "1,5;3@4" );
Tokenizer tknzr= new Tokenizer(astr, ',');
tknzr.Next() .CopyTo( res ); UT_EQ ( "1", res ); UT_EQ( true, tknzr.HasNext() );
tknzr.Next( Whitespaces.Trim, ';' ).CopyTo( res ); UT_EQ ( "5", res ); UT_EQ( true, tknzr.HasNext() );
tknzr.Next( Whitespaces.Trim, '@' ).CopyTo( res ); UT_EQ ( "3", res ); UT_EQ( true, tknzr.HasNext() );
tknzr.Next( Whitespaces.Trim, '-' ).CopyTo( res ); UT_EQ ( "4", res ); UT_EQ( false, tknzr.HasNext() );
}
// tokenizing with different delimiters
{
astr.Clear()._( "abc, 5;\t3;;; 4 " );
Tokenizer tknzr= new Tokenizer(astr,',');
tknzr.Next() .CopyTo( res ); UT_EQ ( "abc", res ); UT_EQ( true, tknzr.HasNext() );
tknzr.Next( Whitespaces.Trim, ';' ).CopyTo( res ); UT_EQ ( "5", res ); UT_EQ( true, tknzr.HasNext() );
tknzr.Next() .CopyTo( res ); UT_EQ ( "3", res ); UT_EQ( true, tknzr.HasNext() );
tknzr.Next() .CopyTo( res ); UT_EQ ( "", res ); UT_EQ( true, tknzr.HasNext() );
tknzr.GetRest() .CopyTo( res ); UT_EQ ( "; 4", res ); UT_EQ( false, tknzr.HasNext() );
}
// sub-tokens
{
astr.Clear()._( "1,2;3 , 4;5,;," );
Tokenizer tknzr= new Tokenizer(astr, ';');
Tokenizer tknzr2= new Tokenizer( tknzr.Next(), ',');
tknzr2.Next().CopyTo( res ); UT_EQ ( "1", res ); UT_TRUE( tknzr2.HasNext() );
tknzr2.Next().CopyTo( res ); UT_EQ ( "2", res ); UT_TRUE( !tknzr2.HasNext() );
UT_TRUE( tknzr.HasNext() );
tknzr2.Set( tknzr.Next(), ',');
tknzr2.Next().CopyTo( res ); UT_EQ ( "3", res ); UT_TRUE( tknzr2.HasNext() );
tknzr2.Next().CopyTo( res ); UT_EQ ( "4", res ); UT_TRUE( !tknzr2.HasNext() );
UT_TRUE( tknzr.HasNext() );
tknzr2.Set( tknzr.Next(), ',');
tknzr2.Next().CopyTo( res ); UT_EQ ( "5", res ); UT_TRUE( tknzr2.HasNext() );
tknzr2.Next().CopyTo( res ); UT_EQ ( "", res ); UT_TRUE( !tknzr2.HasNext() );
UT_TRUE( tknzr.HasNext() );
tknzr2.Set( tknzr.Next(), ',');
tknzr2.Next().CopyTo( res ); UT_EQ ( "", res ); UT_TRUE( tknzr2.HasNext() );
tknzr2.Next().CopyTo( res ); UT_EQ ( "", res ); UT_TRUE( !tknzr2.HasNext() );
UT_TRUE( !tknzr.HasNext() );
}
}
//--------------------------------------------------------------------------------------------------
//--- Test Tokenizer
//--------------------------------------------------------------------------------------------------
void tokenizerTest( string inputString, AString res, char delim, char newDelim,
Whitespaces trim, int inpStart= -1, int inpEnd= -1 )
{
Substring inp= new Substring( inputString );
if ( inpStart >= 0 ) inp.Start= inpStart;
if ( inpEnd >= 0 ) inp.End= inpEnd;
res.Clear();
Tokenizer tknzr= new Tokenizer( inp, delim );
while( tknzr.HasNext() )
{
tknzr.Next(trim).CopyTo( res, true );
res._( newDelim );
}
}