//----< constructor >------------------------------------------------
public CSemiExp()
{
toker = new CToker();
semiExp = new List<string>();
discardComments = true; // not implemented yet
returnNewLines = true;
displayNewLines = false;
}
static void Main(string[] args)
{
Console.Write("\n Testing CToker - Tokenizer ");
Console.Write("\n ============================\n");
/* If the package is run as stand alone application
* then add the default values for the member variables
*/
try
{
CToker toker = new CToker();
if (args.Length == 0)
{
Console.Write("\n Please enter name of file to tokenize\n\n");
return;
}
foreach (string file in args)
{
string msg1;
if (!toker.openFile(file))
{
msg1 = "Can't open file " + file;
Console.Write("\n\n {0}", msg1);
Console.Write("\n {0}", new string('-', msg1.Length));
}
else
{
msg1 = "Processing file " + file;
Console.Write("\n\n {0}", msg1);
Console.Write("\n {0}", new string('-', msg1.Length));
string tok = "";
while ((tok = toker.getTok()) != "")
{
if (tok != "\n")
{
Console.Write("\n{0}", tok);
}
}
toker.close();
}
}
Console.Write("\n");
}
catch (Exception ex)
{
Console.Write("\n\n token \"{0}\" has embedded newline\n\n", ex.Message);
}
Console.Read();
}
static void Main(string[] args)
{
Console.Write("\n Testing CToker - Tokenizer ");
Console.Write("\n ============================\n");
try
{
CToker toker = new CToker();
if (args.Length == 0)
{
Console.Write("\n Please enter name of file to tokenize\n\n");
return;
}
foreach (string file in args)
{
string msg1;
if (!toker.openFile(file))
{
msg1 = "Can't open file " + file;
Console.Write("\n\n {0}", msg1);
Console.Write("\n {0}", new string('-', msg1.Length));
}
else
{
msg1 = "Processing file " + file;
Console.Write("\n\n {0}", msg1);
Console.Write("\n {0}", new string('-', msg1.Length));
string tok = "";
while ((tok = toker.getTok()) != "")
{
if (tok != "\n")
{
Console.Write("\n{0}", tok);
}
}
toker.close();
}
}
Console.Write("\n");
//
string[] msgs = new string[12];
msgs[0] = "abc";
msgs[11] = "-- \"abc def\" --";
msgs[1] = "string with double quotes \"first quote\""
+ " and \"second quote\" but no more";
msgs[2] = "string with single quotes \'1\' and \'2\'";
msgs[3] = "string with quotes \"first quote\" and \'2\'";
msgs[4] = "string with C comments /* first */ and /*second*/ but no more";
msgs[10] = @"string with @ \\stuff";
msgs[5] = "/* single C comment */";
msgs[6] = " -- /* another single comment */ --";
msgs[7] = "// a C++ comment\n";
msgs[8] = "// another C++ comment\n";
msgs[9] = "/*\n *\n *\n */";
foreach (string msg in msgs)
{
if (!toker.openString(msg))
{
string msg2 = "Can't open string for reading";
Console.Write("\n\n {0}", msg2);
Console.Write("\n {0}", new string('-', msg2.Length));
}
else
{
string msg2 = "Processing \"" + msg + "\"";
Console.Write("\n\n {0}", msg2);
Console.Write("\n {0}", new string('-', msg2.Length));
string tok = "";
while ((tok = toker.getTok()) != "")
{
if (tok != "\n")
{
Console.Write("\n{0}", tok);
}
else
{
Console.Write("\nnewline");
}
}
toker.close();
}
}
Console.Write("\n\n");
}
catch (Exception ex)
{
Console.Write("\n\n token \"{0}\" has embedded newline\n\n", ex.Message);
}
}
static void Main(string[] args)
{
Console.Write("\n Testing CToker - Tokenizer ");
Console.Write("\n ============================\n");
try
{
CToker toker = new CToker();
if (args.Length == 0)
{
Console.Write("\n Please enter name of file to tokenize\n\n");
return;
}
foreach (string file in args)
{
string msg1;
if (!toker.openFile(file))
{
msg1 = "Can't open file " + file;
Console.Write("\n\n {0}", msg1);
Console.Write("\n {0}", new string('-', msg1.Length));
}
else
{
msg1 = "Processing file " + file;
Console.Write("\n\n {0}", msg1);
Console.Write("\n {0}", new string('-', msg1.Length));
string tok = "";
while ((tok = toker.getTok()) != "")
if (tok != "\n")
Console.Write("\n{0}", tok);
toker.close();
}
}
Console.Write("\n");
//
string[] msgs = new string[12];
msgs[0] = "abc";
msgs[11] = "-- \"abc def\" --";
msgs[1] = "string with double quotes \"first quote\""
+ " and \"second quote\" but no more";
msgs[2] = "string with single quotes \'1\' and \'2\'";
msgs[3] = "string with quotes \"first quote\" and \'2\'";
msgs[4] = "string with C comments /* first */ and /*second*/ but no more";
msgs[10] = @"string with @ \\stuff";
msgs[5] = "/* single C comment */";
msgs[6] = " -- /* another single comment */ --";
msgs[7] = "// a C++ comment\n";
msgs[8] = "// another C++ comment\n";
msgs[9] = "/*\n *\n *\n */";
foreach (string msg in msgs)
{
if (!toker.openString(msg))
{
string msg2 = "Can't open string for reading";
Console.Write("\n\n {0}", msg2);
Console.Write("\n {0}", new string('-', msg2.Length));
}
else
{
string msg2 = "Processing \"" + msg + "\"";
Console.Write("\n\n {0}", msg2);
Console.Write("\n {0}", new string('-', msg2.Length));
string tok = "";
while ((tok = toker.getTok()) != "")
{
if (tok != "\n")
Console.Write("\n{0}", tok);
else
Console.Write("\nnewline");
}
toker.close();
}
}
Console.Write("\n\n");
}
catch (Exception ex)
{
Console.Write("\n\n token \"{0}\" has embedded newline\n\n", ex.Message);
}
}
static void Main(string[] args)
{
Console.Write("\n Testing CToker - Tokenizer ");
Console.Write("\n ============================\n");
CToker toker = new CToker();
// comment next line if you want to see comments in tokenized files
//toker.returnComments(false);
if(args.Length == 0)
{
Console.Write("\n Please enter name of file to tokenize\n\n");
return;
}
foreach(string file in args)
{
string msg1;
if(!toker.openFile(file))
{
msg1 = "Can't open file " + file;
Console.Write("\n\n {0}",msg1);
Console.Write("\n {0}",new string('-',msg1.Length));
}
else
{
msg1 = "Processing file " + file;
Console.Write("\n\n {0}",msg1);
Console.Write("\n {0}",new string('-',msg1.Length));
string tok = "";
while((tok = toker.getTok()) != "")
if(tok != "\n")
Console.Write("\n{0}",tok);
toker.close();
}
}
Console.Write("\n");
toker.returnComments(true);
string[] msgs = new string[9];
msgs[0] = "abc";
msgs[1] = "string with double quotes \"first quote\""
+ " and \"second quote\" but no more";
msgs[2] = "string with single quotes \'1\' and \'2\'";
msgs[3] = "string with quotes \"first quote\" and \'2\'";
msgs[4] = "string with C comments /* first */ and /*second*/ but no more";
msgs[5] = "/* single C comment */";
msgs[6] = " -- /* another single comment */ --";
msgs[7] = "// a C++ comment\n";
msgs[8] = "// another C++ comment\n";
foreach(string msg in msgs)
{
if(!toker.openString(msg))
{
string msg2 = "Can't open string for reading";
Console.Write("\n\n {0}",msg2);
Console.Write("\n {0}",new string('-',msg2.Length));
}
else
{
string msg2 = "Processing \"" + msg + "\"";
Console.Write("\n\n {0}",msg2);
Console.Write("\n {0}",new string('-',msg2.Length));
string tok = "";
while((tok = toker.getTok()) != "")
{
if(tok != "\n")
Console.Write("\n{0}",tok);
else
Console.Write("\nnewline");
}
toker.close();
}
}
Console.Write("\n\n");
}
//----< constructor >------------------------------------------------
public CSemiExp()
{
toker = new CToker();
if (!returnComments_)
toker.returnComments(false);
semiExp = new ArrayList();
}
static void Main(string[] args)
{
Console.Write("\n Testing CToker - Tokenizer ");
Console.Write("\n ============================\n");
try
{
CToker toker = new CToker();
if (args.Length == 0)
{
Console.Write("\n Please enter name of file to tokenize\n\n");
return;
}
foreach (string file in args)
{
string msg1;
if (!toker.openFile(file))
{
msg1 = "Can't open file " + file;
Console.Write("\n\n {0}", msg1);
Console.Write("\n {0}", new string('-', msg1.Length));
}
else
{
msg1 = "Processing file " + file;
Console.Write("\n\n {0}", msg1);
Console.Write("\n {0}", new string('-', msg1.Length));
string tok = "";
while ((tok = toker.getTok()) != "")
if (tok != "\n")
Console.Write("\n{0}", tok);
toker.close();
}
}
Console.Write("\n\n");
}
catch (Exception ex) { Console.Write("\n\n token \"{0}\" has embedded newline\n\n", ex.Message); }
Console.Read();
}
