/// <summary>
/// Use the supplied tokenizer to tokenize the specified stream
/// and time it.
/// </summary>
/// <param name="tokenizer"></param>
/// <param name="stream"></param>
/// <returns>Total milliseconds per parse.</returns>
protected static double SpeedTestParse(StreamTokenizer tokenizer,
Stream stream)
{
GC.Collect();
List<Token> tokens = new List<Token>();
DateTime start = System.DateTime.UtcNow;
int cycles = 100;
for (int i = 0; i < cycles; i++)
{
tokenizer.TokenizeStream(stream, tokens);
stream.Position = 0;
}
TimeSpan duration = System.DateTime.UtcNow - start;
return (duration.TotalMilliseconds / cycles);
}
/// <summary>
/// Speed test. This tests the speed of the parse.
/// </summary>
/// <returns>bool - true for ran, false for failed to run.</returns>
public static bool SpeedTest()
{
Logger log = new Logger("SpeedTest");
log.Verbosity = VerbosityLevel.Debug;
log.Info("Starting...");
Random rand = new Random(0);
// setup tokenizer
StreamTokenizer tokenizer = new StreamTokenizer();
tokenizer.Settings.ParseNumbers = true;
int nTokens = 1024;
MemoryStream ms;
StreamWriter writer;
// int
ms = new MemoryStream();
writer = new StreamWriter(ms);
for (int i = 0; i < nTokens; i++)
{
writer.WriteLine("{0}", (int)(rand.NextDouble() * 256));
}
writer.Flush();
ms.Position = 0;
Console.WriteLine("Parse {0} integers took {1:f2} ms", nTokens,
SpeedTestParse(tokenizer, ms));
// float
ms = new MemoryStream();
writer = new StreamWriter(ms);
ms.Position = 0;
for (int i = 0; i < nTokens; i++)
{
writer.WriteLine("{0:f9}", rand.NextDouble() * 10);
}
writer.Flush();
ms.Position = 0;
Console.WriteLine("Parse {0} floats took {1:f2} ms", nTokens,
SpeedTestParse(tokenizer, ms));
// exponential
ms = new MemoryStream();
writer = new StreamWriter(ms);
ms.Position = 0;
for (int i = 0; i < nTokens; i++)
{
writer.WriteLine("{0:e9}", rand.NextDouble() * 1000);
}
writer.Flush();
ms.Position = 0;
Console.WriteLine("Parse {0} exponential floats took {1:f2} ms", nTokens,
SpeedTestParse(tokenizer, ms));
// words
ms = new MemoryStream();
writer = new StreamWriter(ms);
for (int i = 0; i < nTokens; i++)
{
writer.WriteLine("foo ");
}
writer.Flush();
ms.Position = 0;
Console.WriteLine("Parse {0} words took {1:f2} ms", nTokens,
SpeedTestParse(tokenizer, ms));
// hex
ms = new MemoryStream();
writer = new StreamWriter(ms);
for (int i = 0; i < nTokens; i++)
{
writer.WriteLine("0x{0:x}", (int)(rand.NextDouble() * 256));
}
writer.Flush();
ms.Position = 0;
Console.WriteLine("Parse {0} hex numbers took {1:f2} ms", nTokens,
SpeedTestParse(tokenizer, ms));
// Console.WriteLine("Buffer to parse is:");
// Console.WriteLine("{0}", Encoding.ASCII.GetString(ms.GetBuffer()));
return (true);
}
// ---------------------------------------------------------------------
#region TestSelf
// ---------------------------------------------------------------------
/// <summary>
/// Simple self test. See StreamTokenizerTestCase for full
/// tests.
/// </summary>
/// <returns>bool - true for success, false for failure.</returns>
public static bool TestSelf()
{
Logger log = new Logger("testSelf");
log.Verbosity = VerbosityLevel.Debug;
log.Info("Starting...");
string testString;
List<Token> tokens;
// setup tokenizer
StreamTokenizer tokenizer = new StreamTokenizer();
tokenizer.Settings.SetupForCodeParse();
tokenizer.Verbosity = VerbosityLevel.Debug;
//
// try string parse
//
log.Write("--------------------------------------------------------\n");
log.Info("string parse:");
log.Write("--------------------------------------------------------\n");
tokens = new List<Token>();
testString = "-1.2ej";
tokenizer.Settings.DoUntermCheck = false;
tokenizer.Settings.GrabWhitespace = false;
if (!tokenizer.TokenizeString(testString, tokens))
{
log.Error("Unable to parse into token vector.");
return (false);
}
foreach (Token t in tokens) log.Info("Token = '{0}'", t.ToString());
tokens = new List<Token>();
//
// try NextToken style
//
// log.Write("--------------------------------------------------------\n");
// log.Info("NextToken use");
// log.Write("--------------------------------------------------------\n");
//string fileName = "st-testSelf.tmp";
//testString = "this is a simple string";
//tokenizer.TextReader = new StringReader(testString);
//tokenizer.TextReader = File.OpenText(fileName);
//Token token;
//while (tokenizer.NextToken(out token)) log.Info("Token = '{0}'", token);
//
// try TokenizeFile
//
log.Write("--------------------------------------------------------\n");
log.Info("Tokenize missing file");
log.Write("--------------------------------------------------------\n");
string nonExistentFileName = "ThisFile better not exist";
bool caughtIt = false;
try
{
tokenizer.TokenizeFile(nonExistentFileName);
}
catch (FileNotFoundException e)
{
log.Info("Correctly caught exception: {0}: {1}", e.GetType().ToString(), e.Message);
caughtIt = true;
}
if (!caughtIt)
{
log.Error("Didn't get a file not found exception from TokenizeFile.");
return (false);
}
//
// test line numbers in tokens
//
// done
log.Info("Done.");
return (true);
}
