public AssertTokenizer /*!*/ Load(object /*!*/ source) // source: byte[] or string
{
_tests.Assert(_log.Errors.Count == 0, "Previous test case reported unexpected error/warning(s)");
SourceUnit sourceUnit;
RubyEncoding encoding;
byte[] binarySource = source as byte[];
if (binarySource != null)
{
encoding = RubyEncoding.Binary;
sourceUnit = _context.CreateSourceUnit(new BinaryContentProvider(binarySource), null, encoding.Encoding, SourceCodeKind.File);
}
else
{
encoding = DefaultEncoding;
sourceUnit = _context.CreateSnippet((string)source, SourceCodeKind.File);
}
_tokenizer = new Tokenizer(false, DummyVariableResolver.AllMethodNames)
{
ErrorSink = _log,
Compatibility = Compatibility,
Encoding = encoding,
Verbatim = Verbatim,
};
_tokenizer.Initialize(sourceUnit);
_allTokens = new List <Tokens>();
_allValues = new List <object>();
return(this);
}
public AssertTokenizer /*!*/ Load(byte[] /*!*/ source)
{
Tests.Assert(_log.Errors.Count == 0, "Previous test case reported unexpected error/warning(s)");
_tokenizer = new Tokenizer(false, _log);
_tokenizer.Compatibility = _context.RubyOptions.Compatibility;
_tokenizer.Initialize(_context.CreateSourceUnit(
new BinaryContentProvider(source), null, BinaryEncoding.Instance, SourceCodeKind.File)
);
_allTokens = new List <Tokens>();
_allValues = new List <object>();
return(this);
}
private void Benchmark(List <string> /*!*/ files)
{
var sources = new List <SourceUnit>();
Stopwatch readTime = new Stopwatch();
long totalSize = 0;
readTime.Start();
foreach (string path in files)
{
try {
byte[] data = File.ReadAllBytes(path);
sources.Add(_context.CreateSourceUnit(new BinaryContentProvider(data), path, Encoding.Default, SourceCodeKind.File));
totalSize += data.Length;
} catch (Exception) {
Console.WriteLine("Error: {0}", path);
}
}
readTime.Stop();
Console.WriteLine("Read: {0} kB in {1}", totalSize / 1024, readTime.Elapsed);
#if F
Stopwatch tokenizeTime = new Stopwatch();
tokenizeTime.Start();
foreach (var source in sources)
{
try {
var tokenizer = new Tokenizer();
tokenizer.Initialize(source);
Tokens token;
do
{
token = tokenizer.GetNextToken();
} while (token != Tokens.EndOfFile);
} catch (Exception) {
Console.WriteLine("Tokenization error: {0}", source.Path);
break;
}
}
tokenizeTime.Stop();
#endif
//var stackSizes = new Dictionary<int, int>();
var options = new RubyCompilerOptions();
Stopwatch parseTime = new Stopwatch();
Stopwatch transformTime = new Stopwatch();
foreach (var source in sources)
{
try {
parseTime.Start();
var parser = new Parser();
parser.Parse(source, options, ErrorSink.Null);
//int mt;
//stackSizes[parser.StackMaxTop] = stackSizes.TryGetValue(parser.StackMaxTop, out mt) ? mt + 1 : 1;
parseTime.Stop();
#if F
if (rubyTree != null)
{
transformTime.Start();
var lambda = _context.TransformTree <DlrMainCallTarget>(rubyTree, source, options);
transformTime.Stop();
}
else
{
Console.WriteLine("SyntaxError: {0}", source.Path);
}
#endif
} catch (Exception e) {
Console.WriteLine("{0}: {1}: {2}", e.GetType().Name, source.Path, e.Message);
break;
}
}
// Console.WriteLine("Tokenize: {0}", tokenizeTime.Elapsed);
Console.WriteLine("Parse: {0}", parseTime.Elapsed);
//Console.WriteLine("Idf/Kwd/Loc: {0}/{1}/{2}", Tokenizer.IdfLength, Tokenizer.KwdLength, Tokenizer.LocLength);
// Console.WriteLine("Transform: {0}", transformTime.Elapsed);
//PerfTrack.DumpHistogram(Parser.Reductions);
//PerfTrack.DumpHistogram(stackSizes);
}
