public void TestComments(string str) { ITokenizer tokenizer = new SimpleTokenizer(); var tokens = tokenizer.EnumerateTokens(str.AsSpan()); Assert.Equal(0, tokens.Length); }
public void TestVariableLengthUnicodeEscapeSequencesWork() { Span <char> code = stackalloc char[] { '\'', '\\', 'x', '0', '0', '0', '0', '\'' }; ReadOnlySpan <char> format = "X"; Span <char> innerStorage = code.Slice(3); for (int i = 0; i < char.MaxValue; i++) { ITokenizer tokenizer = new SimpleTokenizer(); i.TryFormat(innerStorage, out var charsWritten, format); Span <char> codeStorage = code.Slice(0, charsWritten + 4); codeStorage[codeStorage.Length - 1] = '\''; var tokens = tokenizer.EnumerateTokens(codeStorage); Assert.Equal(1, tokens.Length); var charConstToken = Assert.IsType <CharacterConstantToken>(tokens[0]); Assert.Equal((char)i, charConstToken.Value); } }
public void TestCommentsNewLine(string str) { ITokenizer tokenizer = new SimpleTokenizer(); var tokens = tokenizer.EnumerateTokens(str.AsSpan()); Assert.Equal(1, tokens.Length); Assert.IsType <IntegerConstantToken>(tokens[0]); }
public void TestTokenizerDoubleCharTokenCaseWithSpaceAfterFirstChar(string str, Type tokenType) { ITokenizer tokenizer = new SimpleTokenizer(); var tokens = tokenizer.EnumerateTokens(str.AsSpan()); Assert.Equal(1, tokens.Length); Assert.IsType(tokenType, tokens[0]); }
public void TestAliasedKeywordsParsing(string alias) { var tokenizer = new SimpleTokenizer(); var tokens = tokenizer.EnumerateTokens(alias); Assert.Equal(1, tokens.Length); var aliasedToken = Assert.IsType <AliasedIdentifierToken>(tokens[0]); }
public void TestTokenizerDoubleCharTokensWithSpace(string str, Type firstToken, Type secondToken) { ITokenizer tokenizer = new SimpleTokenizer(); var tokens = tokenizer.EnumerateTokens(str.AsSpan()); Assert.Equal(2, tokens.Length); Assert.IsType(firstToken, tokens[0]); Assert.IsType(secondToken, tokens[1]); }
public void TestDoubleParsing(double value) { var input = value.ToString(); ITokenizer tokenizer = new SimpleTokenizer(); var tokens = tokenizer.EnumerateTokens(input.AsSpan()); Assert.Equal(1, tokens.Length); var dctoken = Assert.IsType <DoubleConstantToken>(tokens[0]); Assert.Equal(value, dctoken.Value, 5); }
public void TestCustomIdentifierArray(string str) { str = str + "[]"; var tokenizer = new SimpleTokenizer(); var tokens = tokenizer.EnumerateTokens(str); Assert.Equal(1, tokens.Length); var idToken = Assert.IsType <IdentifierToken>(tokens[0]); Assert.Equal(str, idToken.Name); }
public void TestAliasedKeywordsArrayParsing(string alias) { var tokenizer = new SimpleTokenizer(); if (alias == "void") { return; // skip void } var tokens = tokenizer.EnumerateTokens(alias + "[]"); Assert.Equal(1, tokens.Length); var aliasedToken = Assert.IsType <AliasedIdentifierToken>(tokens[0]); }
public void TestKeywordsParseAsSpecialIdentifiers() { var keywordClasses = typeof(IKeywordToken).Assembly.GetTypes().Where(x => x.GetInterfaces().Contains(typeof(IKeywordToken))); foreach (var tokenClass in keywordClasses) { var tokenCharValueField = tokenClass.GetField("KeywordValue", BindingFlags.Public | BindingFlags.Static); var keyword = (string)tokenCharValueField.GetValue(null); var tokenizer = new SimpleTokenizer(); var tokens = tokenizer.EnumerateTokens(keyword); Assert.Equal(1, tokens.Length); Assert.IsType(tokenClass, tokens[0]); } }
static void Main(string?programName = null, string[]?args = null) { var tracer = new Tracer(); tracer.Restart(); if (args == null) { throw new InvalidOperationException("You must pass in a file"); } var libraries = args.Where(x => { var ext = Path.GetExtension(x); return(ext == ".exe" || ext == ".dll"); }).ToArray(); args = args.Except(libraries).ToArray(); if (args.Length == 0) { throw new InvalidOperationException("You must pass in a file to actually compile"); } #pragma warning disable CS8619 // Nullability of reference types in value doesn't match target type. Assembly[] assemblies = libraries.Select(x => { try { return(Assembly.LoadFrom(x)); } #pragma warning disable CA1031 // Do not catch general exception types catch #pragma warning restore CA1031 // Do not catch general exception types { return(null); } }).Where(x => x != null).Append(typeof(object).Assembly).ToArray(); #pragma warning restore CS8619 // Nullability of reference types in value doesn't match target type. if (programName == null) { programName = Path.GetFileNameWithoutExtension(args[0]); } tracer.AddEpoch("Initialization"); var tokenizer = new SimpleTokenizer(); var parser = new SimpleParser(); var rootNodes = new List <ImmutableRootSyntaxNode>(); foreach (var file in args) { var code = File.ReadAllText(file); var fileTokens = tokenizer.EnumerateTokens(code.AsSpan()); tracer.AddEpoch($"Tokenizing {file}"); var immutableNode = parser.ParseTokens(fileTokens); rootNodes.Add(immutableNode); tracer.AddEpoch($"Parsing {file}"); } EmitMain(programName + "Emit", assemblies, tracer, rootNodes); // Skip the ilasm backend for demonstation. IlAsmMain(programName, assemblies, tracer, rootNodes); tracer.PrintEpochs(); Console.WriteLine("Compilation Complete!"); }
public void TestVoidArrayFails() { var tokenizer = new SimpleTokenizer(); Assert.Throws <InvalidTokenParsingException>(() => tokenizer.EnumerateTokens("void[]")); }