public void LexPreprocessor() { string[] preprocessors = { "#define", "#undef", "#if", "#ifdef", "#ifndef", "#else", "#elif", "#endif", "#error", "#pragma", "#extension", "#version", "#line" }; SyntaxType[] types = { SyntaxType.DefinePreprocessorKeyword, SyntaxType.UndefinePreprocessorKeyword, SyntaxType.IfPreprocessorKeyword, SyntaxType.IfDefinedPreprocessorKeyword, SyntaxType.IfNotDefinedPreprocessorKeyword, SyntaxType.ElsePreprocessorKeyword, SyntaxType.ElseIfPreprocessorKeyword, SyntaxType.EndIfPreprocessorKeyword, SyntaxType.ErrorPreprocessorKeyword, SyntaxType.PragmaPreprocessorKeyword, SyntaxType.ExtensionPreprocessorKeyword, SyntaxType.VersionPreprocessorKeyword, SyntaxType.LinePreprocessorKeyword }; GLSLLexer lexer = new GLSLLexer(); TextSource source; LinkedList <Token> tokens; for (int i = 0; i < preprocessors.Length; i++) { source = new TextSource(preprocessors[i]); tokens = lexer.Run(source.CurrentSnapshot); Assert.AreEqual(2, tokens.Count); Assert.AreEqual(types[i], tokens.First.Value.SyntaxType); } source = new TextSource("#unknown"); tokens = lexer.Run(source.CurrentSnapshot); Assert.AreEqual(2, tokens.Count); Assert.AreEqual(SyntaxType.InvalidToken, tokens.First.Value.SyntaxType); InvalidToken token = (InvalidToken)tokens.First.Value; Assert.AreEqual("#unknown", token.Text); Assert.AreEqual(SyntaxType.PreprocessorToken, token.ErrorType); Assert.AreEqual("#unknown is not a valid preprocessor", token.ErrorMessage); }
public void LineContinuation() { string[] lines = { "in\\\n", "t ab\\\n", "cde\\\n", "fgh;", "1234\\\n", "56789" }; GLSLLexer lexer = new GLSLLexer(); MultiLineTextSource source = MultiLineTextSource.FromString(lines); List <Token> tokens = lexer.Run(source.CurrentSnapshot).ToList(); Assert.AreEqual(5, tokens.Count); Assert.AreEqual(SyntaxType.IntKeyword, tokens[0].SyntaxType); Assert.AreEqual(SyntaxType.IdentifierToken, tokens[1].SyntaxType); Assert.AreEqual("abcdefgh", tokens[1].Text); Assert.AreEqual(SyntaxType.SemicolonToken, tokens[2].SyntaxType); Assert.AreEqual(SyntaxType.IntConstToken, tokens[3].SyntaxType); Assert.AreEqual("123456789", tokens[3].Text); }
public void LexKeyword() { SyntaxType[] resultTypes = { SyntaxType.AttributeKeyword, SyntaxType.ConstKeyword, SyntaxType.UniformKeyword, SyntaxType.VaryingKeyword, SyntaxType.BufferKeyword, SyntaxType.SharedKeyword, SyntaxType.CoherentKeyword, SyntaxType.VolatileKeyword, SyntaxType.RestrictKeyword, SyntaxType.ReadOnlyKeyword, SyntaxType.WriteOnlyKeyword, SyntaxType.AtomicUIntKeyword, SyntaxType.LayoutKeyword, SyntaxType.CentroidKeyword, SyntaxType.FlatKeyword, SyntaxType.SmoothKeyword, SyntaxType.NoPerspectiveKeyword, SyntaxType.PatchKeyword, SyntaxType.SampleKeyword, SyntaxType.BreakKeyword, SyntaxType.ContinueKeyword, SyntaxType.DoKeyword, SyntaxType.ForKeyword, SyntaxType.WhileKeyword, SyntaxType.SwitchKeyword, SyntaxType.CaseKeyword, SyntaxType.DefaultKeyword, SyntaxType.IfKeyword, SyntaxType.ElseKeyword, SyntaxType.SubroutineKeyword, SyntaxType.InKeyword, SyntaxType.OutKeyword, SyntaxType.InOutKeyword, SyntaxType.FloatKeyword, SyntaxType.DoubleKeyword, SyntaxType.IntKeyword, SyntaxType.VoidKeyword, SyntaxType.BoolKeyword, SyntaxType.TrueKeyword, SyntaxType.FalseKeyword, SyntaxType.InvariantKeyword, SyntaxType.PreciseKeyword, SyntaxType.DiscardKeyword, SyntaxType.ReturnKeyword, SyntaxType.Mat2Keyword, SyntaxType.Mat3Keyword, SyntaxType.Mat4Keyword, SyntaxType.DMat2Keyword, SyntaxType.DMat3Keyword, SyntaxType.DMat4Keyword, SyntaxType.Mat2X2Keyword, SyntaxType.Mat2X3Keyword, SyntaxType.Mat2X4Keyword, SyntaxType.Mat3X2Keyword, SyntaxType.Mat3X3Keyword, SyntaxType.Mat3X4Keyword, SyntaxType.Mat4X2Keyword, SyntaxType.Mat4X3Keyword, SyntaxType.Mat4X4Keyword, SyntaxType.DMat2X2Keyword, SyntaxType.DMat2X3Keyword, SyntaxType.DMat2X4Keyword, SyntaxType.DMat3X2Keyword, SyntaxType.DMat3X3Keyword, SyntaxType.DMat3X4Keyword, SyntaxType.DMat4X2Keyword, SyntaxType.DMat4X3Keyword, SyntaxType.DMat4X4Keyword, SyntaxType.Vec2Keyword, SyntaxType.Vec3Keyword, SyntaxType.Vec4Keyword, SyntaxType.IVec2Keyword, SyntaxType.IVec3Keyword, SyntaxType.IVec4Keyword, SyntaxType.BVec2Keyword, SyntaxType.BVec3Keyword, SyntaxType.BVec4Keyword, SyntaxType.DVec2Keyword, SyntaxType.DVec3Keyword, SyntaxType.DVec4Keyword, SyntaxType.UIntKeyword, SyntaxType.UVec2Keyword, SyntaxType.UVec3Keyword, SyntaxType.UVec4Keyword, SyntaxType.LowPrecisionKeyword, SyntaxType.MediumPrecisionKeyword, SyntaxType.HighPrecisionKeyword, SyntaxType.PrecisionKeyword, SyntaxType.Sampler1DKeyword, SyntaxType.Sampler2DKeyword, SyntaxType.Sampler3DKeyword, SyntaxType.SamplerCubeKeyword, SyntaxType.Sampler1DShadowKeyword, SyntaxType.Sampler2DShadowKeyword, SyntaxType.SamplerCubeShadowKeyword, SyntaxType.Sampler1DArrayKeyword, SyntaxType.Sampler2DArrayKeyword, SyntaxType.Sampler1DArrayShadowKeyword, SyntaxType.Sampler2DArrayShadowKeyword, SyntaxType.ISampler1DKeyword, SyntaxType.ISampler2DKeyword, SyntaxType.ISampler3DKeyword, SyntaxType.ISamplerCubeKeyword, SyntaxType.ISampler1DArrayKeyword, SyntaxType.ISampler2DArrayKeyword, SyntaxType.USampler1DKeyword, SyntaxType.USampler2DKeyword, SyntaxType.USampler3DKeyword, SyntaxType.USamplerCubeKeyword, SyntaxType.USampler1DArrayKeyword, SyntaxType.USampler2DArrayKeyword, SyntaxType.Sampler2DRectKeyword, SyntaxType.Sampler2DRectShadowKeyword, SyntaxType.ISampler2DRectKeyword, SyntaxType.USampler2DRectKeyword, SyntaxType.SamplerBufferKeyword, SyntaxType.ISamplerBufferKeyword, SyntaxType.USamplerBufferKeyword, SyntaxType.Sampler2DMSKeyword, SyntaxType.ISampler2DMSKeyword, SyntaxType.USampler2DMSKeyword, SyntaxType.Sampler2DMSArrayKeyword, SyntaxType.ISampler2DMSArrayKeyword, SyntaxType.USampler2DMSArrayKeyword, SyntaxType.SamplerCubeArrayKeyword, SyntaxType.SamplerCubeArrayShadowKeyword, SyntaxType.ISamplerCubeArrayKeyword, SyntaxType.USamplerCubeArrayKeyword, SyntaxType.Image1DKeyword, SyntaxType.IImage1DKeyword, SyntaxType.UImage1DKeyword, SyntaxType.Image2DKeyword, SyntaxType.IImage2DKeyword, SyntaxType.UImage2DKeyword, SyntaxType.Image3DKeyword, SyntaxType.IImage3DKeyword, SyntaxType.UImage3DKeyword, SyntaxType.Image2DRectKeyword, SyntaxType.IImage2DRectKeyword, SyntaxType.UImage2DRectKeyword, SyntaxType.ImageCubeKeyword, SyntaxType.IImageCubeKeyword, SyntaxType.UImageCubeKeyword, SyntaxType.ImageBufferKeyword, SyntaxType.IImageBufferKeyword, SyntaxType.UImageBufferKeyword, SyntaxType.Image1DArrayKeyword, SyntaxType.IImage1DArrayKeyword, SyntaxType.UImage1DArrayKeyword, SyntaxType.Image2DArrayKeyword, SyntaxType.IImage2DArrayKeyword, SyntaxType.UImage2DArrayKeyword, SyntaxType.ImageCubeArrayKeyword, SyntaxType.IImageCubeArrayKeyword, SyntaxType.UImageCubeArrayKeyword, SyntaxType.Image2DMSKeyword, SyntaxType.IImage2DMSKeyword, SyntaxType.UImage2DMSKeyword, SyntaxType.Image2DMSArrayKeyword,SyntaxType.IImage2DMSArrayKeyword, SyntaxType.UImage2DMSArrayKeyword, SyntaxType.StructKeyword, SyntaxType.CommonKeyword, SyntaxType.PartitionKeyword, SyntaxType.ActiveKeyword, SyntaxType.ASMKeyword, SyntaxType.ClassKeyword, SyntaxType.UnionKeyword, SyntaxType.EnumKeyword, SyntaxType.TypedefKeyword, SyntaxType.TemplateKeyword, SyntaxType.ThisKeyword, SyntaxType.ResourceKeyword, SyntaxType.GotoKeyword, SyntaxType.InlineKeyword, SyntaxType.NoInlineKeyword, SyntaxType.PublicKeyword, SyntaxType.StaticKeyword, SyntaxType.ExternKeyword, SyntaxType.ExternalKeyword, SyntaxType.InterfaceKeyword, SyntaxType.LongKeyword, SyntaxType.ShortKeyword, SyntaxType.HalfKeyword, SyntaxType.FixedKeyword, SyntaxType.UnsignedKeyword, SyntaxType.SuperPKeyword, SyntaxType.InputKeyword, SyntaxType.OutputKeyword, SyntaxType.HVec2Keyword, SyntaxType.HVec3Keyword, SyntaxType.HVec4Keyword, SyntaxType.FVec2Keyword, SyntaxType.FVec3Keyword, SyntaxType.FVec4Keyword, SyntaxType.Sampler3DRectKeyword, SyntaxType.FilterKeyword, SyntaxType.SizeofKeyword, SyntaxType.CastKeyword, SyntaxType.NamespaceKeyword, SyntaxType.UsingKeyword }; string[] keywords = { "attribute", "const", "uniform", "varying", "buffer", "shared", "coherent", "volatile", "restrict", "readonly", "writeonly", "atomic_uint", "layout", "centroid", "flat", "smooth", "noperspective", "patch", "sample", "break", "continue", "do", "for", "while", "switch", "case", "default", "if", "else", "subroutine", "in", "out", "inout", "float", "double", "int", "void", "bool", "true", "false", "invariant", "precise", "discard", "return", "mat2", "mat3", "mat4", "dmat2", "dmat3", "dmat4", "mat2x2", "mat2x3", "mat2x4", "mat3x2", "mat3x3", "mat3x4", "mat4x2", "mat4x3", "mat4x4", "dmat2x2", "dmat2x3", "dmat2x4", "dmat3x2", "dmat3x3", "dmat3x4", "dmat4x2", "dmat4x3", "dmat4x4", "vec2", "vec3", "vec4", "ivec2", "ivec3", "ivec4", "bvec2", "bvec3", "bvec4", "dvec2", "dvec3", "dvec4", "uint", "uvec2", "uvec3", "uvec4", "lowp", "mediump", "highp", "precision", "sampler1D", "sampler2D", "sampler3D", "samplerCube", "sampler1DShadow", "sampler2DShadow", "samplerCubeShadow", "sampler1DArray", "sampler2DArray", "sampler1DArrayShadow", "sampler2DArrayShadow", "isampler1D", "isampler2D", "isampler3D", "isamplerCube", "isampler1DArray", "isampler2DArray", "usampler1D", "usampler2D", "usampler3D", "usamplerCube", "usampler1DArray", "usampler2DArray", "sampler2DRect", "sampler2DRectShadow", "isampler2DRect", "usampler2DRect", "samplerBuffer", "isamplerBuffer", "usamplerBuffer", "sampler2DMS", "isampler2DMS", "usampler2DMS", "sampler2DMSArray", "isampler2DMSArray", "usampler2DMSArray", "samplerCubeArray", "samplerCubeArrayShadow", "isamplerCubeArray", "usamplerCubeArray", "image1D", "iimage1D", "uimage1D", "image2D", "iimage2D", "uimage2D", "image3D", "iimage3D", "uimage3D", "image2DRect", "iimage2DRect", "uimage2DRect", "imageCube", "iimageCube", "uimageCube", "imageBuffer", "iimageBuffer", "uimageBuffer", "image1DArray", "iimage1DArray", "uimage1DArray", "image2DArray", "iimage2DArray", "uimage2DArray", "imageCubeArray", "iimageCubeArray", "uimageCubeArray", "image2DMS", "iimage2DMS", "uimage2DMS", "image2DMSArray", "iimage2DMSArray", "uimage2DMSArray", "struct", "common", "partition", "active", "asm", "class", "union", "enum", "typedef", "template", "this", "resource", "goto", "inline", "noinline", "public", "static", "extern", "external", "interface", "long", "short", "half", "fixed", "unsigned", "superp", "input", "output", "hvec2", "hvec3", "hvec4", "fvec2", "fvec3", "fvec4", "sampler3DRect", "filter", "sizeof", "cast", "namespace", "using" }; GLSLLexer lexer = new GLSLLexer(); for (int i = 0; i < keywords.Length; i++) { TextSource source = new TextSource(keywords[i]); LinkedList <Token> tokens = lexer.Run(source.CurrentSnapshot); Assert.AreEqual(2, tokens.Count); Assert.AreEqual(resultTypes[i], tokens.First.Value.SyntaxType); } }
public void FullParse() { string[] lines = File.ReadAllLines("test.glsl"); GLSLLexer lexer = new GLSLLexer(); MultiLineTextSource source = MultiLineTextSource.FromString(lines, true); GLSLParser parser = new GLSLParser(source.Settings); LinkedList <Token> tokens = lexer.Run(source.CurrentSnapshot); SyntaxTree tree = parser.Run(source.CurrentSnapshot, tokens); tree.WriteToXml("tree.xml", source.CurrentSnapshot); }
public void LexDecimal() { string[] lines = { "1234567890", "1234567890u", "1234567890U" }; SyntaxType[] resultTypes = { SyntaxType.IntConstToken, SyntaxType.UIntConstToken, SyntaxType.UIntConstToken }; GLSLLexer lexer = new GLSLLexer(); for (int i = 0; i < lines.Length; i++) { TextSource source = new TextSource(lines[i]); LinkedList <Token> tokens = lexer.Run(source.CurrentSnapshot); Assert.AreEqual(2, tokens.Count); Assert.AreEqual(resultTypes[i], tokens.First.Value.SyntaxType); } }
public void LexHex() { string[] lines = { "0x123456789abcdef", "0x123456789abcdefu", "0x123456789abcedfU", "0x123456789ABCDEF", "0x123456789ABCDEFu", "0x123456789ABCDEFU", "0X123456789abcdef", "0X123456789abcdefu", "0X123456789abcedfU", "0X123456789ABCDEF", "0X123456789ABCDEFu", "0X123456789ABCDEFU" }; SyntaxType[] resultTypes = { SyntaxType.IntConstToken, SyntaxType.UIntConstToken, SyntaxType.UIntConstToken }; GLSLLexer lexer = new GLSLLexer(); for (int i = 0; i < lines.Length; i++) { TextSource source = new TextSource(lines[i]); LinkedList <Token> tokens = lexer.Run(source.CurrentSnapshot); Assert.AreEqual(2, tokens.Count); Assert.AreEqual(resultTypes[i % 3], tokens.First.Value.SyntaxType); } }
public void LexComment() { string[] lines = { "/*\n", "this is in a block comment/* */\n", "*/\n", "\n", "// this is in a comment\n", "/* */ /* /* */ \n", "Assert.AreEqual(resultTypes[i], lexer.GetTokens().First.Value.Type);*/\n", "// this is a line comment \\\n", "that continues to the next line \\\n", "and still continues \\\n" }; GLSLLexer lexer = new GLSLLexer(); MultiLineTextSource source = MultiLineTextSource.FromString(lines); LinkedList <Token> list = lexer.Run(source.CurrentSnapshot); LinkedListNode <Token> node = list.First; Assert.AreEqual(1, list.Count); Assert.AreEqual(SyntaxType.EOF, node.Value.SyntaxType); Assert.AreEqual(false, node.Value.HasTrailingTrivia); Assert.AreEqual(true, node.Value.HasLeadingTrivia); Assert.AreEqual(SyntaxType.TriviaList, node.Value.LeadingTrivia.SyntaxType); SyntaxTriviaList triviaList = node.Value.LeadingTrivia as SyntaxTriviaList; Assert.AreEqual(10, triviaList.List.Count); Assert.AreEqual(SyntaxType.BlockCommentTrivia, triviaList.List[0].SyntaxType); Assert.AreEqual(SyntaxType.NewLineTrivia, triviaList.List[1].SyntaxType); Assert.AreEqual(SyntaxType.NewLineTrivia, triviaList.List[2].SyntaxType); Assert.AreEqual(SyntaxType.LineCommentTrivia, triviaList.List[3].SyntaxType); Assert.AreEqual(SyntaxType.NewLineTrivia, triviaList.List[4].SyntaxType); Assert.AreEqual(SyntaxType.BlockCommentTrivia, triviaList.List[5].SyntaxType); Assert.AreEqual(SyntaxType.WhiteSpaceTrivia, triviaList.List[6].SyntaxType); Assert.AreEqual(SyntaxType.BlockCommentTrivia, triviaList.List[7].SyntaxType); Assert.AreEqual(SyntaxType.NewLineTrivia, triviaList.List[8].SyntaxType); Assert.AreEqual(SyntaxType.LineCommentTrivia, triviaList.List[9].SyntaxType); }
public static string Minify(string shaderCode) { AntlrInputStream input = new AntlrInputStream(shaderCode); GLSLLexer lexer = new GLSLLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); GLSLParser parser = new GLSLParser(tokens); //parser.Ac // need to add visitors so //https://www.antlr2.org/doc/trees.html //https://stackoverflow.com/questions/29929062/antlr4-ast-vistor-and-node-return-type-should-i-have-more-than-one-visitor // https://stackoverflow.com/questions/23092081/antlr4-visitor-pattern-on-simple-arithmetic-example //var unit = parser.translation_unit(); //unit.external_declaration_list().external_declaration_list().GetChild(0) return(shaderCode);// unit.ToStringTree(); }
public void LexSymbols() { string line = "( ) [ ] { } . , : = == ; ! != - -- -= ~ + ++ += * *= / /= % %= < <= << <<= > >= >> >>= | || |= ^ ^^ ^= & && &= ?"; SyntaxType[] expectedTypes = { SyntaxType.LeftParenToken, SyntaxType.RightParenToken, SyntaxType.LeftBracketToken, SyntaxType.RightBracketToken, SyntaxType.LeftBraceToken, SyntaxType.RightBraceToken, SyntaxType.DotToken, SyntaxType.CommaToken, SyntaxType.ColonToken, SyntaxType.EqualToken, SyntaxType.EqualEqualToken, SyntaxType.SemicolonToken, SyntaxType.ExclamationToken, SyntaxType.ExclamationEqualToken, SyntaxType.MinusToken, SyntaxType.MinusMinusToken, SyntaxType.MinusEqualToken, SyntaxType.TildeToken, SyntaxType.PlusToken, SyntaxType.PlusPlusToken, SyntaxType.PlusEqualToken, SyntaxType.StarToken, SyntaxType.StarEqualToken, SyntaxType.SlashToken, SyntaxType.SlashEqualToken, SyntaxType.PercentToken, SyntaxType.PercentEqualToken, SyntaxType.LessThenToken, SyntaxType.LessThenEqualToken, SyntaxType.LessThenLessThenToken, SyntaxType.LessThenLessThenEqualToken, SyntaxType.GreaterThenToken, SyntaxType.GreaterThenEqualToken, SyntaxType.GreaterThenGreaterThenToken, SyntaxType.GreaterThenGreaterThenEqualToken, SyntaxType.VerticalBarToken, SyntaxType.BarBarToken, SyntaxType.BarEqualToken, SyntaxType.CaretToken, SyntaxType.CaretCaretToken, SyntaxType.CaretEqualToken, SyntaxType.AmpersandToken, SyntaxType.AmpersandAmpersandToken, SyntaxType.AmpersandEqualToken, SyntaxType.QuestionToken, SyntaxType.EOF }; GLSLLexer lexer = new GLSLLexer(); TextSource source = new TextSource(line); int index = 0; foreach (Token token in lexer.Run(source.CurrentSnapshot)) { Assert.AreEqual(expectedTypes[index++], token.SyntaxType); } Assert.AreEqual(expectedTypes.Length, index); }
public void LexFloat() { string[] lines = { "5.5", "5.", ".5", "5.5e5", "5.e5", ".5e5", "5.5e+5", "5.e+5", ".5e+5", "5.5e-5", "5.e-5", ".5e-5", "5.5e5F", "5.e5F", ".5e5F", "5.5e+5F", "5.e+5F", ".5e+5F", "5.5e-5F", "5.e-5F", ".5e-5F", "5.5e5f", "5.e5f", ".5e5f", "5.5e+5f", "5.e+5f", ".5e+5f", "5.5e-5f", "5.e-5f", ".5e-5f", "5.e5lf", ".5e5lf", "5.5e+5lf", "5.e+5lf", ".5e+5lf", "5.5e-5lf", "5.e-5lf", ".5e-5lf", "5.5e5LF", "5.e5LF", ".5e5LF", "5.5e+5LF", "5.e+5LF", ".5e+5LF", "5.5e-5LF", "5.e-5LF", ".5e-5LF", "5.5f", "5.f", ".5f", "5.5F", "5.F", ".5F", "5.5lf", "5.lf", ".5lf", "5.5LF", "5.LF", ".5LF", "5e5" }; GLSLLexer lexer = new GLSLLexer(); for (int i = 0; i < lines.Length; i++) { TextSource source = new TextSource(lines[i]); LinkedList <Token> tokens = lexer.Run(source.CurrentSnapshot); Assert.AreEqual(2, tokens.Count); if (lines[i].Contains("lf") || lines[i].Contains("LF")) { Assert.AreEqual(SyntaxType.DoubleConstToken, tokens.First.Value.SyntaxType); } else { Assert.AreEqual(SyntaxType.FloatConstToken, tokens.First.Value.SyntaxType); } } }