public void TestFrontMatter() { var options = new LexerOptions() { Mode = ScriptMode.FrontMatterAndContent }; var input = @"+++ variable = 1 name = 'yes' +++ This is after the frontmatter: {{ name }} {{ variable + 1 }}"; input = input.Replace("\r\n", "\n"); var template = ParseTemplate(input, options); // Make sure that we have a front matter Assert.NotNull(template.Page.FrontMatter); var context = new TemplateContext(); // Evaluate front-matter var frontResult = context.Evaluate(template.Page.FrontMatter); Assert.Null(frontResult); // Evaluate page-content context.Evaluate(template.Page); var pageResult = context.Output.ToString(); TextAssert.AreEqual("This is after the frontmatter: yes\n2", pageResult); }
/// <summary> /// Parses the specified Liquid script text into a <see cref="Template"/> . /// </summary> /// <param name="text">The liquid scripting text.</param> /// <param name="sourceFilePath">The source file path. Optional, used for better error reporting if the source file has a location on the disk</param> /// <param name="parserOptions">The templating parsing parserOptions.</param> /// <param name="lexerOptions">The options passed to the lexer</param> /// <returns>A template</returns> public static Template ParseLiquid(string text, string sourceFilePath = null, ParserOptions?parserOptions = null, LexerOptions?lexerOptions = null) { LexerOptions localLexerOptions = lexerOptions ?? new LexerOptions(); localLexerOptions.Mode = ScriptMode.Liquid; return(Parse(text, sourceFilePath, parserOptions, localLexerOptions)); }
public void TestScriptOnly() { var options = new LexerOptions() { Mode = ScriptMode.ScriptOnly }; var template = ParseTemplate(@" variable = 1 name = 'yes' ", options); var context = new TemplateContext(); template.Render(context); var outputStr = context.Output.ToString(); Assert.AreEqual(string.Empty, outputStr); var global = context.CurrentGlobal; object value; Assert.True(global.TryGetValue("name", out value)); Assert.AreEqual("yes", value); Assert.True(global.TryGetValue("variable", out value)); Assert.AreEqual(1, value); }
private Template(ParserOptions?parserOptions, LexerOptions?lexerOptions, string sourceFilePath) { _parserOptions = parserOptions ?? new ParserOptions(); _lexerOptions = lexerOptions ?? new LexerOptions(); Messages = new List <LogMessage>(); this.SourceFilePath = sourceFilePath; }
public Lexer(LexerOptions options = null) { Options = options ?? new LexerOptions() { Target = RFCTarget.RFC1960 }; Tokens = new Tokens1960(); }
public idScriptParser(LexerOptions options) { _options = options; _markerPosition = -1; _scriptStack = new Stack<idLexer>(); _tokens = new Stack<idToken>(); _defines = new List<ScriptDefinition>(); _indentStack = new Stack<ScriptIndentation>(); }
public idScriptParser(LexerOptions options) { _options = options; _markerPosition = -1; _scriptStack = new Stack <idLexer>(); _tokens = new Stack <idToken>(); _defines = new List <ScriptDefinition>(); _indentStack = new Stack <ScriptIndentation>(); }
public static object Evaluate(string expression, object model, MemberRenamerDelegate memberRenamer = null, MemberFilterDelegate memberFilter = null) { if (expression == null) { throw new ArgumentNullException(nameof(expression)); } var lexerOption = new LexerOptions() { Mode = ScriptMode.ScriptOnly }; var template = Parse(expression, lexerOptions: lexerOption); return(template.Evaluate(model, memberRenamer, memberFilter)); }
public static object Evaluate(string expression, TemplateContext context) { if (expression == null) { throw new ArgumentNullException(nameof(expression)); } var lexerOption = new LexerOptions() { Mode = ScriptMode.ScriptOnly }; var template = Parse(expression, lexerOptions: lexerOption); return(template.Evaluate(context)); }
public LiquidTemplateContext() : base(new LiquidBuiltinsFunctions()) { // In liquid, if we have a break/continue outside a loop, we return from the current script EnableBreakAndContinueAsReturnOutsideLoop = true; EnableRelaxedMemberAccess = true; TemplateLoaderLexerOptions = new LexerOptions() { Mode = ScriptMode.Liquid }; TemplateLoaderParserOptions = new ParserOptions() { LiquidFunctionsToScriban = true }; }
public void TestEvaluateScriptOnly() { { var lexerOptions = new LexerOptions() { Mode = ScriptMode.ScriptOnly }; var template = Template.Parse("y = x + 1; y;", lexerOptions: lexerOptions); var result = template.Evaluate(new { x = 10 }); Assert.AreEqual(11, result); } { var result = Template.Evaluate("y = x + 1; y;", new { x = 10 }); Assert.AreEqual(11, result); } }
public void TestFrontMatter() { var options = new LexerOptions() { Mode = ScriptMode.FrontMatterAndContent }; var input = @"+++ variable = 1 name = 'yes' +++ This is after the frontmatter: {{ name }} {{ variable + 1 }}"; input = input.Replace("\r\n", "\n"); var template = ParseTemplate(_output, input, options); // Make sure that we have a front matter Assert.NotNull(template.Page.FrontMatter); var context = new TemplateContext(); // Evaluate front-matter var frontResult = context.Evaluate(template.Page.FrontMatter); Assert.Null(frontResult); // Evaluate page-content context.Evaluate(template.Page); var pageResult = context.Output.ToString(); EqualityCompareResult compareResult = TextAssert.Equal("This is after the frontmatter: yes\n2", pageResult); if (compareResult.VerboseMessage != null && compareResult.VerboseMessage.Count > 0) { foreach (string vmsg in compareResult.VerboseMessage) { _output.WriteLine(vmsg); } } Assert.True(compareResult.IsEqual); }
private Template LoadTemplate(string inputName) { var templateSource = TestFilesHelper.LoadTestFile(inputName); var parser = inputName.Contains("500-liquid") ? (Func <string, string, ParserOptions?, LexerOptions?, Template>)Template.ParseLiquid : Template.Parse; var options = new LexerOptions(); if (inputName.Contains("liquid")) { options.Lang = ScriptLang.Liquid; } else if (inputName.Contains("scientific")) { options.Lang = ScriptLang.Scientific; } var template = parser(templateSource, inputName, default, options);
public Lexer(string input, LexerOptions options = LexerOptions.IgnoreComments | LexerOptions.IgnoreWhitespace) { Input = input.Trim(); Options = options; }
public static void AssertTemplate(string expected, string input, ScriptLang lang = ScriptLang.Default, bool isRoundtripTest = false, bool supportExactRoundtrip = true, object model = null, bool specialLiquid = false, bool expectParsingErrorForRountrip = false, bool supportRoundTrip = true) { bool isLiquid = lang == ScriptLang.Liquid; var parserOptions = new ParserOptions() { LiquidFunctionsToScriban = isLiquid, }; var lexerOptions = new LexerOptions() { Lang = lang }; if (isRoundtripTest) { lexerOptions.KeepTrivia = true; } if (specialLiquid) { parserOptions.ExpressionDepthLimit = 500; } #if EnableTokensOutput { Console.WriteLine("Tokens"); Console.WriteLine("======================================"); var lexer = new Lexer(input, options: lexerOptions); foreach (var token in lexer) { Console.WriteLine($"{token.Type}: {token.GetText(input)}"); } Console.WriteLine(); } #endif string roundtripText = null; // We loop first on input text, then on roundtrip while (true) { bool isRoundtrip = roundtripText != null; bool hasErrors = false; bool hasException = false; if (isRoundtrip) { Console.WriteLine("Roundtrip"); Console.WriteLine("======================================"); Console.WriteLine(roundtripText); lexerOptions.Lang = lang == ScriptLang.Scientific ? lang : ScriptLang.Default; if (!isLiquid && supportExactRoundtrip) { Console.WriteLine("Checking Exact Roundtrip - Input"); Console.WriteLine("======================================"); TextAssert.AreEqual(input, roundtripText); } input = roundtripText; } else { Console.WriteLine("Input"); Console.WriteLine("======================================"); Console.WriteLine(input); } var template = Template.Parse(input, "text", parserOptions, lexerOptions); var result = string.Empty; var resultAsync = string.Empty; if (template.HasErrors) { hasErrors = true; for (int i = 0; i < template.Messages.Count; i++) { var message = template.Messages[i]; if (i > 0) { result += "\n"; } result += message; } if (specialLiquid && !isRoundtrip) { throw new InvalidOperationException("Parser errors: " + result); } } else { if (isRoundtripTest) { result = template.ToText(); } else { Assert.NotNull(template.Page); if (!isRoundtrip) { // Dumps the roundtrip version var lexerOptionsForTrivia = lexerOptions; lexerOptionsForTrivia.KeepTrivia = true; var templateWithTrivia = Template.Parse(input, "input", parserOptions, lexerOptionsForTrivia); roundtripText = templateWithTrivia.ToText(); } try { // Setup a default model context for the tests if (model == null) { var scriptObj = new ScriptObject { ["page"] = new ScriptObject { ["title"] = "This is a title" }, ["user"] = new ScriptObject { ["name"] = "John" }, ["product"] = new ScriptObject { ["title"] = "Orange", ["type"] = "fruit" }, ["products"] = new ScriptArray() { new ScriptObject { ["title"] = "Orange", ["type"] = "fruit" }, new ScriptObject { ["title"] = "Banana", ["type"] = "fruit" }, new ScriptObject { ["title"] = "Apple", ["type"] = "fruit" }, new ScriptObject { ["title"] = "Computer", ["type"] = "electronics" }, new ScriptObject { ["title"] = "Mobile Phone", ["type"] = "electronics" }, new ScriptObject { ["title"] = "Table", ["type"] = "furniture" }, new ScriptObject { ["title"] = "Sofa", ["type"] = "furniture" }, } }; scriptObj.Import(typeof(SpecialFunctionProvider)); model = scriptObj; } // Render sync { var context = NewTemplateContext(lang); context.PushOutput(new TextWriterOutput(new StringWriter() { NewLine = "\n" })); var contextObj = new ScriptObject(); contextObj.Import(model); context.PushGlobal(contextObj); result = template.Render(context); } // Render async { var asyncContext = NewTemplateContext(lang); asyncContext.PushOutput(new TextWriterOutput(new StringWriter() { NewLine = "\n" })); var contextObj = new ScriptObject(); contextObj.Import(model); asyncContext.PushGlobal(contextObj); resultAsync = template.RenderAsync(asyncContext).Result; } } catch (Exception exception) { hasException = true; if (specialLiquid) { throw; } else { result = GetReason(exception); } } } } var testContext = isRoundtrip ? "Roundtrip - " : String.Empty; Console.WriteLine($"{testContext}Result"); Console.WriteLine("======================================"); Console.WriteLine(result); Console.WriteLine($"{testContext}Expected"); Console.WriteLine("======================================"); Console.WriteLine(expected); if (isRoundtrip && expectParsingErrorForRountrip) { Assert.True(hasErrors, "The roundtrip test is expecting an error"); Assert.AreNotEqual(expected, result); } else { TextAssert.AreEqual(expected, result); } if (!isRoundtrip && !isRoundtripTest && !hasErrors && !hasException) { Console.WriteLine("Checking async"); Console.WriteLine("======================================"); TextAssert.AreEqual(expected, resultAsync); } if (!supportRoundTrip || isRoundtripTest || isRoundtrip || hasErrors) { break; } } }
public Lexer(LexerOptions options) { _options = options; }
public idLexer(LexerOptions options) : this() { _options = options; }
public Lexer2254(LexerOptions options = null) { Options = options ?? new LexerOptions(); Tokens = new Tokens2254(); }
public object Invoke(TemplateContext context, ScriptNode callerContext, ScriptArray arguments, ScriptBlockStatement blockStatement) { if (arguments.Count == 0) { throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.BadFunctionInvokeArgEmpty, "include")); } string templateName = context.ToString(callerContext.Span, arguments[0]); // If template name is empty, throw an exception if (string.IsNullOrEmpty(templateName)) { // In a liquid template context, we let an include to continue without failing if (context is LiquidTemplateContext) { return(null); } throw new ScriptRuntimeException(callerContext.Span, RS.IncludeNameRequired); } ITemplateLoader templateLoader = context.TemplateLoader; if (templateLoader == null) { throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.NoTemplateLoader, "include")); } string templatePath; try { templatePath = templateLoader.GetPath(context, callerContext.Span, templateName); } catch (Exception ex) when(!(ex is ScriptRuntimeException)) { throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.IncludeResolvePathError, templateName), ex); } // If template name is empty, throw an exception if (templatePath == null) { throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.IncludePathNullError, templateName)); } // Compute a new parameters for the include ScriptArray newParameters = new ScriptArray(arguments.Count - 1); for (int i = 1; i < arguments.Count; i++) { newParameters[i] = arguments[i]; } context.SetValue(ScriptVariable.Arguments, newParameters, true); Template template; if (!context.CachedTemplates.TryGetValue(templatePath, out template)) { string templateText; try { templateText = templateLoader.Load(context, callerContext.Span, templatePath); } catch (Exception ex) when(!(ex is ScriptRuntimeException)) { throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.IncludeLoadError, templateName, templatePath), ex); } if (templateText == null) { throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.IncludeContentEmpty, templateName, templatePath)); } // Clone parser options ParserOptions parserOptions = context.TemplateLoaderParserOptions; LexerOptions lexerOptions = context.TemplateLoaderLexerOptions; template = Template.Parse(templateText, templatePath, parserOptions, lexerOptions); // If the template has any errors, throw an exception if (template.HasErrors) { throw new ScriptParserRuntimeException(callerContext.Span, string.Format(RS.IncludeParseError, templateName, templatePath), template.Messages); } context.CachedTemplates.Add(templatePath, template); } // Make sure that we cannot recursively include a template context.PushOutput(); object result = null; try { context.EnterRecursive(callerContext); result = template.Render(context); context.ExitRecursive(callerContext); } finally { context.PopOutput(); } return(result); }
public Lexer(ICollection<IFilter> filters, LexerOptions options) { _options = options; _filters = filters; }
public static TemplateCompareResult TestTemplate(string expected, string input, bool isLiquid = false, bool isRoundtripTest = false, bool supportExactRoundtrip = true, object model = null, bool specialLiquid = false) { var parserOptions = new ParserOptions() { ConvertLiquidFunctions = isLiquid }; var lexerOptions = new LexerOptions() { Mode = isLiquid ? ScriptMode.Liquid : ScriptMode.Default }; if (isRoundtripTest) { lexerOptions.KeepTrivia = true; } if (specialLiquid) { parserOptions.ExpressionDepthLimit = 500; } List <string> tokensDesc = new List <string>(); var lexer = new Lexer(input, options: lexerOptions); foreach (var token in lexer) { tokensDesc.Add($"{token.Type}: {token.GetText(input)}"); } string roundtripText = null; List <string> verboseMessage = new List <string>(); // We loop first on input text, then on rountrip while (true) { bool isRoundtrip = roundtripText != null; bool hasErrors = false; if (isRoundtrip) { verboseMessage.Add("Rountrip"); verboseMessage.Add("======================================"); verboseMessage.Add(roundtripText); lexerOptions.Mode = ScriptMode.Default; if (lexerOptions.Mode == ScriptMode.Default && !isLiquid && supportExactRoundtrip) { verboseMessage.Add("Checking Exact Roundtrip - Input"); verboseMessage.Add("======================================"); EqualityCompareResult equalResult = TextAssert.Equal(input, roundtripText); if (equalResult.VerboseMessage != null && equalResult.VerboseMessage.Count > 0) { verboseMessage.AddRange(equalResult.VerboseMessage); } if (equalResult.IsEqual == false) { return(new TemplateCompareResult() { Tokens = tokensDesc, IsEqual = false, VerboseMessage = verboseMessage, DebugCode = "INPUT_ROUNDTRIP_TEXT_NOT_EQUAL" }); } } input = roundtripText; } else { verboseMessage.Add("Input"); verboseMessage.Add("======================================"); verboseMessage.Add(input); } var template = Template.Parse(input, "text", parserOptions, lexerOptions); var result = string.Empty; if (template.HasErrors) { hasErrors = true; for (int i = 0; i < template.Messages.Count; i++) { var message = template.Messages[i]; if (i > 0) { result += "\n"; } result += message; } if (specialLiquid) { throw new InvalidOperationException("Parser errors: " + result); } } else { if (isRoundtripTest) { result = template.ToText(); } else { if (template.Page == null) { return(new TemplateCompareResult() { Tokens = tokensDesc, IsEqual = false, VerboseMessage = verboseMessage, DebugCode = "PAGE_IS_NULL", Result = result, Expected = expected }); } if (!isRoundtrip) { // Dumps the rountrip version var lexerOptionsForTrivia = lexerOptions; lexerOptionsForTrivia.KeepTrivia = true; var templateWithTrivia = Template.Parse(input, "input", parserOptions, lexerOptionsForTrivia); roundtripText = templateWithTrivia.ToText(); } try { // Setup a default model context for the tests if (model == null) { var scriptObj = new ScriptObject { ["page"] = new ScriptObject { ["title"] = "This is a title" }, ["user"] = new ScriptObject { ["name"] = "John" }, ["product"] = new ScriptObject { ["title"] = "Orange", ["type"] = "fruit" }, ["products"] = new ScriptArray() { new ScriptObject { ["title"] = "Orange", ["type"] = "fruit" }, new ScriptObject { ["title"] = "Banana", ["type"] = "fruit" }, new ScriptObject { ["title"] = "Apple", ["type"] = "fruit" }, new ScriptObject { ["title"] = "Computer", ["type"] = "electronics" }, new ScriptObject { ["title"] = "Mobile Phone", ["type"] = "electronics" }, new ScriptObject { ["title"] = "Table", ["type"] = "furniture" }, new ScriptObject { ["title"] = "Sofa", ["type"] = "furniture" }, } }; scriptObj.Import(typeof(SpecialFunctionProvider)); model = scriptObj; } var context = isLiquid ? new LiquidTemplateContext() { TemplateLoader = new LiquidCustomTemplateLoader() } : new TemplateContext() { TemplateLoader = new CustomTemplateLoader() }; // We use a custom output to make sure that all output is using the "\n" context.PushOutput(new TextWriterOutput(new StringWriter() { NewLine = "\n" })); var contextObj = new ScriptObject(); contextObj.Import(model); context.PushGlobal(contextObj); result = template.Render(context); } catch (Exception exception) { if (specialLiquid) { throw; } else { result = GetReason(exception); } } } } var testContext = isRoundtrip ? "Roundtrip - " : String.Empty; verboseMessage.Add($"{testContext}Result"); verboseMessage.Add("======================================"); verboseMessage.Add(result); verboseMessage.Add($"{testContext}Expected"); verboseMessage.Add("======================================"); verboseMessage.Add(expected); EqualityCompareResult testEqualResult = TextAssert.Equal(expected, result); if (testEqualResult.VerboseMessage != null && testEqualResult.VerboseMessage.Count > 0) { verboseMessage.AddRange(testEqualResult.VerboseMessage); } if (testEqualResult.IsEqual == false) { return(new TemplateCompareResult { IsEqual = false, VerboseMessage = verboseMessage, DebugCode = "UNKNOWN", Tokens = tokensDesc, Result = result, Expected = expected }); } if (isRoundtripTest || isRoundtrip || hasErrors) { break; } } return(new TemplateCompareResult { IsEqual = true, VerboseMessage = verboseMessage, DebugCode = "", Tokens = tokensDesc }); }
public KalkEngine() : base(new KalkObjectWithAlias()) { KalkSettings.Initialize(); KalkEngineFolder = AppContext.BaseDirectory; // Enforce UTF8 encoding Console.OutputEncoding = Encoding.UTF8; EnableEngineOutput = true; EchoEnabled = true; DisplayVersion = true; CurrentDisplay = KalkDisplayMode.Standard; KalkUserFolder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile, Environment.SpecialFolderOption.DoNotVerify), ".kalk"); HighlightOutput = new ConsoleText(); InputReader = Console.In; OutputWriter = Console.Out; ErrorWriter = Console.Error; IsOutputSupportHighlighting = ConsoleHelper.SupportEscapeSequences; // Fetch version var assemblyInfoVersion = (AssemblyInformationalVersionAttribute)typeof(KalkEngine).Assembly.GetCustomAttribute(typeof(AssemblyInformationalVersionAttribute)); Version = assemblyInfoVersion.InformationalVersion; Builtins = BuiltinObject; ((KalkObjectWithAlias)Builtins).Engine = this; Units = new KalkUnits(this); Shortcuts = new KalkShortcuts(); Aliases = new KalkAliases(); _currentShortcutKeyMap = Shortcuts.ShortcutKeyMap; _completionMatchingList = new List <string>(); Config = new KalkConfig(); Variables = new ScriptVariables(this); Descriptors = new Dictionary <string, KalkDescriptor>(); EnableRelaxedMemberAccess = false; _modules = new Dictionary <Type, KalkModule>(); TryConverters = new List <TryToObjectDelegate>(); ErrorForStatementFunctionAsExpression = true; StrictVariables = true; UseScientific = true; LoopLimit = int.MaxValue; // no limits for loops RecursiveLimit = int.MaxValue; // no limits (still guarded by Scriban) // Setup default clipboard methods _localClipboard = string.Empty; GetClipboardText = GetClipboardTextImpl; SetClipboardText = SetClipboardTextImpl; _cancellationTokenSource = new CancellationTokenSource(); PushGlobal(Units); PushGlobal(Variables); _parserOptions = new ParserOptions(); _lexerOptions = new LexerOptions() { KeepTrivia = true, Mode = ScriptMode.ScriptOnly, Lang = ScriptLang.Scientific }; _lexerInterpolatedOptions = new LexerOptions() { KeepTrivia = true, Mode = ScriptMode.Default, Lang = ScriptLang.Scientific }; _tempOutputHighlight = new ConsoleText(); // Init last result with 0 _lastResult = 0; HistoryList = new List <string>(); _isInitializing = true; RegisterFunctions(); _isInitializing = false; }
public Lexer(string input, LexerOptions options = LexerOptions.IgnoreComments | LexerOptions.IgnoreWhitespace) { this.Input = input; this.Options = options; }