public void TestFrontMatter()
{
var options = new LexerOptions()
{
Mode = ScriptMode.FrontMatterAndContent
};
var input = @"+++
variable = 1
name = 'yes'
+++
This is after the frontmatter: {{ name }}
{{
variable + 1
}}";
input = input.Replace("\r\n", "\n");
var template = ParseTemplate(input, options);
// Make sure that we have a front matter
Assert.NotNull(template.Page.FrontMatter);
var context = new TemplateContext();
// Evaluate front-matter
var frontResult = context.Evaluate(template.Page.FrontMatter);
Assert.Null(frontResult);
// Evaluate page-content
context.Evaluate(template.Page);
var pageResult = context.Output.ToString();
TextAssert.AreEqual("This is after the frontmatter: yes\n2", pageResult);
}
/// <summary>
/// Parses the specified Liquid script text into a <see cref="Template"/> .
/// </summary>
/// <param name="text">The liquid scripting text.</param>
/// <param name="sourceFilePath">The source file path. Optional, used for better error reporting if the source file has a location on the disk</param>
/// <param name="parserOptions">The templating parsing parserOptions.</param>
/// <param name="lexerOptions">The options passed to the lexer</param>
/// <returns>A template</returns>
public static Template ParseLiquid(string text, string sourceFilePath = null, ParserOptions?parserOptions = null, LexerOptions?lexerOptions = null)
{
LexerOptions localLexerOptions = lexerOptions ?? new LexerOptions();
localLexerOptions.Mode = ScriptMode.Liquid;
return(Parse(text, sourceFilePath, parserOptions, localLexerOptions));
}
public void TestScriptOnly()
{
var options = new LexerOptions()
{
Mode = ScriptMode.ScriptOnly
};
var template = ParseTemplate(@"
variable = 1
name = 'yes'
", options);
var context = new TemplateContext();
template.Render(context);
var outputStr = context.Output.ToString();
Assert.AreEqual(string.Empty, outputStr);
var global = context.CurrentGlobal;
object value;
Assert.True(global.TryGetValue("name", out value));
Assert.AreEqual("yes", value);
Assert.True(global.TryGetValue("variable", out value));
Assert.AreEqual(1, value);
}
private Template(ParserOptions?parserOptions, LexerOptions?lexerOptions, string sourceFilePath)
{
_parserOptions = parserOptions ?? new ParserOptions();
_lexerOptions = lexerOptions ?? new LexerOptions();
Messages = new List <LogMessage>();
this.SourceFilePath = sourceFilePath;
}
public Lexer(LexerOptions options = null)
{
Options = options ?? new LexerOptions()
{
Target = RFCTarget.RFC1960
};
Tokens = new Tokens1960();
}
public idScriptParser(LexerOptions options)
{
_options = options;
_markerPosition = -1;
_scriptStack = new Stack<idLexer>();
_tokens = new Stack<idToken>();
_defines = new List<ScriptDefinition>();
_indentStack = new Stack<ScriptIndentation>();
}
public idScriptParser(LexerOptions options)
{
_options = options;
_markerPosition = -1;
_scriptStack = new Stack <idLexer>();
_tokens = new Stack <idToken>();
_defines = new List <ScriptDefinition>();
_indentStack = new Stack <ScriptIndentation>();
}
public static object Evaluate(string expression, object model, MemberRenamerDelegate memberRenamer = null, MemberFilterDelegate memberFilter = null)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
var lexerOption = new LexerOptions()
{
Mode = ScriptMode.ScriptOnly
};
var template = Parse(expression, lexerOptions: lexerOption);
return(template.Evaluate(model, memberRenamer, memberFilter));
}
public static object Evaluate(string expression, TemplateContext context)
{
if (expression == null)
{
throw new ArgumentNullException(nameof(expression));
}
var lexerOption = new LexerOptions()
{
Mode = ScriptMode.ScriptOnly
};
var template = Parse(expression, lexerOptions: lexerOption);
return(template.Evaluate(context));
}
public LiquidTemplateContext() : base(new LiquidBuiltinsFunctions())
{
// In liquid, if we have a break/continue outside a loop, we return from the current script
EnableBreakAndContinueAsReturnOutsideLoop = true;
EnableRelaxedMemberAccess = true;
TemplateLoaderLexerOptions = new LexerOptions()
{
Mode = ScriptMode.Liquid
};
TemplateLoaderParserOptions = new ParserOptions()
{
LiquidFunctionsToScriban = true
};
}
public void TestEvaluateScriptOnly()
{
{
var lexerOptions = new LexerOptions()
{
Mode = ScriptMode.ScriptOnly
};
var template = Template.Parse("y = x + 1; y;", lexerOptions: lexerOptions);
var result = template.Evaluate(new { x = 10 });
Assert.AreEqual(11, result);
}
{
var result = Template.Evaluate("y = x + 1; y;", new { x = 10 });
Assert.AreEqual(11, result);
}
}
public void TestFrontMatter()
{
var options = new LexerOptions()
{
Mode = ScriptMode.FrontMatterAndContent
};
var input = @"+++
variable = 1
name = 'yes'
+++
This is after the frontmatter: {{ name }}
{{
variable + 1
}}";
input = input.Replace("\r\n", "\n");
var template = ParseTemplate(_output, input, options);
// Make sure that we have a front matter
Assert.NotNull(template.Page.FrontMatter);
var context = new TemplateContext();
// Evaluate front-matter
var frontResult = context.Evaluate(template.Page.FrontMatter);
Assert.Null(frontResult);
// Evaluate page-content
context.Evaluate(template.Page);
var pageResult = context.Output.ToString();
EqualityCompareResult compareResult = TextAssert.Equal("This is after the frontmatter: yes\n2", pageResult);
if (compareResult.VerboseMessage != null && compareResult.VerboseMessage.Count > 0)
{
foreach (string vmsg in compareResult.VerboseMessage)
{
_output.WriteLine(vmsg);
}
}
Assert.True(compareResult.IsEqual);
}
private Template LoadTemplate(string inputName)
{
var templateSource = TestFilesHelper.LoadTestFile(inputName);
var parser =
inputName.Contains("500-liquid")
? (Func <string, string, ParserOptions?, LexerOptions?, Template>)Template.ParseLiquid
: Template.Parse;
var options = new LexerOptions();
if (inputName.Contains("liquid"))
{
options.Lang = ScriptLang.Liquid;
}
else if (inputName.Contains("scientific"))
{
options.Lang = ScriptLang.Scientific;
}
var template = parser(templateSource, inputName, default, options);
public Lexer(string input, LexerOptions options = LexerOptions.IgnoreComments | LexerOptions.IgnoreWhitespace)
{
Input = input.Trim();
Options = options;
}
public static void AssertTemplate(string expected, string input, ScriptLang lang = ScriptLang.Default, bool isRoundtripTest = false, bool supportExactRoundtrip = true, object model = null, bool specialLiquid = false, bool expectParsingErrorForRountrip = false, bool supportRoundTrip = true)
{
bool isLiquid = lang == ScriptLang.Liquid;
var parserOptions = new ParserOptions()
{
LiquidFunctionsToScriban = isLiquid,
};
var lexerOptions = new LexerOptions()
{
Lang = lang
};
if (isRoundtripTest)
{
lexerOptions.KeepTrivia = true;
}
if (specialLiquid)
{
parserOptions.ExpressionDepthLimit = 500;
}
#if EnableTokensOutput
{
Console.WriteLine("Tokens");
Console.WriteLine("======================================");
var lexer = new Lexer(input, options: lexerOptions);
foreach (var token in lexer)
{
Console.WriteLine($"{token.Type}: {token.GetText(input)}");
}
Console.WriteLine();
}
#endif
string roundtripText = null;
// We loop first on input text, then on roundtrip
while (true)
{
bool isRoundtrip = roundtripText != null;
bool hasErrors = false;
bool hasException = false;
if (isRoundtrip)
{
Console.WriteLine("Roundtrip");
Console.WriteLine("======================================");
Console.WriteLine(roundtripText);
lexerOptions.Lang = lang == ScriptLang.Scientific ? lang : ScriptLang.Default;
if (!isLiquid && supportExactRoundtrip)
{
Console.WriteLine("Checking Exact Roundtrip - Input");
Console.WriteLine("======================================");
TextAssert.AreEqual(input, roundtripText);
}
input = roundtripText;
}
else
{
Console.WriteLine("Input");
Console.WriteLine("======================================");
Console.WriteLine(input);
}
var template = Template.Parse(input, "text", parserOptions, lexerOptions);
var result = string.Empty;
var resultAsync = string.Empty;
if (template.HasErrors)
{
hasErrors = true;
for (int i = 0; i < template.Messages.Count; i++)
{
var message = template.Messages[i];
if (i > 0)
{
result += "\n";
}
result += message;
}
if (specialLiquid && !isRoundtrip)
{
throw new InvalidOperationException("Parser errors: " + result);
}
}
else
{
if (isRoundtripTest)
{
result = template.ToText();
}
else
{
Assert.NotNull(template.Page);
if (!isRoundtrip)
{
// Dumps the roundtrip version
var lexerOptionsForTrivia = lexerOptions;
lexerOptionsForTrivia.KeepTrivia = true;
var templateWithTrivia = Template.Parse(input, "input", parserOptions, lexerOptionsForTrivia);
roundtripText = templateWithTrivia.ToText();
}
try
{
// Setup a default model context for the tests
if (model == null)
{
var scriptObj = new ScriptObject
{
["page"] = new ScriptObject {
["title"] = "This is a title"
},
["user"] = new ScriptObject {
["name"] = "John"
},
["product"] = new ScriptObject {
["title"] = "Orange", ["type"] = "fruit"
},
["products"] = new ScriptArray()
{
new ScriptObject {
["title"] = "Orange", ["type"] = "fruit"
},
new ScriptObject {
["title"] = "Banana", ["type"] = "fruit"
},
new ScriptObject {
["title"] = "Apple", ["type"] = "fruit"
},
new ScriptObject {
["title"] = "Computer", ["type"] = "electronics"
},
new ScriptObject {
["title"] = "Mobile Phone", ["type"] = "electronics"
},
new ScriptObject {
["title"] = "Table", ["type"] = "furniture"
},
new ScriptObject {
["title"] = "Sofa", ["type"] = "furniture"
},
}
};
scriptObj.Import(typeof(SpecialFunctionProvider));
model = scriptObj;
}
// Render sync
{
var context = NewTemplateContext(lang);
context.PushOutput(new TextWriterOutput(new StringWriter()
{
NewLine = "\n"
}));
var contextObj = new ScriptObject();
contextObj.Import(model);
context.PushGlobal(contextObj);
result = template.Render(context);
}
// Render async
{
var asyncContext = NewTemplateContext(lang);
asyncContext.PushOutput(new TextWriterOutput(new StringWriter()
{
NewLine = "\n"
}));
var contextObj = new ScriptObject();
contextObj.Import(model);
asyncContext.PushGlobal(contextObj);
resultAsync = template.RenderAsync(asyncContext).Result;
}
}
catch (Exception exception)
{
hasException = true;
if (specialLiquid)
{
throw;
}
else
{
result = GetReason(exception);
}
}
}
}
var testContext = isRoundtrip ? "Roundtrip - " : String.Empty;
Console.WriteLine($"{testContext}Result");
Console.WriteLine("======================================");
Console.WriteLine(result);
Console.WriteLine($"{testContext}Expected");
Console.WriteLine("======================================");
Console.WriteLine(expected);
if (isRoundtrip && expectParsingErrorForRountrip)
{
Assert.True(hasErrors, "The roundtrip test is expecting an error");
Assert.AreNotEqual(expected, result);
}
else
{
TextAssert.AreEqual(expected, result);
}
if (!isRoundtrip && !isRoundtripTest && !hasErrors && !hasException)
{
Console.WriteLine("Checking async");
Console.WriteLine("======================================");
TextAssert.AreEqual(expected, resultAsync);
}
if (!supportRoundTrip || isRoundtripTest || isRoundtrip || hasErrors)
{
break;
}
}
}
public Lexer(LexerOptions options)
{
_options = options;
}
public idLexer(LexerOptions options)
: this()
{
_options = options;
}
public Lexer2254(LexerOptions options = null)
{
Options = options ?? new LexerOptions();
Tokens = new Tokens2254();
}
public object Invoke(TemplateContext context, ScriptNode callerContext, ScriptArray arguments, ScriptBlockStatement blockStatement)
{
if (arguments.Count == 0)
{
throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.BadFunctionInvokeArgEmpty, "include"));
}
string templateName = context.ToString(callerContext.Span, arguments[0]);
// If template name is empty, throw an exception
if (string.IsNullOrEmpty(templateName))
{
// In a liquid template context, we let an include to continue without failing
if (context is LiquidTemplateContext)
{
return(null);
}
throw new ScriptRuntimeException(callerContext.Span, RS.IncludeNameRequired);
}
ITemplateLoader templateLoader = context.TemplateLoader;
if (templateLoader == null)
{
throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.NoTemplateLoader, "include"));
}
string templatePath;
try
{
templatePath = templateLoader.GetPath(context, callerContext.Span, templateName);
}
catch (Exception ex) when(!(ex is ScriptRuntimeException))
{
throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.IncludeResolvePathError, templateName), ex);
}
// If template name is empty, throw an exception
if (templatePath == null)
{
throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.IncludePathNullError, templateName));
}
// Compute a new parameters for the include
ScriptArray newParameters = new ScriptArray(arguments.Count - 1);
for (int i = 1; i < arguments.Count; i++)
{
newParameters[i] = arguments[i];
}
context.SetValue(ScriptVariable.Arguments, newParameters, true);
Template template;
if (!context.CachedTemplates.TryGetValue(templatePath, out template))
{
string templateText;
try
{
templateText = templateLoader.Load(context, callerContext.Span, templatePath);
}
catch (Exception ex) when(!(ex is ScriptRuntimeException))
{
throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.IncludeLoadError, templateName, templatePath), ex);
}
if (templateText == null)
{
throw new ScriptRuntimeException(callerContext.Span, string.Format(RS.IncludeContentEmpty, templateName, templatePath));
}
// Clone parser options
ParserOptions parserOptions = context.TemplateLoaderParserOptions;
LexerOptions lexerOptions = context.TemplateLoaderLexerOptions;
template = Template.Parse(templateText, templatePath, parserOptions, lexerOptions);
// If the template has any errors, throw an exception
if (template.HasErrors)
{
throw new ScriptParserRuntimeException(callerContext.Span, string.Format(RS.IncludeParseError, templateName, templatePath), template.Messages);
}
context.CachedTemplates.Add(templatePath, template);
}
// Make sure that we cannot recursively include a template
context.PushOutput();
object result = null;
try
{
context.EnterRecursive(callerContext);
result = template.Render(context);
context.ExitRecursive(callerContext);
}
finally
{
context.PopOutput();
}
return(result);
}
public Lexer(ICollection<IFilter> filters, LexerOptions options)
{
_options = options;
_filters = filters;
}
public static TemplateCompareResult TestTemplate(string expected, string input, bool isLiquid = false, bool isRoundtripTest = false, bool supportExactRoundtrip = true, object model = null, bool specialLiquid = false)
{
var parserOptions = new ParserOptions()
{
ConvertLiquidFunctions = isLiquid
};
var lexerOptions = new LexerOptions()
{
Mode = isLiquid ? ScriptMode.Liquid : ScriptMode.Default
};
if (isRoundtripTest)
{
lexerOptions.KeepTrivia = true;
}
if (specialLiquid)
{
parserOptions.ExpressionDepthLimit = 500;
}
List <string> tokensDesc = new List <string>();
var lexer = new Lexer(input, options: lexerOptions);
foreach (var token in lexer)
{
tokensDesc.Add($"{token.Type}: {token.GetText(input)}");
}
string roundtripText = null;
List <string> verboseMessage = new List <string>();
// We loop first on input text, then on rountrip
while (true)
{
bool isRoundtrip = roundtripText != null;
bool hasErrors = false;
if (isRoundtrip)
{
verboseMessage.Add("Rountrip");
verboseMessage.Add("======================================");
verboseMessage.Add(roundtripText);
lexerOptions.Mode = ScriptMode.Default;
if (lexerOptions.Mode == ScriptMode.Default && !isLiquid && supportExactRoundtrip)
{
verboseMessage.Add("Checking Exact Roundtrip - Input");
verboseMessage.Add("======================================");
EqualityCompareResult equalResult = TextAssert.Equal(input, roundtripText);
if (equalResult.VerboseMessage != null && equalResult.VerboseMessage.Count > 0)
{
verboseMessage.AddRange(equalResult.VerboseMessage);
}
if (equalResult.IsEqual == false)
{
return(new TemplateCompareResult()
{
Tokens = tokensDesc,
IsEqual = false,
VerboseMessage = verboseMessage,
DebugCode = "INPUT_ROUNDTRIP_TEXT_NOT_EQUAL"
});
}
}
input = roundtripText;
}
else
{
verboseMessage.Add("Input");
verboseMessage.Add("======================================");
verboseMessage.Add(input);
}
var template = Template.Parse(input, "text", parserOptions, lexerOptions);
var result = string.Empty;
if (template.HasErrors)
{
hasErrors = true;
for (int i = 0; i < template.Messages.Count; i++)
{
var message = template.Messages[i];
if (i > 0)
{
result += "\n";
}
result += message;
}
if (specialLiquid)
{
throw new InvalidOperationException("Parser errors: " + result);
}
}
else
{
if (isRoundtripTest)
{
result = template.ToText();
}
else
{
if (template.Page == null)
{
return(new TemplateCompareResult()
{
Tokens = tokensDesc,
IsEqual = false,
VerboseMessage = verboseMessage,
DebugCode = "PAGE_IS_NULL",
Result = result,
Expected = expected
});
}
if (!isRoundtrip)
{
// Dumps the rountrip version
var lexerOptionsForTrivia = lexerOptions;
lexerOptionsForTrivia.KeepTrivia = true;
var templateWithTrivia = Template.Parse(input, "input", parserOptions, lexerOptionsForTrivia);
roundtripText = templateWithTrivia.ToText();
}
try
{
// Setup a default model context for the tests
if (model == null)
{
var scriptObj = new ScriptObject
{
["page"] = new ScriptObject {
["title"] = "This is a title"
},
["user"] = new ScriptObject {
["name"] = "John"
},
["product"] = new ScriptObject {
["title"] = "Orange", ["type"] = "fruit"
},
["products"] = new ScriptArray()
{
new ScriptObject {
["title"] = "Orange", ["type"] = "fruit"
},
new ScriptObject {
["title"] = "Banana", ["type"] = "fruit"
},
new ScriptObject {
["title"] = "Apple", ["type"] = "fruit"
},
new ScriptObject {
["title"] = "Computer", ["type"] = "electronics"
},
new ScriptObject {
["title"] = "Mobile Phone", ["type"] = "electronics"
},
new ScriptObject {
["title"] = "Table", ["type"] = "furniture"
},
new ScriptObject {
["title"] = "Sofa", ["type"] = "furniture"
},
}
};
scriptObj.Import(typeof(SpecialFunctionProvider));
model = scriptObj;
}
var context = isLiquid
? new LiquidTemplateContext()
{
TemplateLoader = new LiquidCustomTemplateLoader()
}
: new TemplateContext()
{
TemplateLoader = new CustomTemplateLoader()
};
// We use a custom output to make sure that all output is using the "\n"
context.PushOutput(new TextWriterOutput(new StringWriter()
{
NewLine = "\n"
}));
var contextObj = new ScriptObject();
contextObj.Import(model);
context.PushGlobal(contextObj);
result = template.Render(context);
}
catch (Exception exception)
{
if (specialLiquid)
{
throw;
}
else
{
result = GetReason(exception);
}
}
}
}
var testContext = isRoundtrip ? "Roundtrip - " : String.Empty;
verboseMessage.Add($"{testContext}Result");
verboseMessage.Add("======================================");
verboseMessage.Add(result);
verboseMessage.Add($"{testContext}Expected");
verboseMessage.Add("======================================");
verboseMessage.Add(expected);
EqualityCompareResult testEqualResult = TextAssert.Equal(expected, result);
if (testEqualResult.VerboseMessage != null && testEqualResult.VerboseMessage.Count > 0)
{
verboseMessage.AddRange(testEqualResult.VerboseMessage);
}
if (testEqualResult.IsEqual == false)
{
return(new TemplateCompareResult
{
IsEqual = false,
VerboseMessage = verboseMessage,
DebugCode = "UNKNOWN",
Tokens = tokensDesc,
Result = result,
Expected = expected
});
}
if (isRoundtripTest || isRoundtrip || hasErrors)
{
break;
}
}
return(new TemplateCompareResult
{
IsEqual = true,
VerboseMessage = verboseMessage,
DebugCode = "",
Tokens = tokensDesc
});
}
public KalkEngine() : base(new KalkObjectWithAlias())
{
KalkSettings.Initialize();
KalkEngineFolder = AppContext.BaseDirectory;
// Enforce UTF8 encoding
Console.OutputEncoding = Encoding.UTF8;
EnableEngineOutput = true;
EchoEnabled = true;
DisplayVersion = true;
CurrentDisplay = KalkDisplayMode.Standard;
KalkUserFolder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile, Environment.SpecialFolderOption.DoNotVerify), ".kalk");
HighlightOutput = new ConsoleText();
InputReader = Console.In;
OutputWriter = Console.Out;
ErrorWriter = Console.Error;
IsOutputSupportHighlighting = ConsoleHelper.SupportEscapeSequences;
// Fetch version
var assemblyInfoVersion = (AssemblyInformationalVersionAttribute)typeof(KalkEngine).Assembly.GetCustomAttribute(typeof(AssemblyInformationalVersionAttribute));
Version = assemblyInfoVersion.InformationalVersion;
Builtins = BuiltinObject;
((KalkObjectWithAlias)Builtins).Engine = this;
Units = new KalkUnits(this);
Shortcuts = new KalkShortcuts();
Aliases = new KalkAliases();
_currentShortcutKeyMap = Shortcuts.ShortcutKeyMap;
_completionMatchingList = new List <string>();
Config = new KalkConfig();
Variables = new ScriptVariables(this);
Descriptors = new Dictionary <string, KalkDescriptor>();
EnableRelaxedMemberAccess = false;
_modules = new Dictionary <Type, KalkModule>();
TryConverters = new List <TryToObjectDelegate>();
ErrorForStatementFunctionAsExpression = true;
StrictVariables = true;
UseScientific = true;
LoopLimit = int.MaxValue; // no limits for loops
RecursiveLimit = int.MaxValue; // no limits (still guarded by Scriban)
// Setup default clipboard methods
_localClipboard = string.Empty;
GetClipboardText = GetClipboardTextImpl;
SetClipboardText = SetClipboardTextImpl;
_cancellationTokenSource = new CancellationTokenSource();
PushGlobal(Units);
PushGlobal(Variables);
_parserOptions = new ParserOptions();
_lexerOptions = new LexerOptions()
{
KeepTrivia = true,
Mode = ScriptMode.ScriptOnly,
Lang = ScriptLang.Scientific
};
_lexerInterpolatedOptions = new LexerOptions()
{
KeepTrivia = true,
Mode = ScriptMode.Default,
Lang = ScriptLang.Scientific
};
_tempOutputHighlight = new ConsoleText();
// Init last result with 0
_lastResult = 0;
HistoryList = new List <string>();
_isInitializing = true;
RegisterFunctions();
_isInitializing = false;
}
public Lexer(string input, LexerOptions options = LexerOptions.IgnoreComments | LexerOptions.IgnoreWhitespace)
{
this.Input = input;
this.Options = options;
}
