public static DoLoopStaNode Parse(TokenString tStr, ref int index) { int startIndex = index; if (!tStr.Match(index, TokenType.doStatement, TokenType.cBraceOpen)) { return(null); } index++; StatementNode body = Sta.Parse(tStr, ref index); if (!tStr.Match(index, TokenType.whileStatement, TokenType.rBraceOpen)) { index = startIndex; return(null); } index++; ExpressionNode condition = Exp.Parse(tStr.GetRangeInBrackets(ref index)); if (!tStr.Match(index, TokenType.lineEnd)) { index = startIndex; return(null); } index++; DoLoopStaNode node = new DoLoopStaNode(condition, body); return(node); }
public void TwoLetterTokensDoNotMatchIfWordTokenDontContainTheLetter() { TokenString testCase = new TokenString("(ab)"); TokenString word = new TokenString("c"); Assert.IsFalse(testCase.Matches(word)); }
public void TokenStringBuilder_WithTokens() { // arrange const string configKey = "configKey"; const string configValue = "A"; const string tokenKey = "season"; const string raw = " assets/{{configKey}}_{{season}}_{{ invalid }}.png "; var context = new GenericTokenContext(modId => false); context.Save(new ImmutableToken(configKey, new InvariantHashSet { configValue })); context.Save(new ImmutableToken(tokenKey, new InvariantHashSet { "A" })); // act TokenString tokenStr = new TokenString(raw, context); IContextualState diagnosticState = tokenStr.GetDiagnosticState(); // assert tokenStr.Raw.Should().Be("assets/{{configKey}}_{{season}}_{{invalid}}.png"); tokenStr.GetTokenPlaceholders(recursive: false).Should().HaveCount(3); tokenStr.GetTokenPlaceholders(recursive: false).Select(name => name.Text).Should().BeEquivalentTo(new[] { "{{configKey}}", "{{season}}", "{{invalid}}" }); tokenStr.IsReady.Should().BeFalse(); tokenStr.HasAnyTokens.Should().BeTrue(); tokenStr.IsSingleTokenOnly.Should().BeFalse(); diagnosticState.IsReady.Should().BeFalse(); diagnosticState.UnreadyTokens.Should().BeEmpty(); diagnosticState.InvalidTokens.Should().HaveCount(1).And.BeEquivalentTo("invalid"); diagnosticState.Errors.Should().BeEmpty(); }
/********* ** Public methods *********/ /// <summary>Construct an instance.</summary> /// <param name="key">The unique key for the entry in the data file.</param> /// <param name="value">The entry value to set.</param> public EditDataPatchRecord(TokenString key, TokenisableJToken value) { this.Key = key; this.Value = value; this.ContextualValues = new IContextual[] { key, value }.Where(p => p != null).ToArray(); }
public static CompoundStaNode Parse(TokenString tStr, ref int index) { if (!tStr.Match(index, TokenType.cBraceOpen)) { return(null); } index++; CompoundStaNode node = new CompoundStaNode(); while (!tStr.Match(index, TokenType.cBraceClose)) { var statementNode = Sta.Parse(tStr, ref index); if (statementNode != null) { node.AddStatement(statementNode); } else { index++; } } index++; return(node); }
public void TokenStringBuilder_WithTokens() { // arrange const string configKey = "configKey"; const string configValue = "A"; const string raw = " assets/{{configKey}}_{{season}}_{{invalid}}.png "; // act TokenStringBuilder builder = new TokenStringBuilder(raw, new InvariantDictionary <ConfigField> { [configKey] = new ConfigField(allowValues: new InvariantHashSet(configValue), allowBlank: false, allowMultiple: false, defaultValues: new InvariantHashSet { configValue }) { Value = new InvariantHashSet(configValue) } }); TokenString tokenStr = builder.Build(); // assert builder builder.RawValue.Should().Be(raw); builder.ConditionTokens.Should().HaveCount(1).And.BeEquivalentTo(ConditionKey.Season); builder.ConfigTokens.Should().HaveCount(1).And.BeEquivalentTo(configKey); builder.InvalidTokens.Should().HaveCount(1).And.BeEquivalentTo("invalid"); builder.HasAnyTokens.Should().BeTrue(); // assert token string tokenStr.Raw.Should().Be(raw.Replace("{{" + configKey + "}}", configValue), $"the config token should be substituted when the {nameof(TokenString)} is built"); tokenStr.ConditionTokens.Should().HaveCount(1).And.BeEquivalentTo(ConditionKey.Season); }
public void TwoSetsOfTokensMatchIfBothMatch() { TokenString testCase = new TokenString("(ab)(cd)"); TokenString word = new TokenString("bc"); Assert.IsTrue(testCase.Matches(word)); }
public static NamespaceNode Parse(TokenString tStr, ref int index) { int startIndex = index; if (tStr[index].Type != TokenType.@namespace) { return(null); } index++; var nameExpression = Exp.Parse(tStr.GetRangeUntil(ref index, TokenType.cBraceOpen)); if (nameExpression == null) { index = startIndex; return(null); } NamespaceNode node = new NamespaceNode(nameExpression); // Get internals var classesRange = tStr.GetRangeInBrackets(ref index); int cRangeIndex = 0; while (cRangeIndex < classesRange.Count) { Token currentToken = classesRange[cRangeIndex]; if (currentToken.Type == TokenType.accessor || currentToken.Type == TokenType.@class) { ClassNode classNode = ClassNode.Parse(classesRange, ref cRangeIndex); if (classNode == null) { break; } node.AddClass(classNode); } else if (currentToken.Type == TokenType.@namespace) { NamespaceNode namespaceNode = new NamespaceNode(Exp.Parse(tStr.GetRangeUntil(ref index, TokenType.cBraceOpen))); if (namespaceNode == null) { break; } node.AddNamespace(namespaceNode); } else { cRangeIndex++; } } index += cRangeIndex; return(node); }
/// <summary>Parse a string which can contain tokens, and validate that it's valid.</summary> /// <param name="rawValue">The raw string which may contain tokens.</param> /// <param name="tokenContext">The tokens available for this content pack.</param> /// <param name="migrator">The migrator which validates and migrates content pack data.</param> /// <param name="error">An error phrase indicating why parsing failed (if applicable).</param> /// <param name="parsed">The parsed value.</param> private bool TryParseStringTokens(string rawValue, IContext tokenContext, IMigration migrator, out string error, out ITokenString parsed) { // parse parsed = new TokenString(rawValue, tokenContext); if (!migrator.TryMigrate(parsed, out error)) { return(false); } // validate unknown tokens IContextualState state = parsed.GetDiagnosticState(); if (state.InvalidTokens.Any()) { error = $"found unknown tokens ({string.Join(", ", state.InvalidTokens.OrderBy(p => p))})"; parsed = null; return(false); } // validate tokens foreach (LexTokenToken lexToken in parsed.GetTokenPlaceholders(recursive: false)) { IToken token = tokenContext.GetToken(lexToken.Name, enforceContext: false); if (token == null) { error = $"'{lexToken}' can't be used as a token because that token could not be found."; // should never happen parsed = null; return(false); } } // looks OK error = null; return(true); }
/// <summary>Parse a string which can contain tokens, and validate that it's valid.</summary> /// <param name="rawValue">The raw string which may contain tokens.</param> /// <param name="assumeModIds">Mod IDs to assume are installed for purposes of token validation.</param> /// <param name="path">The path to the value from the root content file.</param> /// <param name="error">An error phrase indicating why parsing failed (if applicable).</param> /// <param name="parsed">The parsed value.</param> public bool TryParseString(string rawValue, InvariantHashSet assumeModIds, LogPathBuilder path, out string error, out IManagedTokenString parsed) { // parse lexical bits var bits = new Lexer().ParseBits(rawValue, impliedBraces: false).ToArray(); foreach (ILexToken bit in bits) { if (!this.Migrator.TryMigrate(bit, out error)) { parsed = null; return(false); } } // get token string parsed = new TokenString(bits, this.Context, path); if (!this.Migrator.TryMigrate(parsed, out error)) { return(false); } // validate tokens foreach (LexTokenToken lexToken in parsed.GetTokenPlaceholders(recursive: false)) { if (!this.TryValidateToken(lexToken, assumeModIds, out error)) { return(false); } } // looks OK error = null; return(true); }
public static ConditionalStaNode Parse(TokenString tStr, ref int index) { int startIndex = index; if (!tStr.Match(index, TokenType.ifStatement, TokenType.rBraceOpen)) { return(null); } index++; ExpressionNode condition = Exp.Parse(tStr.GetRangeInBrackets(ref index)); StatementNode ifTrue = Sta.Parse(tStr, ref index); StatementNode ifFalse = null; if (tStr.Match(index, TokenType.elseStatement)) { index++; ifFalse = Sta.Parse(tStr, ref index); } ConditionalStaNode node = new ConditionalStaNode(condition, ifTrue, ifFalse); return(node); }
/// <summary> /// Returns the list of tokens corresponding to the {&variable} to replace /// </summary> private List <Token> GetPreProcVariableTokens(TokenString previousTokenString, Token bracketToken, string value) { // do we have a definition for the var? if (value == null) { return(null); } List <Token> valueTokens; // Parse it if (previousTokenString != null && !string.IsNullOrEmpty(previousTokenString.Value)) { valueTokens = new ProTokenizer(value, previousTokenString.Value[0] == '"', previousTokenString.Value[0] == '\'').GetTokensList.ToList(); } else { valueTokens = new ProTokenizer(value).GetTokensList.ToList(); } // Remove EOF and change owner number List <Token> copiedTokens = new List <Token>(); for (int i = 0; i < valueTokens.Count - 1; i++) { var copiedToken = valueTokens[i].Copy(bracketToken.Line, bracketToken.Column, bracketToken.StartPosition, bracketToken.EndPosition); copiedToken.OwnerNumber = bracketToken.OwnerNumber; copiedTokens.Add(copiedToken); } return(copiedTokens); }
public static StatementNode Parse(TokenString tStr, ref int index) { switch (tStr[index].Type) { case TokenType.cBraceOpen: return(CompoundStaNode.Parse(tStr, ref index)); case TokenType.ifStatement: return(ConditionalStaNode.Parse(tStr, ref index)); case TokenType.doStatement: return(DoLoopStaNode.Parse(tStr, ref index)); case TokenType.whileStatement: return(WhileLoopStaNode.Parse(tStr, ref index)); case TokenType.forStatement: return(ForLoopStaNode.Parse(tStr, ref index)); case TokenType.returnStatement: return(ReturnStaNode.Parse(tStr, ref index)); case TokenType.lineEnd: return(NopStaNode.Parse(tStr, ref index)); default: return(ExpressionStaNode.Parse(tStr, ref index)); } }
public void TwoSetsOfTokensDontMatchIfNotBothMatch() { TokenString testCase = new TokenString("(ab)(cd)"); TokenString word = new TokenString("bb"); Assert.IsFalse(testCase.Matches(word)); }
/// <summary>Parse a string which can contain tokens, and validate that it's valid.</summary> /// <param name="rawValue">The raw string which may contain tokens.</param> /// <param name="tokenContext">The tokens available for this content pack.</param> /// <param name="error">An error phrase indicating why parsing failed (if applicable).</param> /// <param name="parsed">The parsed value.</param> private bool TryParseTokenString(string rawValue, IContext tokenContext, out string error, out TokenString parsed) { // parse parsed = new TokenString(rawValue, tokenContext); // validate unknown tokens if (parsed.InvalidTokens.Any()) { error = $"found unknown tokens ({string.Join(", ", parsed.InvalidTokens.OrderBy(p => p))})"; parsed = null; return(false); } // validate tokens foreach (TokenName tokenName in parsed.Tokens) { IToken token = tokenContext.GetToken(tokenName, enforceContext: false); if (token == null) { error = $"{{{{{tokenName}}}}} can't be used as a token because that token could not be found."; // should never happen parsed = null; return(false); } if (token.CanHaveMultipleValues(tokenName)) { error = $"{{{{{tokenName}}}}} can't be used as a token because it can have multiple values."; parsed = null; return(false); } } // looks OK error = null; return(true); }
public BinaryExpNode(string @operator, TokenString left, TokenString right) { Operator = @operator; Left = Exp.Parse(left); Right = Exp.Parse(right); }
public void TokenStringBuilder_WithTokens() { // arrange const string configKey = "configKey"; const string configValue = "A"; const string tokenKey = "season"; const string raw = " assets/{{configKey}}_{{season}}_{{invalid}}.png "; var context = new GenericTokenContext(); context.Save(new ImmutableToken(configKey, new InvariantHashSet { configValue })); context.Save(new ImmutableToken(tokenKey, new InvariantHashSet { "A" })); // act TokenString tokenStr = new TokenString(raw, context); // assert tokenStr.Raw.Should().Be(raw.Trim()); tokenStr.Tokens.Should().HaveCount(2); tokenStr.Tokens.Select(name => name.ToString()).Should().BeEquivalentTo(new[] { configKey, tokenKey }); tokenStr.InvalidTokens.Should().HaveCount(1).And.BeEquivalentTo("invalid"); tokenStr.HasAnyTokens.Should().BeTrue(); tokenStr.IsSingleTokenOnly.Should().BeFalse(); }
public void CompareIdenticalOneLetterStrings() { TokenString testCase = new TokenString("a"); TokenString word = new TokenString("a"); Assert.IsTrue(testCase.Matches(word)); }
public void TwoLetterTokensMatchWordTokenIfItContainsTheLetter() { TokenString testCase = new TokenString("(ab)"); TokenString word = new TokenString("a"); Assert.IsTrue(testCase.Matches(word)); }
public void TokenStringBuilder_WithSingleToken(string raw, string parsed) { // arrange const string configKey = "tokenKey"; var context = new GenericTokenContext(modId => false); context.Save(new ImmutableToken(configKey, new InvariantHashSet { "value" })); // act TokenString tokenStr = new TokenString(raw, context); IContextualState diagnosticState = tokenStr.GetDiagnosticState(); // assert tokenStr.Raw.Should().Be(parsed); tokenStr.GetTokenPlaceholders(recursive: false).Should().HaveCount(1); tokenStr.GetTokenPlaceholders(recursive: false).Select(p => p.Text).Should().BeEquivalentTo("{{" + configKey + "}}"); tokenStr.HasAnyTokens.Should().BeTrue(); tokenStr.IsSingleTokenOnly.Should().BeTrue(); diagnosticState.IsReady.Should().BeTrue(); diagnosticState.UnreadyTokens.Should().BeEmpty(); diagnosticState.InvalidTokens.Should().BeEmpty(); diagnosticState.Errors.Should().BeEmpty(); }
/********* ** Public methods *********/ /// <summary>Construct an instance.</summary> /// <param name="logName">A unique name for this patch shown in log messages.</param> /// <param name="contentPack">The content pack which requested the patch.</param> /// <param name="assetName">The normalised asset name to intercept.</param> /// <param name="conditions">The conditions which determine whether this patch should be applied.</param> /// <param name="records">The data records to edit.</param> /// <param name="fields">The data fields to edit.</param> /// <param name="monitor">Encapsulates monitoring and logging.</param> /// <param name="normaliseAssetName">Normalise an asset name.</param> public EditDataPatch(string logName, ManagedContentPack contentPack, TokenString assetName, ConditionDictionary conditions, IDictionary <string, string> records, IDictionary <string, IDictionary <int, string> > fields, IMonitor monitor, Func <string, string> normaliseAssetName) : base(logName, PatchType.EditData, contentPack, assetName, conditions, normaliseAssetName) { this.Records = records; this.Fields = fields; this.Monitor = monitor; }
public Task <string> Detokenize(TokenString token, Guid correlationId, CancellationToken cancellationToken) { if (token == null) { return(Task.FromResult((string)null)); } return(_tokenizeClient.Detokenize(token.RawToken(), correlationId)); }
/********* ** Public methods *********/ /// <summary>Construct an instance.</summary> /// <param name="logName">A unique name for this patch shown in log messages.</param> /// <param name="contentPack">The content pack which requested the patch.</param> /// <param name="assetName">The normalised asset name to intercept.</param> /// <param name="conditions">The conditions which determine whether this patch should be applied.</param> /// <param name="records">The data records to edit.</param> /// <param name="fields">The data fields to edit.</param> /// <param name="monitor">Encapsulates monitoring and logging.</param> /// <param name="normaliseAssetName">Normalise an asset name.</param> public EditDataPatch(string logName, ManagedContentPack contentPack, TokenString assetName, ConditionDictionary conditions, IEnumerable <EditDataPatchRecord> records, IEnumerable <EditDataPatchField> fields, IMonitor monitor, Func <string, string> normaliseAssetName) : base(logName, PatchType.EditData, contentPack, assetName, conditions, normaliseAssetName) { this.Records = records.ToArray(); this.Fields = fields.ToArray(); this.Monitor = monitor; this.MutableTokenStrings = this.GetTokenStrings(this.Records, this.Fields).Where(str => str.Tokens.Any()).ToArray(); }
/********* ** Public methods *********/ /// <summary>Construct an instance.</summary> /// <param name="entryKey">The unique key for the entry in the data file.</param> /// <param name="fieldKey">The field number to change.</param> /// <param name="value">The entry value to set.</param> public EditDataPatchField(TokenString entryKey, TokenString fieldKey, TokenisableJToken value) { this.EntryKey = entryKey; this.FieldKey = fieldKey; this.Value = value; this.ContextualValues = new IContextual[] { entryKey, fieldKey, value }.Where(p => p != null).ToArray(); }
public void Visit(TokenString tok) { if (tok.Line < FromLine || tok.Line > ToLine) { return; } //Npp.StyleText((int)TextStyle.String, tok.StartPosition, tok.EndPosition); }
public static void Main(string[] args) { double total = 0.0; for (int i = 0; i <= 2; i++) { Stopwatch sw = new Stopwatch(); sw.Start(); var items = TokenString.NewTokenString(113); var b = items.Value.IsValid(113); Console.WriteLine(items.HexValue.TokenFormat("10-4-6-10-3-7")); Console.WriteLine(b); sw.Stop(); //Console.WriteLine("Cycle {0} completed at {1:0.00} ms with {2} items", (i + 1), sw.Elapsed.TotalMilliseconds, items.Count); if (i > 0) { total += sw.Elapsed.TotalMilliseconds; } //foreach (var q in items) { // Console.WriteLine(q.Value); //} } Console.WriteLine("Total = {0:#,##0.00}", total / 3); //foreach (var x in TokenString.AlphaNumericAny) { // Console.WriteLine(x); //} //var lt = DateTime.Now.Ticks; //var ts = TokenString.NewTokenString(); //Console.WriteLine(ts.HexValue); //var tb = Encoding.UTF8.GetBytes(ts); //var xb = BitConverter.GetBytes(lt); //foreach (var x in xb) { // Console.WriteLine(x); //} //sw.Reset(); //sw.Start(); //Parallel.For(0, items.Count, i => { // if (items[i] != "~###~-#####-~####") { // Console.WriteLine("Failed Validation."); // } //}); //sw.Stop(); //Console.WriteLine("{0} items", items.Count()); //Console.WriteLine("Validation {0} ms", sw.Elapsed.TotalMilliseconds); Console.Write("Press any key to continue.... "); Console.ReadKey(true); }
/********* ** Public methods *********/ /// <summary>Construct an instance.</summary> /// <param name="logName">A unique name for this patch shown in log messages.</param> /// <param name="assetLoader">Handles loading assets from content packs.</param> /// <param name="contentPack">The content pack which requested the patch.</param> /// <param name="assetName">The normalised asset name to intercept.</param> /// <param name="conditions">The conditions which determine whether this patch should be applied.</param> /// <param name="fromLocalAsset">The asset key to load from the content pack instead.</param> /// <param name="fromArea">The sprite area from which to read an image.</param> /// <param name="toArea">The sprite area to overwrite.</param> /// <param name="patchMode">Indicates how the image should be patched.</param> /// <param name="monitor">Encapsulates monitoring and logging.</param> /// <param name="normaliseAssetName">Normalise an asset name.</param> public EditImagePatch(string logName, AssetLoader assetLoader, IContentPack contentPack, TokenString assetName, ConditionDictionary conditions, TokenString fromLocalAsset, Rectangle fromArea, Rectangle toArea, PatchMode patchMode, IMonitor monitor, Func <string, string> normaliseAssetName) : base(logName, PatchType.EditImage, assetLoader, contentPack, assetName, conditions, normaliseAssetName) { this.FromLocalAsset = fromLocalAsset; this.FromArea = fromArea != Rectangle.Empty ? fromArea : null as Rectangle?; this.ToArea = toArea != Rectangle.Empty ? toArea : null as Rectangle?; this.PatchMode = patchMode; this.Monitor = monitor; }
/// <summary>Parse a boolean <see cref="PatchConfig.Enabled"/> value from a string which can contain tokens, and validate that it's valid.</summary> /// <param name="rawValue">The raw string which may contain tokens.</param> /// <param name="tokenContext">The tokens available for this content pack.</param> /// <param name="migrator">The migrator which validates and migrates content pack data.</param> /// <param name="error">An error phrase indicating why parsing failed (if applicable).</param> /// <param name="parsed">The parsed value.</param> private bool TryParseEnabled(string rawValue, IContext tokenContext, IMigration migrator, out string error, out bool parsed) { parsed = false; // analyse string if (!this.TryParseStringTokens(rawValue, tokenContext, migrator, out error, out ITokenString tokenString)) { return(false); } // validate & extract tokens string text = rawValue; if (tokenString.HasAnyTokens) { // only one token allowed if (!tokenString.IsSingleTokenOnly) { error = "can't be treated as a true/false value because it contains multiple tokens."; return(false); } // parse token LexTokenToken lexToken = tokenString.GetTokenPlaceholders(recursive: false).Single(); IToken token = tokenContext.GetToken(lexToken.Name, enforceContext: false); ITokenString input = new TokenString(lexToken.InputArg, tokenContext); // check token options InvariantHashSet allowedValues = token?.GetAllowedValues(input); if (token == null || token.IsMutable || !token.IsReady) { error = $"can only use static tokens in this field, consider using a {nameof(PatchConfig.When)} condition instead."; return(false); } if (allowedValues == null || !allowedValues.All(p => bool.TryParse(p, out _))) { error = "that token isn't restricted to 'true' or 'false'."; return(false); } if (token.CanHaveMultipleValues(input)) { error = "can't be treated as a true/false value because that token can have multiple values."; return(false); } text = token.GetValues(input).First(); } // parse text if (!bool.TryParse(text, out parsed)) { error = $"can't parse {tokenString.Raw} as a true/false value."; return(false); } return(true); }
public void StartsWithKeywordTest_NotFound() { string CommandLine = "Some_keyword in this line"; string Keyword = "Somekeyword"; bool expected = false; bool actual; actual = TokenString.StartsWithKeyword(CommandLine, Keyword); Assert.AreEqual(expected, actual); }
public static ExpressionStaNode Parse(TokenString tStr, ref int index) { var expression = Exp.Parse(tStr.GetRangeUntil(ref index, TokenType.lineEnd)); ExpressionStaNode node = new ExpressionStaNode(expression); index++; return node; }