public static DoLoopStaNode Parse(TokenString tStr, ref int index)
{
int startIndex = index;
if (!tStr.Match(index, TokenType.doStatement, TokenType.cBraceOpen))
{
return(null);
}
index++;
StatementNode body = Sta.Parse(tStr, ref index);
if (!tStr.Match(index, TokenType.whileStatement, TokenType.rBraceOpen))
{
index = startIndex;
return(null);
}
index++;
ExpressionNode condition = Exp.Parse(tStr.GetRangeInBrackets(ref index));
if (!tStr.Match(index, TokenType.lineEnd))
{
index = startIndex;
return(null);
}
index++;
DoLoopStaNode node = new DoLoopStaNode(condition, body);
return(node);
}
public void TwoLetterTokensDoNotMatchIfWordTokenDontContainTheLetter()
{
TokenString testCase = new TokenString("(ab)");
TokenString word = new TokenString("c");
Assert.IsFalse(testCase.Matches(word));
}
public void TokenStringBuilder_WithTokens()
{
// arrange
const string configKey = "configKey";
const string configValue = "A";
const string tokenKey = "season";
const string raw = " assets/{{configKey}}_{{season}}_{{ invalid }}.png ";
var context = new GenericTokenContext(modId => false);
context.Save(new ImmutableToken(configKey, new InvariantHashSet {
configValue
}));
context.Save(new ImmutableToken(tokenKey, new InvariantHashSet {
"A"
}));
// act
TokenString tokenStr = new TokenString(raw, context);
IContextualState diagnosticState = tokenStr.GetDiagnosticState();
// assert
tokenStr.Raw.Should().Be("assets/{{configKey}}_{{season}}_{{invalid}}.png");
tokenStr.GetTokenPlaceholders(recursive: false).Should().HaveCount(3);
tokenStr.GetTokenPlaceholders(recursive: false).Select(name => name.Text).Should().BeEquivalentTo(new[] { "{{configKey}}", "{{season}}", "{{invalid}}" });
tokenStr.IsReady.Should().BeFalse();
tokenStr.HasAnyTokens.Should().BeTrue();
tokenStr.IsSingleTokenOnly.Should().BeFalse();
diagnosticState.IsReady.Should().BeFalse();
diagnosticState.UnreadyTokens.Should().BeEmpty();
diagnosticState.InvalidTokens.Should().HaveCount(1).And.BeEquivalentTo("invalid");
diagnosticState.Errors.Should().BeEmpty();
}
/*********
** Public methods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="key">The unique key for the entry in the data file.</param>
/// <param name="value">The entry value to set.</param>
public EditDataPatchRecord(TokenString key, TokenisableJToken value)
{
this.Key = key;
this.Value = value;
this.ContextualValues = new IContextual[] { key, value }.Where(p => p != null).ToArray();
}
public static CompoundStaNode Parse(TokenString tStr, ref int index)
{
if (!tStr.Match(index, TokenType.cBraceOpen))
{
return(null);
}
index++;
CompoundStaNode node = new CompoundStaNode();
while (!tStr.Match(index, TokenType.cBraceClose))
{
var statementNode = Sta.Parse(tStr, ref index);
if (statementNode != null)
{
node.AddStatement(statementNode);
}
else
{
index++;
}
}
index++;
return(node);
}
public void TokenStringBuilder_WithTokens()
{
// arrange
const string configKey = "configKey";
const string configValue = "A";
const string raw = " assets/{{configKey}}_{{season}}_{{invalid}}.png ";
// act
TokenStringBuilder builder = new TokenStringBuilder(raw, new InvariantDictionary <ConfigField>
{
[configKey] = new ConfigField(allowValues: new InvariantHashSet(configValue), allowBlank: false, allowMultiple: false, defaultValues: new InvariantHashSet {
configValue
})
{
Value = new InvariantHashSet(configValue)
}
});
TokenString tokenStr = builder.Build();
// assert builder
builder.RawValue.Should().Be(raw);
builder.ConditionTokens.Should().HaveCount(1).And.BeEquivalentTo(ConditionKey.Season);
builder.ConfigTokens.Should().HaveCount(1).And.BeEquivalentTo(configKey);
builder.InvalidTokens.Should().HaveCount(1).And.BeEquivalentTo("invalid");
builder.HasAnyTokens.Should().BeTrue();
// assert token string
tokenStr.Raw.Should().Be(raw.Replace("{{" + configKey + "}}", configValue), $"the config token should be substituted when the {nameof(TokenString)} is built");
tokenStr.ConditionTokens.Should().HaveCount(1).And.BeEquivalentTo(ConditionKey.Season);
}
public void TwoSetsOfTokensMatchIfBothMatch()
{
TokenString testCase = new TokenString("(ab)(cd)");
TokenString word = new TokenString("bc");
Assert.IsTrue(testCase.Matches(word));
}
public static NamespaceNode Parse(TokenString tStr, ref int index)
{
int startIndex = index;
if (tStr[index].Type != TokenType.@namespace)
{
return(null);
}
index++;
var nameExpression = Exp.Parse(tStr.GetRangeUntil(ref index, TokenType.cBraceOpen));
if (nameExpression == null)
{
index = startIndex;
return(null);
}
NamespaceNode node = new NamespaceNode(nameExpression);
// Get internals
var classesRange = tStr.GetRangeInBrackets(ref index);
int cRangeIndex = 0;
while (cRangeIndex < classesRange.Count)
{
Token currentToken = classesRange[cRangeIndex];
if (currentToken.Type == TokenType.accessor || currentToken.Type == TokenType.@class)
{
ClassNode classNode = ClassNode.Parse(classesRange, ref cRangeIndex);
if (classNode == null)
{
break;
}
node.AddClass(classNode);
}
else if (currentToken.Type == TokenType.@namespace)
{
NamespaceNode namespaceNode = new NamespaceNode(Exp.Parse(tStr.GetRangeUntil(ref index, TokenType.cBraceOpen)));
if (namespaceNode == null)
{
break;
}
node.AddNamespace(namespaceNode);
}
else
{
cRangeIndex++;
}
}
index += cRangeIndex;
return(node);
}
/// <summary>Parse a string which can contain tokens, and validate that it's valid.</summary>
/// <param name="rawValue">The raw string which may contain tokens.</param>
/// <param name="tokenContext">The tokens available for this content pack.</param>
/// <param name="migrator">The migrator which validates and migrates content pack data.</param>
/// <param name="error">An error phrase indicating why parsing failed (if applicable).</param>
/// <param name="parsed">The parsed value.</param>
private bool TryParseStringTokens(string rawValue, IContext tokenContext, IMigration migrator, out string error, out ITokenString parsed)
{
// parse
parsed = new TokenString(rawValue, tokenContext);
if (!migrator.TryMigrate(parsed, out error))
{
return(false);
}
// validate unknown tokens
IContextualState state = parsed.GetDiagnosticState();
if (state.InvalidTokens.Any())
{
error = $"found unknown tokens ({string.Join(", ", state.InvalidTokens.OrderBy(p => p))})";
parsed = null;
return(false);
}
// validate tokens
foreach (LexTokenToken lexToken in parsed.GetTokenPlaceholders(recursive: false))
{
IToken token = tokenContext.GetToken(lexToken.Name, enforceContext: false);
if (token == null)
{
error = $"'{lexToken}' can't be used as a token because that token could not be found."; // should never happen
parsed = null;
return(false);
}
}
// looks OK
error = null;
return(true);
}
/// <summary>Parse a string which can contain tokens, and validate that it's valid.</summary>
/// <param name="rawValue">The raw string which may contain tokens.</param>
/// <param name="assumeModIds">Mod IDs to assume are installed for purposes of token validation.</param>
/// <param name="path">The path to the value from the root content file.</param>
/// <param name="error">An error phrase indicating why parsing failed (if applicable).</param>
/// <param name="parsed">The parsed value.</param>
public bool TryParseString(string rawValue, InvariantHashSet assumeModIds, LogPathBuilder path, out string error, out IManagedTokenString parsed)
{
// parse lexical bits
var bits = new Lexer().ParseBits(rawValue, impliedBraces: false).ToArray();
foreach (ILexToken bit in bits)
{
if (!this.Migrator.TryMigrate(bit, out error))
{
parsed = null;
return(false);
}
}
// get token string
parsed = new TokenString(bits, this.Context, path);
if (!this.Migrator.TryMigrate(parsed, out error))
{
return(false);
}
// validate tokens
foreach (LexTokenToken lexToken in parsed.GetTokenPlaceholders(recursive: false))
{
if (!this.TryValidateToken(lexToken, assumeModIds, out error))
{
return(false);
}
}
// looks OK
error = null;
return(true);
}
public static ConditionalStaNode Parse(TokenString tStr, ref int index)
{
int startIndex = index;
if (!tStr.Match(index, TokenType.ifStatement, TokenType.rBraceOpen))
{
return(null);
}
index++;
ExpressionNode condition = Exp.Parse(tStr.GetRangeInBrackets(ref index));
StatementNode ifTrue = Sta.Parse(tStr, ref index);
StatementNode ifFalse = null;
if (tStr.Match(index, TokenType.elseStatement))
{
index++;
ifFalse = Sta.Parse(tStr, ref index);
}
ConditionalStaNode node = new ConditionalStaNode(condition, ifTrue, ifFalse);
return(node);
}
/// <summary>
/// Returns the list of tokens corresponding to the {&variable} to replace
/// </summary>
private List <Token> GetPreProcVariableTokens(TokenString previousTokenString, Token bracketToken, string value)
{
// do we have a definition for the var?
if (value == null)
{
return(null);
}
List <Token> valueTokens;
// Parse it
if (previousTokenString != null && !string.IsNullOrEmpty(previousTokenString.Value))
{
valueTokens = new ProTokenizer(value, previousTokenString.Value[0] == '"', previousTokenString.Value[0] == '\'').GetTokensList.ToList();
}
else
{
valueTokens = new ProTokenizer(value).GetTokensList.ToList();
}
// Remove EOF and change owner number
List <Token> copiedTokens = new List <Token>();
for (int i = 0; i < valueTokens.Count - 1; i++)
{
var copiedToken = valueTokens[i].Copy(bracketToken.Line, bracketToken.Column, bracketToken.StartPosition, bracketToken.EndPosition);
copiedToken.OwnerNumber = bracketToken.OwnerNumber;
copiedTokens.Add(copiedToken);
}
return(copiedTokens);
}
public static StatementNode Parse(TokenString tStr, ref int index)
{
switch (tStr[index].Type)
{
case TokenType.cBraceOpen:
return(CompoundStaNode.Parse(tStr, ref index));
case TokenType.ifStatement:
return(ConditionalStaNode.Parse(tStr, ref index));
case TokenType.doStatement:
return(DoLoopStaNode.Parse(tStr, ref index));
case TokenType.whileStatement:
return(WhileLoopStaNode.Parse(tStr, ref index));
case TokenType.forStatement:
return(ForLoopStaNode.Parse(tStr, ref index));
case TokenType.returnStatement:
return(ReturnStaNode.Parse(tStr, ref index));
case TokenType.lineEnd:
return(NopStaNode.Parse(tStr, ref index));
default:
return(ExpressionStaNode.Parse(tStr, ref index));
}
}
public void TwoSetsOfTokensDontMatchIfNotBothMatch()
{
TokenString testCase = new TokenString("(ab)(cd)");
TokenString word = new TokenString("bb");
Assert.IsFalse(testCase.Matches(word));
}
/// <summary>Parse a string which can contain tokens, and validate that it's valid.</summary>
/// <param name="rawValue">The raw string which may contain tokens.</param>
/// <param name="tokenContext">The tokens available for this content pack.</param>
/// <param name="error">An error phrase indicating why parsing failed (if applicable).</param>
/// <param name="parsed">The parsed value.</param>
private bool TryParseTokenString(string rawValue, IContext tokenContext, out string error, out TokenString parsed)
{
// parse
parsed = new TokenString(rawValue, tokenContext);
// validate unknown tokens
if (parsed.InvalidTokens.Any())
{
error = $"found unknown tokens ({string.Join(", ", parsed.InvalidTokens.OrderBy(p => p))})";
parsed = null;
return(false);
}
// validate tokens
foreach (TokenName tokenName in parsed.Tokens)
{
IToken token = tokenContext.GetToken(tokenName, enforceContext: false);
if (token == null)
{
error = $"{{{{{tokenName}}}}} can't be used as a token because that token could not be found."; // should never happen
parsed = null;
return(false);
}
if (token.CanHaveMultipleValues(tokenName))
{
error = $"{{{{{tokenName}}}}} can't be used as a token because it can have multiple values.";
parsed = null;
return(false);
}
}
// looks OK
error = null;
return(true);
}
public BinaryExpNode(string @operator, TokenString left, TokenString right)
{
Operator = @operator;
Left = Exp.Parse(left);
Right = Exp.Parse(right);
}
public void TokenStringBuilder_WithTokens()
{
// arrange
const string configKey = "configKey";
const string configValue = "A";
const string tokenKey = "season";
const string raw = " assets/{{configKey}}_{{season}}_{{invalid}}.png ";
var context = new GenericTokenContext();
context.Save(new ImmutableToken(configKey, new InvariantHashSet {
configValue
}));
context.Save(new ImmutableToken(tokenKey, new InvariantHashSet {
"A"
}));
// act
TokenString tokenStr = new TokenString(raw, context);
// assert
tokenStr.Raw.Should().Be(raw.Trim());
tokenStr.Tokens.Should().HaveCount(2);
tokenStr.Tokens.Select(name => name.ToString()).Should().BeEquivalentTo(new[] { configKey, tokenKey });
tokenStr.InvalidTokens.Should().HaveCount(1).And.BeEquivalentTo("invalid");
tokenStr.HasAnyTokens.Should().BeTrue();
tokenStr.IsSingleTokenOnly.Should().BeFalse();
}
public void CompareIdenticalOneLetterStrings()
{
TokenString testCase = new TokenString("a");
TokenString word = new TokenString("a");
Assert.IsTrue(testCase.Matches(word));
}
public void TwoLetterTokensMatchWordTokenIfItContainsTheLetter()
{
TokenString testCase = new TokenString("(ab)");
TokenString word = new TokenString("a");
Assert.IsTrue(testCase.Matches(word));
}
public void TokenStringBuilder_WithSingleToken(string raw, string parsed)
{
// arrange
const string configKey = "tokenKey";
var context = new GenericTokenContext(modId => false);
context.Save(new ImmutableToken(configKey, new InvariantHashSet {
"value"
}));
// act
TokenString tokenStr = new TokenString(raw, context);
IContextualState diagnosticState = tokenStr.GetDiagnosticState();
// assert
tokenStr.Raw.Should().Be(parsed);
tokenStr.GetTokenPlaceholders(recursive: false).Should().HaveCount(1);
tokenStr.GetTokenPlaceholders(recursive: false).Select(p => p.Text).Should().BeEquivalentTo("{{" + configKey + "}}");
tokenStr.HasAnyTokens.Should().BeTrue();
tokenStr.IsSingleTokenOnly.Should().BeTrue();
diagnosticState.IsReady.Should().BeTrue();
diagnosticState.UnreadyTokens.Should().BeEmpty();
diagnosticState.InvalidTokens.Should().BeEmpty();
diagnosticState.Errors.Should().BeEmpty();
}
/*********
** Public methods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="logName">A unique name for this patch shown in log messages.</param>
/// <param name="contentPack">The content pack which requested the patch.</param>
/// <param name="assetName">The normalised asset name to intercept.</param>
/// <param name="conditions">The conditions which determine whether this patch should be applied.</param>
/// <param name="records">The data records to edit.</param>
/// <param name="fields">The data fields to edit.</param>
/// <param name="monitor">Encapsulates monitoring and logging.</param>
/// <param name="normaliseAssetName">Normalise an asset name.</param>
public EditDataPatch(string logName, ManagedContentPack contentPack, TokenString assetName, ConditionDictionary conditions, IDictionary <string, string> records, IDictionary <string, IDictionary <int, string> > fields, IMonitor monitor, Func <string, string> normaliseAssetName)
: base(logName, PatchType.EditData, contentPack, assetName, conditions, normaliseAssetName)
{
this.Records = records;
this.Fields = fields;
this.Monitor = monitor;
}
public Task <string> Detokenize(TokenString token, Guid correlationId, CancellationToken cancellationToken)
{
if (token == null)
{
return(Task.FromResult((string)null));
}
return(_tokenizeClient.Detokenize(token.RawToken(), correlationId));
}
/*********
** Public methods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="logName">A unique name for this patch shown in log messages.</param>
/// <param name="contentPack">The content pack which requested the patch.</param>
/// <param name="assetName">The normalised asset name to intercept.</param>
/// <param name="conditions">The conditions which determine whether this patch should be applied.</param>
/// <param name="records">The data records to edit.</param>
/// <param name="fields">The data fields to edit.</param>
/// <param name="monitor">Encapsulates monitoring and logging.</param>
/// <param name="normaliseAssetName">Normalise an asset name.</param>
public EditDataPatch(string logName, ManagedContentPack contentPack, TokenString assetName, ConditionDictionary conditions, IEnumerable <EditDataPatchRecord> records, IEnumerable <EditDataPatchField> fields, IMonitor monitor, Func <string, string> normaliseAssetName)
: base(logName, PatchType.EditData, contentPack, assetName, conditions, normaliseAssetName)
{
this.Records = records.ToArray();
this.Fields = fields.ToArray();
this.Monitor = monitor;
this.MutableTokenStrings = this.GetTokenStrings(this.Records, this.Fields).Where(str => str.Tokens.Any()).ToArray();
}
/*********
** Public methods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="entryKey">The unique key for the entry in the data file.</param>
/// <param name="fieldKey">The field number to change.</param>
/// <param name="value">The entry value to set.</param>
public EditDataPatchField(TokenString entryKey, TokenString fieldKey, TokenisableJToken value)
{
this.EntryKey = entryKey;
this.FieldKey = fieldKey;
this.Value = value;
this.ContextualValues = new IContextual[] { entryKey, fieldKey, value }.Where(p => p != null).ToArray();
}
public void Visit(TokenString tok)
{
if (tok.Line < FromLine || tok.Line > ToLine)
{
return;
}
//Npp.StyleText((int)TextStyle.String, tok.StartPosition, tok.EndPosition);
}
public static void Main(string[] args)
{
double total = 0.0;
for (int i = 0; i <= 2; i++)
{
Stopwatch sw = new Stopwatch();
sw.Start();
var items = TokenString.NewTokenString(113);
var b = items.Value.IsValid(113);
Console.WriteLine(items.HexValue.TokenFormat("10-4-6-10-3-7"));
Console.WriteLine(b);
sw.Stop();
//Console.WriteLine("Cycle {0} completed at {1:0.00} ms with {2} items", (i + 1), sw.Elapsed.TotalMilliseconds, items.Count);
if (i > 0)
{
total += sw.Elapsed.TotalMilliseconds;
}
//foreach (var q in items) {
// Console.WriteLine(q.Value);
//}
}
Console.WriteLine("Total = {0:#,##0.00}", total / 3);
//foreach (var x in TokenString.AlphaNumericAny) {
// Console.WriteLine(x);
//}
//var lt = DateTime.Now.Ticks;
//var ts = TokenString.NewTokenString();
//Console.WriteLine(ts.HexValue);
//var tb = Encoding.UTF8.GetBytes(ts);
//var xb = BitConverter.GetBytes(lt);
//foreach (var x in xb) {
// Console.WriteLine(x);
//}
//sw.Reset();
//sw.Start();
//Parallel.For(0, items.Count, i => {
// if (items[i] != "~###~-#####-~####") {
// Console.WriteLine("Failed Validation.");
// }
//});
//sw.Stop();
//Console.WriteLine("{0} items", items.Count());
//Console.WriteLine("Validation {0} ms", sw.Elapsed.TotalMilliseconds);
Console.Write("Press any key to continue.... ");
Console.ReadKey(true);
}
/*********
** Public methods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="logName">A unique name for this patch shown in log messages.</param>
/// <param name="assetLoader">Handles loading assets from content packs.</param>
/// <param name="contentPack">The content pack which requested the patch.</param>
/// <param name="assetName">The normalised asset name to intercept.</param>
/// <param name="conditions">The conditions which determine whether this patch should be applied.</param>
/// <param name="fromLocalAsset">The asset key to load from the content pack instead.</param>
/// <param name="fromArea">The sprite area from which to read an image.</param>
/// <param name="toArea">The sprite area to overwrite.</param>
/// <param name="patchMode">Indicates how the image should be patched.</param>
/// <param name="monitor">Encapsulates monitoring and logging.</param>
/// <param name="normaliseAssetName">Normalise an asset name.</param>
public EditImagePatch(string logName, AssetLoader assetLoader, IContentPack contentPack, TokenString assetName, ConditionDictionary conditions, TokenString fromLocalAsset, Rectangle fromArea, Rectangle toArea, PatchMode patchMode, IMonitor monitor, Func <string, string> normaliseAssetName)
: base(logName, PatchType.EditImage, assetLoader, contentPack, assetName, conditions, normaliseAssetName)
{
this.FromLocalAsset = fromLocalAsset;
this.FromArea = fromArea != Rectangle.Empty ? fromArea : null as Rectangle?;
this.ToArea = toArea != Rectangle.Empty ? toArea : null as Rectangle?;
this.PatchMode = patchMode;
this.Monitor = monitor;
}
/// <summary>Parse a boolean <see cref="PatchConfig.Enabled"/> value from a string which can contain tokens, and validate that it's valid.</summary>
/// <param name="rawValue">The raw string which may contain tokens.</param>
/// <param name="tokenContext">The tokens available for this content pack.</param>
/// <param name="migrator">The migrator which validates and migrates content pack data.</param>
/// <param name="error">An error phrase indicating why parsing failed (if applicable).</param>
/// <param name="parsed">The parsed value.</param>
private bool TryParseEnabled(string rawValue, IContext tokenContext, IMigration migrator, out string error, out bool parsed)
{
parsed = false;
// analyse string
if (!this.TryParseStringTokens(rawValue, tokenContext, migrator, out error, out ITokenString tokenString))
{
return(false);
}
// validate & extract tokens
string text = rawValue;
if (tokenString.HasAnyTokens)
{
// only one token allowed
if (!tokenString.IsSingleTokenOnly)
{
error = "can't be treated as a true/false value because it contains multiple tokens.";
return(false);
}
// parse token
LexTokenToken lexToken = tokenString.GetTokenPlaceholders(recursive: false).Single();
IToken token = tokenContext.GetToken(lexToken.Name, enforceContext: false);
ITokenString input = new TokenString(lexToken.InputArg, tokenContext);
// check token options
InvariantHashSet allowedValues = token?.GetAllowedValues(input);
if (token == null || token.IsMutable || !token.IsReady)
{
error = $"can only use static tokens in this field, consider using a {nameof(PatchConfig.When)} condition instead.";
return(false);
}
if (allowedValues == null || !allowedValues.All(p => bool.TryParse(p, out _)))
{
error = "that token isn't restricted to 'true' or 'false'.";
return(false);
}
if (token.CanHaveMultipleValues(input))
{
error = "can't be treated as a true/false value because that token can have multiple values.";
return(false);
}
text = token.GetValues(input).First();
}
// parse text
if (!bool.TryParse(text, out parsed))
{
error = $"can't parse {tokenString.Raw} as a true/false value.";
return(false);
}
return(true);
}
public void StartsWithKeywordTest_NotFound()
{
string CommandLine = "Some_keyword in this line";
string Keyword = "Somekeyword";
bool expected = false;
bool actual;
actual = TokenString.StartsWithKeyword(CommandLine, Keyword);
Assert.AreEqual(expected, actual);
}
public static ExpressionStaNode Parse(TokenString tStr, ref int index)
{
var expression = Exp.Parse(tStr.GetRangeUntil(ref index, TokenType.lineEnd));
ExpressionStaNode node = new ExpressionStaNode(expression);
index++;
return node;
}
