private void Tokenize(TokenizeResultBase result, object value, Template template, string input)
{
log.Verbose($"Start: Processing: {template.Name}");
using (new LogIndentation())
{
var candidates = new CandidateTokenList();
var enumerator = new TokenEnumerator(input);
var replacement = new StringBuilder();
var matchIds = new HashSet <int>();
var disabledRepeatingTokens = new HashSet <int>();
var replacementLocation = new FileLocation();
var hintsMissing = FindHints(template, enumerator, result);
while (enumerator.IsEmpty == false && hintsMissing == false)
{
var next = enumerator.Peek();
// Handle Windows new lines (normalize to Unix)
if (next == "\r" && enumerator.Peek(1) == "\n")
{
enumerator.Next();
next = "\n";
}
// Check for repeated current token
if (candidates.Any && enumerator.Match(candidates.Preamble) && candidates.Preamble.Length > 0)
{
// Can't assign, so clear current context and move to next match
if (candidates.CanAnyAssign(replacement.ToString()) == false)
{
for (var i = 0; i < candidates.Tokens.Count; i++)
{
// If repeated token was the last match, then this non-match will stop it
// matching any future results
var token = candidates.Tokens[i];
if (WasLastMatchedToken(result, token) && string.IsNullOrWhiteSpace(token.Preamble) && string.IsNullOrWhiteSpace(replacement.ToString()))
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
using (new LogIndentation())
{
log.Verbose(("-> Disabled this repeating token."));
disabledRepeatingTokens.Add(token.Id);
candidates.Remove(token);
i--;
}
}
else if (token.ConsiderOnce)
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping & removing {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
candidates.Remove(token);
result.Tokens.AddMiss(token);
matchIds.Add(token.Id);
}
else
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
}
}
replacement.Clear();
enumerator.Advance(candidates.Preamble.Length);
replacementLocation = enumerator.Location;
continue;
}
}
// Assign newline terminated token
if (candidates.Any && candidates.TerminateOnNewLine && next == "\n")
{
if (candidates.Tokens.First().Repeating&&
string.IsNullOrWhiteSpace(candidates.Preamble) &&
result.Tokens.HasMatches)
{
if (result.Tokens.Matches.Last().Token.Id == candidates.Tokens.First().Id)
{
if (enumerator.Location.Line > result.Tokens.Matches.Last().Location.Line + 1)
{
disabledRepeatingTokens.Add(candidates.Tokens.First().Id);
candidates.Remove(candidates.Tokens.First());
}
}
}
using (new LogIndentation())
{
try
{
if (candidates.TryAssign(value, replacement, template.Options, replacementLocation, out var assigned, out var assignedValue))
{
result.Tokens.AddMatch(assigned, assignedValue, enumerator.Location);
AddMatchedTokenIds(template, assigned, matchIds);
}
else
{
foreach (var token in candidates.Tokens)
{
log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
}
}
}
public Boolean ParseEntrypoints()
{
VerifySyntaxToken(TokenType.OpenBlock, "{");
while (true)
{
if (!Enumerator.HasToken())
{
throw new MicroassemblerParseException(Enumerator.Last, "Pair or } expected");
}
if (Enumerator.Current.TokenType == TokenType.CloseBlock)
{
Enumerator.Advance();
break;
}
if (Enumerator.Current.TokenType == TokenType.Pair)
{
Object[] pair = Enumerator.Current.Value as Object[];
Enumerator.Advance();
if (!(pair[1] is String))
{
throw new MicroassemblerParseException(Enumerator.Last, "Entrypoint pair value must be a symbol");
}
String symbol = (String)pair[1];
if (pair[0] is String)
{
String key = pair[0] as String;
switch (key.ToLower())
{
case "fetch":
Microprogram.FetchEntrypoint = symbol;
break;
case "interrupt":
Microprogram.InterruptEntrypoint = symbol;
break;
default: throw new MicroassemblerParseException(Enumerator.Last, $"Invalid entrypoint '{key}'");
}
}
else if (pair[0] is long)
{
int entryIndex = Convert.ToInt32(pair[0]);
if (Microprogram.InstructionEntrypoints.ContainsKey(entryIndex))
{
Console.WriteLine($"Warning: Duplicate entrypoint definition on line {Enumerator.Last.Line}, previous value overridden");
}
Microprogram.InstructionEntrypoints[entryIndex] = symbol;
}
else
{
throw new MicroassemblerParseException(Enumerator.Last, "Invalid Entrypoint");
}
if (Enumerator.HasNext() && Enumerator.Next.TokenType == TokenType.Pair)
{
VerifySyntaxToken(TokenType.ListDelimeter, ",");
}
if (Enumerator.HasNext() && Enumerator.Next.TokenType == TokenType.CloseBlock)
{
DiscardOptionalToken(TokenType.ListDelimeter, ",");
}
}
else
{
throw new MicroassemblerParseException(Enumerator.Current, "Pair expected");
}
}
return(true);
}
public bool TryParse <T>(Template template, string input, out int matches, out T result) where T : class, new()
{
log.Debug($"Start: Processing: {template.Name}");
Token current = null;
matches = 0;
var value = new T();
var enumerator = new TokenEnumerator(input);
var replacement = new StringBuilder();
var matchIds = new List <int>();
while (enumerator.IsEmpty == false)
{
var next = enumerator.Peek();
// Handle Windows new lines (normalize to Unix)
if (next == "\r" && enumerator.Peek(1) == "\n")
{
enumerator.Next();
next = "\n";
}
// Check for repeated current token
if (current != null && enumerator.Match(current.Preamble))
{
// Can't assign, so clear current context and move to next match
if (current.CanAssign(replacement.ToString()) == false)
{
replacement.Clear();
enumerator.Advance(current.Preamble.Length);
continue;
}
}
// Check for next token
if (enumerator.Match(template.TokensExcluding(matchIds), out var match))
{
if (current == null)
{
current = match;
replacement.Clear();
enumerator.Advance(match.Preamble.Length);
matchIds.AddRange(template.GetTokenIdsUpTo(match));
}
else if (replacement.Length > 0 && current.Assign(value, replacement.ToString(), template.Options, log))
{
matches++;
current = match;
replacement.Clear();
enumerator.Advance(match.Preamble.Length);
matchIds.AddRange(template.GetTokenIdsUpTo(match));
}
else
{
replacement.Append(next);
enumerator.Next();
}
}
// Append to replacement
else
{
replacement.Append(next);
enumerator.Next();
}
}
if (current != null && replacement.Length > 0 && !string.IsNullOrEmpty(current.Name))
{
current.Assign(value, replacement.ToString(), template.Options, log);
matches++;
}
result = value;
log.Debug($" Found {matches} matches.");
log.Debug($"Finished: Processing: {template.Name}");
return(matches > 0);
}
public Microprogram ParseProgram(String program)
{
Microprogram = new Microprogram();
Enumerator = Tokenizer.Tokenize(program);
if (Enumerator == null)
{
return(null);
}
while (Enumerator.HasToken())
{
Token token = Enumerator.Current;
Enumerator.Advance();
if (token.TokenType == TokenType.Word)
{
String keyword = (String)token.Value;
switch (keyword.ToLower())
{
case "config":
if (!ProcessConfigs())
{
return(null);
}
break;
case "const":
if (!ProcessConstant())
{
return(null);
}
break;
case "control":
if (!ProcessControlWordLabel())
{
return(null);
}
break;
case "sequence":
if (!ProcessSequence())
{
return(null);
}
break;
case "macro":
if (!ProcessMacro())
{
return(null);
}
break;
case "empty":
if (!ParseEmpty())
{
return(null);
}
break;
case "entrypoints":
if (!ParseEntrypoints())
{
return(null);
}
break;
default:
throw new MicroassemblerParseException(token, "Keyword expected");
}
}
else
{
throw new MicroassemblerParseException(token, "Keyword expected");
}
}
//Config sanity checks
if (Microprogram.MicroprogramLength <= 0)
{
throw new MicroassemblerParseException($"The microprogram length must be greater than zero");
}
if (Microprogram.OpcodeWidth <= 0)
{
throw new MicroassemblerParseException($"The opcode width must be greater than zero");
}
if (Microprogram.ControlWordWidth <= 0)
{
throw new MicroassemblerParseException($"The control word width must be greater than zero");
}
if (Microprogram.BankSelectorMask.Length <= 0)
{
throw new MicroassemblerParseException($"The bank selector width must be greater than zero");
}
if (Microprogram.BankSelectorMask.Length + 1 >= Microprogram.ControlWordWidth)
{
throw new MicroassemblerParseException($"Control word width of {Microprogram.ControlWordWidth} is invalid, as the bank selector has a width of {Microprogram.BankSelectorMask.Length}");
}
//Check control word label lengths
foreach (ControlWordLabel cw in Microprogram.ControlWordLabels.Values)
{
if (cw.Mask.Length <= 0)
{
throw new MicroassemblerParseException($"Control word label {cw.Name} must have a length that is geater than zero");
}
if (cw.Mask.Length + Microprogram.BankSelectorMask.Length > Microprogram.ControlWordWidth)
{
throw new MicroassemblerParseException($"Control word label {cw.Name} exceeds the maximum control word width of {Microprogram.ControlWordWidth - Microprogram.BankSelectorMask.Length}");
}
}
return(Microprogram);
}