Ejemplo n.º 1
0
        private void Tokenize(TokenizeResultBase result, object value, Template template, string input)
        {
            log.Verbose($"Start: Processing: {template.Name}");

            using (new LogIndentation())
            {
                var candidates = new CandidateTokenList();

                var enumerator              = new TokenEnumerator(input);
                var replacement             = new StringBuilder();
                var matchIds                = new HashSet <int>();
                var disabledRepeatingTokens = new HashSet <int>();
                var replacementLocation     = new FileLocation();

                var hintsMissing = FindHints(template, enumerator, result);

                while (enumerator.IsEmpty == false && hintsMissing == false)
                {
                    var next = enumerator.Peek();

                    // Handle Windows new lines (normalize to Unix)
                    if (next == "\r" && enumerator.Peek(1) == "\n")
                    {
                        enumerator.Next();
                        next = "\n";
                    }

                    // Check for repeated current token
                    if (candidates.Any && enumerator.Match(candidates.Preamble) && candidates.Preamble.Length > 0)
                    {
                        // Can't assign, so clear current context and move to next match
                        if (candidates.CanAnyAssign(replacement.ToString()) == false)
                        {
                            for (var i = 0; i < candidates.Tokens.Count; i++)
                            {
                                // If repeated token was the last match, then this non-match will stop it
                                // matching any future results
                                var token = candidates.Tokens[i];
                                if (WasLastMatchedToken(result, token) && string.IsNullOrWhiteSpace(token.Preamble) && string.IsNullOrWhiteSpace(replacement.ToString()))
                                {
                                    log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
                                    using (new LogIndentation())
                                    {
                                        log.Verbose(("-> Disabled this repeating token."));
                                        disabledRepeatingTokens.Add(token.Id);
                                        candidates.Remove(token);
                                        i--;
                                    }
                                }
                                else if (token.ConsiderOnce)
                                {
                                    log.Verbose("-> Ln: {0} Col: {1} : Skipping & removing {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());

                                    candidates.Remove(token);
                                    result.Tokens.AddMiss(token);
                                    matchIds.Add(token.Id);
                                }
                                else
                                {
                                    log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
                                }
                            }

                            replacement.Clear();
                            enumerator.Advance(candidates.Preamble.Length);
                            replacementLocation = enumerator.Location;
                            continue;
                        }
                    }

                    // Assign newline terminated token
                    if (candidates.Any && candidates.TerminateOnNewLine && next == "\n")
                    {
                        if (candidates.Tokens.First().Repeating&&
                            string.IsNullOrWhiteSpace(candidates.Preamble) &&
                            result.Tokens.HasMatches)
                        {
                            if (result.Tokens.Matches.Last().Token.Id == candidates.Tokens.First().Id)
                            {
                                if (enumerator.Location.Line > result.Tokens.Matches.Last().Location.Line + 1)
                                {
                                    disabledRepeatingTokens.Add(candidates.Tokens.First().Id);
                                    candidates.Remove(candidates.Tokens.First());
                                }
                            }
                        }

                        using (new LogIndentation())
                        {
                            try
                            {
                                if (candidates.TryAssign(value, replacement, template.Options, replacementLocation, out var assigned, out var assignedValue))
                                {
                                    result.Tokens.AddMatch(assigned, assignedValue, enumerator.Location);
                                    AddMatchedTokenIds(template, assigned, matchIds);
                                }
                                else
                                {
                                    foreach (var token in candidates.Tokens)
                                    {
                                        log.Verbose("-> Ln: {0} Col: {1} : Skipping {2} ({3}), '{4}' is not a match.", enumerator.Location.Line, enumerator.Location.Column, token.Name, token.Id, replacement.ToString());
                                    }
                                }
                            }
Ejemplo n.º 2
0
        public Boolean ParseEntrypoints()
        {
            VerifySyntaxToken(TokenType.OpenBlock, "{");
            while (true)
            {
                if (!Enumerator.HasToken())
                {
                    throw new MicroassemblerParseException(Enumerator.Last, "Pair or } expected");
                }
                if (Enumerator.Current.TokenType == TokenType.CloseBlock)
                {
                    Enumerator.Advance();
                    break;
                }
                if (Enumerator.Current.TokenType == TokenType.Pair)
                {
                    Object[] pair = Enumerator.Current.Value as Object[];
                    Enumerator.Advance();
                    if (!(pair[1] is String))
                    {
                        throw new MicroassemblerParseException(Enumerator.Last, "Entrypoint pair value must be a symbol");
                    }
                    String symbol = (String)pair[1];
                    if (pair[0] is String)
                    {
                        String key = pair[0] as String;
                        switch (key.ToLower())
                        {
                        case "fetch":
                            Microprogram.FetchEntrypoint = symbol;
                            break;

                        case "interrupt":
                            Microprogram.InterruptEntrypoint = symbol;
                            break;

                        default: throw new MicroassemblerParseException(Enumerator.Last, $"Invalid entrypoint '{key}'");
                        }
                    }
                    else if (pair[0] is long)
                    {
                        int entryIndex = Convert.ToInt32(pair[0]);
                        if (Microprogram.InstructionEntrypoints.ContainsKey(entryIndex))
                        {
                            Console.WriteLine($"Warning: Duplicate entrypoint definition on line {Enumerator.Last.Line}, previous value overridden");
                        }
                        Microprogram.InstructionEntrypoints[entryIndex] = symbol;
                    }
                    else
                    {
                        throw new MicroassemblerParseException(Enumerator.Last, "Invalid Entrypoint");
                    }

                    if (Enumerator.HasNext() && Enumerator.Next.TokenType == TokenType.Pair)
                    {
                        VerifySyntaxToken(TokenType.ListDelimeter, ",");
                    }
                    if (Enumerator.HasNext() && Enumerator.Next.TokenType == TokenType.CloseBlock)
                    {
                        DiscardOptionalToken(TokenType.ListDelimeter, ",");
                    }
                }
                else
                {
                    throw new MicroassemblerParseException(Enumerator.Current, "Pair expected");
                }
            }
            return(true);
        }
Ejemplo n.º 3
0
        public bool TryParse <T>(Template template, string input, out int matches, out T result) where T : class, new()
        {
            log.Debug($"Start: Processing: {template.Name}");

            Token current = null;

            matches = 0;

            var value       = new T();
            var enumerator  = new TokenEnumerator(input);
            var replacement = new StringBuilder();
            var matchIds    = new List <int>();

            while (enumerator.IsEmpty == false)
            {
                var next = enumerator.Peek();

                // Handle Windows new lines (normalize to Unix)
                if (next == "\r" && enumerator.Peek(1) == "\n")
                {
                    enumerator.Next();
                    next = "\n";
                }

                // Check for repeated current token
                if (current != null && enumerator.Match(current.Preamble))
                {
                    // Can't assign, so clear current context and move to next match
                    if (current.CanAssign(replacement.ToString()) == false)
                    {
                        replacement.Clear();
                        enumerator.Advance(current.Preamble.Length);
                        continue;
                    }
                }

                // Check for next token
                if (enumerator.Match(template.TokensExcluding(matchIds), out var match))
                {
                    if (current == null)
                    {
                        current = match;
                        replacement.Clear();
                        enumerator.Advance(match.Preamble.Length);
                        matchIds.AddRange(template.GetTokenIdsUpTo(match));
                    }
                    else if (replacement.Length > 0 && current.Assign(value, replacement.ToString(), template.Options, log))
                    {
                        matches++;
                        current = match;
                        replacement.Clear();
                        enumerator.Advance(match.Preamble.Length);
                        matchIds.AddRange(template.GetTokenIdsUpTo(match));
                    }
                    else
                    {
                        replacement.Append(next);
                        enumerator.Next();
                    }
                }

                // Append to replacement
                else
                {
                    replacement.Append(next);
                    enumerator.Next();
                }
            }

            if (current != null && replacement.Length > 0 && !string.IsNullOrEmpty(current.Name))
            {
                current.Assign(value, replacement.ToString(), template.Options, log);
                matches++;
            }

            result = value;

            log.Debug($"  Found {matches} matches.");
            log.Debug($"Finished: Processing: {template.Name}");

            return(matches > 0);
        }
Ejemplo n.º 4
0
        public Microprogram ParseProgram(String program)
        {
            Microprogram = new Microprogram();
            Enumerator   = Tokenizer.Tokenize(program);
            if (Enumerator == null)
            {
                return(null);
            }
            while (Enumerator.HasToken())
            {
                Token token = Enumerator.Current;
                Enumerator.Advance();
                if (token.TokenType == TokenType.Word)
                {
                    String keyword = (String)token.Value;
                    switch (keyword.ToLower())
                    {
                    case "config":
                        if (!ProcessConfigs())
                        {
                            return(null);
                        }
                        break;

                    case "const":
                        if (!ProcessConstant())
                        {
                            return(null);
                        }
                        break;

                    case "control":
                        if (!ProcessControlWordLabel())
                        {
                            return(null);
                        }
                        break;

                    case "sequence":
                        if (!ProcessSequence())
                        {
                            return(null);
                        }
                        break;

                    case "macro":
                        if (!ProcessMacro())
                        {
                            return(null);
                        }
                        break;

                    case "empty":
                        if (!ParseEmpty())
                        {
                            return(null);
                        }
                        break;

                    case "entrypoints":
                        if (!ParseEntrypoints())
                        {
                            return(null);
                        }
                        break;

                    default:
                        throw new MicroassemblerParseException(token, "Keyword expected");
                    }
                }
                else
                {
                    throw new MicroassemblerParseException(token, "Keyword expected");
                }
            }
            //Config sanity checks
            if (Microprogram.MicroprogramLength <= 0)
            {
                throw new MicroassemblerParseException($"The microprogram length must be greater than zero");
            }
            if (Microprogram.OpcodeWidth <= 0)
            {
                throw new MicroassemblerParseException($"The opcode width must be greater than zero");
            }
            if (Microprogram.ControlWordWidth <= 0)
            {
                throw new MicroassemblerParseException($"The control word width must be greater than zero");
            }
            if (Microprogram.BankSelectorMask.Length <= 0)
            {
                throw new MicroassemblerParseException($"The bank selector width must be greater than zero");
            }
            if (Microprogram.BankSelectorMask.Length + 1 >= Microprogram.ControlWordWidth)
            {
                throw new MicroassemblerParseException($"Control word width of {Microprogram.ControlWordWidth} is invalid, as the bank selector has a width of {Microprogram.BankSelectorMask.Length}");
            }
            //Check control word label lengths
            foreach (ControlWordLabel cw in Microprogram.ControlWordLabels.Values)
            {
                if (cw.Mask.Length <= 0)
                {
                    throw new MicroassemblerParseException($"Control word label {cw.Name} must have a length that is geater than zero");
                }
                if (cw.Mask.Length + Microprogram.BankSelectorMask.Length > Microprogram.ControlWordWidth)
                {
                    throw new MicroassemblerParseException($"Control word label {cw.Name} exceeds the maximum control word width of {Microprogram.ControlWordWidth - Microprogram.BankSelectorMask.Length}");
                }
            }
            return(Microprogram);
        }