protected ptoken_reader() { m_idx = 0; m_token_store = null; // add a first entry to the stack m_source_location.emplace_back(new plib.source_location("Unknown", 0)); }
public ptokenizer() // NOLINT(misc-forwarding-reference-overload, bugprone-forwarding-reference-overload) { m_strm = null; m_unget = (char)0; m_string = '"'; m_support_line_markers = true; // FIXME m_token_queue = null; clear(); }
// FIXME: used by source_t - need a different approach at some time public bool parse_stream(plib.istream_uptr istrm, string name) //bool parse_stream(plib::istream_uptr &&istrm, const pstring &name); { var filename = istrm.filename(); var preprocessed = new MemoryStream( //auto preprocessed = std::make_unique<std::stringstream>(putf8string( Encoding.ASCII.GetBytes(new plib.ppreprocessor(m_includes, m_defines).process(istrm, filename))); //plib::ppreprocessor(m_includes, &m_defines).process(std::move(istrm), filename))); parser_t_token_store st = new parser_t_token_store(); parser_t parser = new parser_t(this); parser.parse_tokens(new plib.istream_uptr(preprocessed, filename), st); return(parser.parse(st, name)); }
public parser_t(nlparse_t setup) : base() { m_setup = setup; m_cur_local = null; m_tokenizer.identifier_chars("abcdefghijklmnopqrstuvwvxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890_.-$@") .number_chars(".0123456789", "0123456789eE-.") //FIXME: processing of numbers .whitespace("" + " " + (char)9 + (char)10 + (char)13) .comment("/*", "*/", "//"); m_tok_paren_left = m_tokenizer.register_token("("); m_tok_paren_right = m_tokenizer.register_token(")"); m_tok_comma = m_tokenizer.register_token(","); m_tok_static = m_tokenizer.register_token("static"); m_tok_ALIAS = m_tokenizer.register_token("ALIAS"); m_tok_DIPPINS = m_tokenizer.register_token("DIPPINS"); m_tok_NET_C = m_tokenizer.register_token("NET_C"); m_tok_FRONTIER = m_tokenizer.register_token("OPTIMIZE_FRONTIER"); m_tok_PARAM = m_tokenizer.register_token("PARAM"); m_tok_DEFPARAM = m_tokenizer.register_token("DEFPARAM"); m_tok_HINT = m_tokenizer.register_token("HINT"); m_tok_NET_MODEL = m_tokenizer.register_token("NET_MODEL"); m_tok_NET_REGISTER_DEV = m_tokenizer.register_token("NET_REGISTER_DEV"); m_tok_INCLUDE = m_tokenizer.register_token("INCLUDE"); m_tok_LOCAL_SOURCE = m_tokenizer.register_token("LOCAL_SOURCE"); m_tok_LOCAL_LIB_ENTRY = m_tokenizer.register_token("LOCAL_LIB_ENTRY"); m_tok_EXTERNAL_LIB_ENTRY = m_tokenizer.register_token("EXTERNAL_LIB_ENTRY"); m_tok_SUBMODEL = m_tokenizer.register_token("SUBMODEL"); m_tok_NETLIST_START = m_tokenizer.register_token("NETLIST_START"); m_tok_NETLIST_END = m_tokenizer.register_token("NETLIST_END"); m_tok_NETLIST_EXTERNAL = m_tokenizer.register_token("NETLIST_EXTERNAL"); m_tok_EXTERNAL_SOURCE = m_tokenizer.register_token("EXTERNAL_SOURCE"); m_tok_TRUTHTABLE_START = m_tokenizer.register_token("TRUTHTABLE_START"); m_tok_TRUTHTABLE_END = m_tokenizer.register_token("TRUTHTABLE_END"); m_tok_TRUTHTABLE_ENTRY = m_tokenizer.register_token("TRUTHTABLE_ENTRY"); m_tok_TT_HEAD = m_tokenizer.register_token("TT_HEAD"); m_tok_TT_LINE = m_tokenizer.register_token("TT_LINE"); m_tok_TT_FAMILY = m_tokenizer.register_token("TT_FAMILY"); m_tokenizer.register_token("RES_R"); m_tokenizer.register_token("RES_K"); m_tokenizer.register_token("RES_M"); m_tokenizer.register_token("CAP_U"); m_tokenizer.register_token("CAP_N"); m_tokenizer.register_token("CAP_P"); }
public void append_to_store(TextReader reader, ptokenizer_token_store tokstor) //void append_to_store(putf8_reader *reader, token_store &tokstor) { clear(); m_strm = reader; // Process tokens into queue ptokenizer_token_t ret = new ptokenizer_token_t(ptokenizer_token_type.UNKNOWN); m_token_queue = tokstor; do { ret = get_token_comment(); tokstor.push_back(ret); } while (!ret.is_type(ptokenizer_token_type.ENDOFFILE)); m_token_queue = null; }
//PCOPYASSIGNMOVE(ptoken_reader, delete) //virtual ~ptoken_reader() = default; protected void set_token_source(ptokenizer_token_store tokstor) { m_token_store = tokstor; }
public void parse_tokens(plib.istream_uptr strm, parser_t_token_store tokstor) { //plib::putf8_reader u8reader(strm.release_stream()); m_tokenizer.append_to_store(new StreamReader(strm.release_stream()), tokstor); }
//bool parse(plib::istream_uptr &&strm, const pstring &nlname); public bool parse(parser_t_token_store tokstor, string nlname) { set_token_source(tokstor); bool in_nl = false; while (true) { // FIXME: line numbers in cached local netlists are wrong // need to process raw tokens here. parser_t_token_t token = get_token_raw(); if (token.is_type(parser_t_token_type.ENDOFFILE)) { return(false); } if (token.is_(m_tok_NETLIST_END) || token.is_(m_tok_TRUTHTABLE_END)) { if (!in_nl) { error(MF_PARSER_UNEXPECTED_1(token.str())); } else { in_nl = false; } require_token(m_tok_paren_left); require_token(m_tok_paren_right); m_cur_local.push_back(token); m_cur_local.push_back(new parser_t_token_t(m_tok_paren_left)); m_cur_local.push_back(new parser_t_token_t(m_tok_paren_right)); } else if (token.is_(m_tok_NETLIST_START) || token.is_(m_tok_TRUTHTABLE_START)) { if (in_nl) { error(MF_PARSER_UNEXPECTED_1(token.str())); } require_token(m_tok_paren_left); parser_t_token_t name = get_token(); if (token.is_(m_tok_NETLIST_START) && (name.str() == nlname || nlname.empty())) { require_token(m_tok_paren_right); parse_netlist(); return(true); } if (token.is_(m_tok_TRUTHTABLE_START) && name.str() == nlname) { net_truthtable_start(nlname); return(true); } // create a new cached local store m_local.emplace(name.str(), new parser_t_token_store()); m_cur_local = m_local[name.str()]; var sl = sourceloc(); var li = new plib.pfmt("# {0} \"{1}\"").op(sl.line(), sl.file_name()); m_cur_local.push_back(new parser_t_token_t(parser_t_token_type.LINEMARKER, li)); m_cur_local.push_back(token); m_cur_local.push_back(new parser_t_token_t(m_tok_paren_left)); m_cur_local.push_back(name); //m_cur_local->push_back(token_t(m_tok_paren_right)); in_nl = true; } // FIXME: do we really need this going forward ? there should be no need // for NETLIST_EXTERNAL in netlist files else if (token.is_(m_tok_NETLIST_EXTERNAL)) { if (in_nl) { error(MF_UNEXPECTED_NETLIST_EXTERNAL()); } require_token(m_tok_paren_left); parser_t_token_t name = get_token(); require_token(m_tok_paren_right); } else if (!in_nl) { if (!token.is_(m_tok_static) && !token.is_type(parser_t_token_type.SOURCELINE) && !token.is_type(parser_t_token_type.LINEMARKER)) { error(MF_EXPECTED_NETLIST_START_1(token.str())); } } else { m_cur_local.push_back(token); } } }