public static void ReadDir(Regulator r, string dir, string to, string boundary)
        {
            Globals.LOG.AppendLocation();
            // working subdirectory
            string sub      = new DirectoryInfo(dir).Name;
            string sub_path = to + "/" + sub;

            string[] sub_paths = SortNumerically(Directory.GetFiles(dir, "*.txt", SearchOption.AllDirectories)).ToArray();

            if (sub_paths.Length > 0)
            {
                Directory.CreateDirectory(sub_path);

                Console.WriteLine("subscription = " + sub);
                Globals.LOG.AppendMessage("Begin read for subscription = " + sub);

                // parse regulation file then purge from working directory
                foreach (string file in sub_paths)
                {
                    ReadFile(file, sub_path, boundary); // read
                    if (!r.ParseReg())
                    {
                        return;
                    }                              // parse + purge
                }
            }
        }
        public static void Main(string[] args)
        {
            Console.WriteLine(transferProcessed);
            // read inflow directory
            try {
                //ReadDir(start_folder + "/" + CFR, end_folder);
                //ReadDir(start_folder + "/" + PUC, end_folder);
                //ReadDir(start_folder + "/" + NCS, end_folder);
                //ReadDir(start_folder + "/" + NSL, end_folder);
                //ReadDir(start_folder + "/" + NYCR, end_folder);
                //ReadDir(start_folder + "/" + NYSL, end_folder);
            } catch (Exception e) {
                Console.WriteLine("Incorrect start_folder. Verify that subscription id exists.");
            }

            // configure nested table of contents [meta] data
            List <string> meta = new List <string> {
                "{http://wwww.w3.org/2005/Atom}title",                                      // 0 - Source / Regulation Body
                "{http://wwww.w3.org/2005/Atom}subtitle",                                   // 1 - File Name
                "{http://wwww.w3.org/2005/Atom}id",                                         // 2 - File Identifier
                "{http://wwww.w3.org/2005/Atom}updated",                                    // 3 - File Date Updated
                "{http://services.lexisnexis.com/interfaces/publish/lnpub/1/}publishType",  // 4 - File Publish Type
                "{http://wwww.w3.org/2005/Atom}entry"                                       // pointer
            };
            List <string> metameta = new List <string> {
                "{http://wwww.w3.org/2005/Atom}title",                                      // 5 - Content Identifier
                "{http://wwww.w3.org/2005/Atom}updated",                                    // 6 - Content Date Updated
                "{http://services.lexisnexis.com/interfaces/publish/lnpub/1/}action"        // 7 - Content Action
            };

            // configure content data
            Regulator c = new Regulator(end_folder, CFR, meta, metameta, new List <string> {
                "citations",                                                                // 0 - Content Citation
                "jurisSystem[normalizedLongName]",                                          // 1 - Content Jurisdiction
                "hierarchyLevel[levelType=@title]/heading",                                 // 2 - Content Title 1                  [Title]
                "hierarchyLevel[levelType=@subtitle]/heading",                              // 3 - Content Title 2                  [Subtitle]
                "hierarchyLevel[levelType=@chapter]/heading",                               // 4 - Content Topic 1                  [Chapter]
                "hierarchyLevel[levelType=@subchapter]/heading",                            // 5 - Content Topic 2                  [Subchapter]
                "hierarchyLevel[levelType=@part]/heading",                                  // 6 - Content Section 1                [Part]
                "hierarchyLevel[levelType=@subpart]/heading",                               // 7 - Content Section 2                [Subpart]
                "hierarchyLevel[levelType=@section]/heading",                               // 8 - Content Section Description      [Section]
                "hierarchyLevel[levelType=@subsection]/heading",                            // 9 - Content Subsection 1             [Subsection]
                "hierarchyLevel[levelType=@unclassified]/heading",                          //10 - Content Subsection 2             [Unclassified]
                "administrativeCode",                                                       //11 - Content Description              [Description]
                "historyItem",                                                              //12 - Content Reference Citation       [Reference Citations]
            });
            Regulator p = new Regulator(end_folder, PUC, meta, metameta, new List <string> {
                "citations",
                "jurisSystem[normalizedLongName]",
                "governmentBodyName[normalizedLongName]",
                null,
                "fullCaseName",
                "docketNumber",
                null,
                "decisionDate",
                null,
                null,
                null,
                "administrativeDocBody",
                null,
            });
            Regulator s = new Regulator(end_folder, NCS, meta, metameta, new List <string> {
                "citations",
                "jurisSystem[normalizedLongName]",
                "hierarchyLevel[levelType=@title]/heading",
                "hierarchyLevel[levelType=@subtitle]/heading",
                "hierarchyLevel[levelType=@chapter]/heading",
                "hierarchyLevel[levelType=@subchapter]/heading",
                "hierarchyLevel[levelType=@part]/heading",
                "hierarchyLevel[levelType=@subpart]/heading",
                "hierarchyLevel[levelType=@section]/heading",
                "hierarchyLevel[levelType=@subsection]/heading",
                "hierarchyLevel[levelType=@unclassified]/heading",
                "legislativeDocBody",
                null,
            });
            Regulator statutes = new Regulator(end_folder, NSL, meta, metameta, new List <string> {
                "citations",
                "jurisSystem[normalizedLongName]",
                "hierarchyLevel[levelType=@topic]/heading",
                null,
                "hierarchyLevel[levelType=@article]/heading",
                "hierarchyLevel[levelType=@title]/heading",
                null,
                null,
                null,
                null,
                null,
                "legislativeDocBody",
                "history",
            });
            Regulator city_regs = new Regulator(end_folder, NYCR, meta, metameta, new List <string> {
                "citations",
                "jurisSystem[normalizedLongName]",
                "hierarchyLevel[levelType=@topic]/heading",
                "hierarchyLevel[levelType=@rule]/heading",
                "hierarchyLevel[levelType=@title]/heading",
                null,
                "hierarchyLevel[levelType=@chapter]/heading",
                null,
                "hierarchyLevel[levelType=@section]/heading",
                null,
                null,
                "legislativeDocBody",
                null,
            });
            Regulator state_legis = new Regulator(end_folder, NYSL, meta, metameta, new List <string> {
                "citations",
                "jurisSystem[normalizedLongName]",
                "hierarchyLevel[levelType=@title]/heading",
                "hierarchyLevel[levelType=@subtitle]/heading",
                "hierarchyLevel[levelType=@chapter]/heading",
                "hierarchyLevel[levelType=@subchapter]/heading",
                "hierarchyLevel[levelType=@part]/heading",
                "hierarchyLevel[levelType=@subpart]/heading",
                "hierarchyLevel[levelType=@section]/heading",
                "hierarchyLevel[levelType=@subsection]/heading",
                "hierarchyLevel[levelType=@unclassified]/heading",
                "legislativeDocBody",
                null,
            });

            // interpret / parse content data as regulations
            List <List <Regulation> > regs = new List <List <Regulation> >();

            //regs.Add(c.ParseReg());
            //regs.Add(p.ParseReg());
            //regs.Add(s.ParseReg());
            //regs.Add(statutes.ParseReg());
            //regs.Add(city_regs.ParseReg());
            //regs.Add(state_legis.ParseReg());

            Directory.CreateDirectory(processed + "/" + CFR); // create the processed directory if it does not exist
            if (transferProcessed)
            {
                MoveDir(start_folder + "/" + CFR, processed + "/" + CFR); // move processed content
            }

            try { Directory.Delete(end_folder, !keepRegulationContent); } catch (Exception e) { }

            // write regulation attributes to db
            try
            {
                cnn.Open();
                Guid       g       = Guid.NewGuid();
                SqlCommand SqlComm = new SqlCommand("INSERT INTO " + job_table + "(Job_Id, Job_Type, Start_Time) VALUES(@guid, @task, @start)", cnn);
                SqlComm.Parameters.AddWithValue("@guid", g);
                SqlComm.Parameters.AddWithValue("@task", "Parsing XML");
                SqlComm.Parameters.AddWithValue("@start", DateTime.Now.ToString("yyyy-MM-dd h:mm:ss tt"));
                try { SqlComm.ExecuteNonQuery(); } catch (Exception e) { Console.WriteLine(e); }
                //SqlComm.Dispose();

                foreach (List <Regulation> body in regs)
                {
                    foreach (Regulation reg in body)
                    {
                        Guid file_g = Guid.NewGuid();

                        SqlComm = new SqlCommand("INSERT INTO " + file_table + "(Tbl_id, Subscription_id, File_name, Start_Time) VALUES(@guid, @sub, @file, @start)", cnn);
                        SqlComm.Parameters.AddWithValue("@guid", file_g);
                        SqlComm.Parameters.AddWithValue("@sub", reg.getSubscription());
                        SqlComm.Parameters.AddWithValue("@file", reg.getMeta(1));
                        SqlComm.Parameters.AddWithValue("@start", DateTime.Now.ToString("yyyy-MM-dd h:mm:ss tt"));
                        SqlComm.ExecuteNonQuery();

                        // general configurations -- string manipulation on params should be localized here
                        Console.WriteLine("Generating SQL Command.");
                        SqlComm = new SqlCommand("INSERT INTO " + content_table + "(Subscription_Id, File_Name, Content_Id, " +
                                                 "Publish_Type, Action, Updated, Jurisdiction, Citation, Regulation_Type, Body, Title1, " +
                                                 "Title2, Topic1, Topic2, Section1, Section2, Section_Description, SubSection1, SubSection2, Description, RefCitation, " +
                                                 "Date_Type, Actual_Date) VALUES(@sub, @file, @content, @pub, @action, @updated, @juris, " +
                                                 "@citation, @type, @source, @title_1, @title_2, @topic_1, @topic_2, @sec_1, @sec_2, @secdescrip," +
                                                 "@subsec1, @subsec2, @descrip, @refcite, @dateType, @actualDate)", cnn);
                        SqlComm.Parameters.AddWithValue("@sub", reg.getSubscription());
                        SqlComm.Parameters.AddWithValue("@file", reg.getMeta(1));
                        SqlComm.Parameters.AddWithValue("@content", reg.getMeta(5).Substring(reg.getMeta(5).IndexOf("urn:contentItem:") + 16));
                        SqlComm.Parameters.AddWithValue("@pub", reg.getMeta(4));
                        SqlComm.Parameters.AddWithValue("@action", reg.getMeta(7));
                        SqlComm.Parameters.AddWithValue("@updated", reg.getMeta(6));
                        SqlComm.Parameters.AddWithValue("@juris", reg.getColumn(1));
                        SqlComm.Parameters.AddWithValue("@citation", reg.getColumn(0));
                        SqlComm.Parameters.AddWithValue("@type", "");
                        SqlComm.Parameters.AddWithValue("@source", reg.getMeta(0));
                        SqlComm.Parameters.AddWithValue("@title_1", reg.getColumn(2));
                        SqlComm.Parameters.AddWithValue("@title_2", reg.getColumn(3));
                        SqlComm.Parameters.AddWithValue("@topic_1", reg.getColumn(4));
                        SqlComm.Parameters.AddWithValue("@topic_2", reg.getColumn(5));
                        SqlComm.Parameters.AddWithValue("@sec_1", reg.getColumn(6));
                        SqlComm.Parameters.AddWithValue("@sec_2", reg.getColumn(7));
                        SqlComm.Parameters.AddWithValue("@secdescrip", reg.getColumn(10));
                        SqlComm.Parameters.AddWithValue("@subsec1", reg.getColumn(8));
                        SqlComm.Parameters.AddWithValue("@subsec2", reg.getColumn(9));
                        SqlComm.Parameters.AddWithValue("@descrip", reg.getColumn(11));
                        SqlComm.Parameters.AddWithValue("@refcite", reg.getColumn(12));
                        SqlComm.Parameters.AddWithValue("@dateType", "");
                        SqlComm.Parameters.AddWithValue("@actualDate", "");

                        SqlComm.ExecuteNonQuery();

                        SqlComm = new SqlCommand("UPDATE " + file_table + " SET End_Time=@end, Content_Item=@content WHERE Tbl_id=@guid", cnn);
                        SqlComm.Parameters.AddWithValue("@guid", file_g);
                        SqlComm.Parameters.AddWithValue("@content", reg.getMeta(2));
                        SqlComm.Parameters.AddWithValue("@end", DateTime.Now.ToString("yyyy-MM-dd h:mm:ss tt"));
                        SqlComm.ExecuteNonQuery();
                    }

                    SqlComm = new SqlCommand("UPDATE " + job_table + " SET End_Time=@end WHERE Job_Id=@guid", cnn);
                    SqlComm.Parameters.AddWithValue("@guid", g);
                    SqlComm.Parameters.AddWithValue("@end", DateTime.Now.ToString("yyyy-MM-dd h:mm:ss tt"));
                    SqlComm.ExecuteNonQuery();
                }
                cnn.Close();
            }
            catch (Exception e) { Console.WriteLine(e); }
        }