/// <summary> Merge this data with another existing set of data  </summary>
        /// <remarks> Used when merging daily sets into a monthly set </remarks>
        /// <param name="Stats_DataSet"> Set of usage data to merge with this one </param>
        public void Merge(SobekCM_Stats_DataSet Stats_DataSet)
        {
            // Add overall stats
            foreach (DataRow thisRow in Stats_DataSet.UFDC_Stats_Table.Rows)
            {
                sobekcm_stats.Rows[0][0] = Convert.ToInt32(sobekcm_stats.Rows[0][0]) + Convert.ToInt32(thisRow[0]);
                sobekcm_stats.Rows[0][1] = Convert.ToInt32(sobekcm_stats.Rows[0][1]) + Convert.ToInt32(thisRow[1]);
                sobekcm_stats.Rows[0][2] = Convert.ToInt32(sobekcm_stats.Rows[0][2]) + Convert.ToInt32(thisRow[2]);
                sobekcm_stats.Rows[0][3] = Convert.ToInt32(sobekcm_stats.Rows[0][3]) + Convert.ToInt32(thisRow[3]);
                sobekcm_stats.Rows[0][4] = Convert.ToInt32(sobekcm_stats.Rows[0][4]) + Convert.ToInt32(thisRow[4]);
                sobekcm_stats.Rows[0][5] = Convert.ToInt32(sobekcm_stats.Rows[0][5]) + Convert.ToInt32(thisRow[5]);
            }

            // Add collection stats
            foreach (DataRow thisRow in Stats_DataSet.Collection_Stats_Table.Rows)
            {
                if (collection_rows.ContainsKey(thisRow[0].ToString()))
                {
                    // Sum the two rows together
                    DataRow matchRow = collection_rows[thisRow[0].ToString()];
                    matchRow[1] = Convert.ToInt32(matchRow[1]) + Convert.ToInt32(thisRow[1]);
                    matchRow[2] = Convert.ToInt32(matchRow[2]) + Convert.ToInt32(thisRow[2]);
                    matchRow[3] = Convert.ToInt32(matchRow[3]) + Convert.ToInt32(thisRow[3]);
                    matchRow[4] = Convert.ToInt32(matchRow[4]) + Convert.ToInt32(thisRow[4]);
                    matchRow[5] = Convert.ToInt32(matchRow[5]) + Convert.ToInt32(thisRow[5]);
                }
                else
                {
                    // Add as a new row
                    DataRow newCollectionRow = collection_stats.NewRow();
                    newCollectionRow[0] = thisRow[0];
                    newCollectionRow[1] = thisRow[1];
                    newCollectionRow[2] = thisRow[2];
                    newCollectionRow[3] = thisRow[3];
                    newCollectionRow[4] = thisRow[4];
                    newCollectionRow[5] = thisRow[5];
                    collection_stats.Rows.Add(newCollectionRow);
                    collection_rows.Add(thisRow[0].ToString(), newCollectionRow);
                }
            }

            // Add institutional stats
            foreach (DataRow thisRow in Stats_DataSet.Institution_Stats_Table.Rows)
            {
                if (institution_rows.ContainsKey(thisRow[0].ToString()))
                {
                    // Sum the two rows together
                    DataRow matchRow = institution_rows[thisRow[0].ToString()];
                    matchRow[1] = Convert.ToInt32(matchRow[1]) + Convert.ToInt32(thisRow[1]);
                    matchRow[2] = Convert.ToInt32(matchRow[2]) + Convert.ToInt32(thisRow[2]);
                    matchRow[3] = Convert.ToInt32(matchRow[3]) + Convert.ToInt32(thisRow[3]);
                    matchRow[4] = Convert.ToInt32(matchRow[4]) + Convert.ToInt32(thisRow[4]);
                    matchRow[5] = Convert.ToInt32(matchRow[5]) + Convert.ToInt32(thisRow[5]);
                }
                else
                {
                    // Add as a new row
                    DataRow newInstitutionRow = institution_stats.NewRow();
                    newInstitutionRow[0] = thisRow[0];
                    newInstitutionRow[1] = thisRow[1];
                    newInstitutionRow[2] = thisRow[2];
                    newInstitutionRow[3] = thisRow[3];
                    newInstitutionRow[4] = thisRow[4];
                    newInstitutionRow[5] = thisRow[5];
                    institution_stats.Rows.Add(newInstitutionRow);
                    institution_rows.Add(thisRow[0].ToString(), newInstitutionRow);
                }
            }

            // Add the bib-level stats
            foreach (DataRow thisRow in Stats_DataSet.Bib_Stats_Table.Rows)
            {
                if (bib_rows.ContainsKey(thisRow[0].ToString()))
                {
                    // Sum the two rows together
                    DataRow matchRow = bib_rows[thisRow[0].ToString()];
                    matchRow[1] = Convert.ToInt32(matchRow[1]) + Convert.ToInt32(thisRow[1]);
                    matchRow[2] = Convert.ToInt32(matchRow[2]) + Convert.ToInt32(thisRow[2]);
                }
                else
                {
                    // Add as a new row
                    DataRow newBibRow = bib_stats.NewRow();
                    newBibRow[0] = thisRow[0];
                    newBibRow[1] = thisRow[1];
                    newBibRow[2] = thisRow[2];
                    bib_stats.Rows.Add(newBibRow);
                    bib_rows.Add(thisRow[0].ToString(), newBibRow);
                }
            }

            // Add the item-level stats
            foreach (DataRow thisRow in Stats_DataSet.Item_Stats_Table.Rows)
            {
                if (item_rows.ContainsKey(Convert.ToInt32(thisRow["itemID"])))
                {
                    // Sum the two rows together
                    DataRow matchRow = item_rows[Convert.ToInt32(thisRow["itemID"])];
                    matchRow[3] = Convert.ToInt32(matchRow[3]) + Convert.ToInt32(thisRow[3]);
                    matchRow[4] = Convert.ToInt32(matchRow[4]) + Convert.ToInt32(thisRow[4]);
                    matchRow[5] = Convert.ToInt32(matchRow[5]) + Convert.ToInt32(thisRow[5]);
                    matchRow[6] = Convert.ToInt32(matchRow[6]) + Convert.ToInt32(thisRow[6]);
                    matchRow[7] = Convert.ToInt32(matchRow[7]) + Convert.ToInt32(thisRow[7]);
                    matchRow[8] = Convert.ToInt32(matchRow[8]) + Convert.ToInt32(thisRow[8]);
                    matchRow[9] = Convert.ToInt32(matchRow[9]) + Convert.ToInt32(thisRow[9]);
                    matchRow[10] = Convert.ToInt32(matchRow[10]) + Convert.ToInt32(thisRow[10]);
                    matchRow[11] = Convert.ToInt32(matchRow[11]) + Convert.ToInt32(thisRow[11]);
                    matchRow[12] = Convert.ToInt32(matchRow[12]) + Convert.ToInt32(thisRow[12]);
                    matchRow[13] = Convert.ToInt32(matchRow[13]) + Convert.ToInt32(thisRow[13]);
                    matchRow[14] = Convert.ToInt32(matchRow[14]) + Convert.ToInt32(thisRow[14]);
                }
                else
                {
                    // Add as a new row
                    DataRow newItemRow = item_stats.NewRow();
                    newItemRow[0] = thisRow[0];
                    newItemRow[1] = thisRow[1];
                    newItemRow[2] = thisRow[2];
                    newItemRow[3] = thisRow[3];
                    newItemRow[4] = thisRow[4];
                    newItemRow[5] = thisRow[5];
                    newItemRow[6] = thisRow[6];
                    newItemRow[7] = thisRow[7];
                    newItemRow[8] = thisRow[8];
                    newItemRow[9] = thisRow[9];
                    newItemRow[10] = thisRow[10];
                    newItemRow[11] = thisRow[11];
                    newItemRow[12] = thisRow[12];
                    newItemRow[13] = thisRow[13];
                    newItemRow[14] = thisRow[14];
                    item_stats.Rows.Add(newItemRow);
                    item_rows.Add(Convert.ToInt32(thisRow["itemID"]), newItemRow);
                }
            }

            // Add the IP addresses
            foreach (DataRow thisRow in Stats_DataSet.IP_Addresses.Rows)
            {
                if (ip_rows.ContainsKey(thisRow[0].ToString()))
                {
                    // Sum the two rows together
                    DataRow matchRow = ip_rows[thisRow[0].ToString()];
                    matchRow[2] = Convert.ToInt32(matchRow[2]) + Convert.ToInt32(thisRow[2]);
                }
                else
                {
                    // Add as a new row
                    DataRow newIpRow = ip_addresses.NewRow();
                    newIpRow[0] = thisRow[0];
                    newIpRow[1] = thisRow[1];
                    newIpRow[2] = thisRow[2];
                    ip_addresses.Rows.Add(newIpRow);
                    ip_rows.Add(thisRow[0].ToString(), newIpRow);
                }
            }

            // Add the portal stats
            if (Stats_DataSet.Portal_Stats_Table != null)
            {
                foreach (DataRow thisRow in Stats_DataSet.Portal_Stats_Table.Rows)
                {
                    string portal = thisRow[0].ToString().ToUpper();
                    if (portal_rows.ContainsKey(portal))
                    {
                        // Sum the two rows together
                        DataRow matchRow = portal_rows[portal];
                        matchRow[1] = Convert.ToInt32(matchRow[1]) + Convert.ToInt32(thisRow[1]);
                    }
                    else
                    {
                        // Add as a new row
                        DataRow newPortalRow = portal_stats.NewRow();
                        newPortalRow[0] = portal;
                        newPortalRow[1] = thisRow[1];
                        portal_stats.Rows.Add(newPortalRow);
                        portal_rows.Add(portal, newPortalRow);
                    }
                }
            }

            // Add the webcontent stats
            if (Stats_DataSet.WebContent_Stats_Table != null)
            {
                foreach (DataRow thisRow in Stats_DataSet.WebContent_Stats_Table.Rows)
                {
                    if (webcontent_rows.ContainsKey(thisRow[0].ToString()))
                    {
                        // Sum the two rows together
                        DataRow matchRow = webcontent_rows[thisRow[0].ToString()];
                        matchRow[1] = Convert.ToInt32(matchRow[1]) + Convert.ToInt32(thisRow[1]);
                    }
                    else
                    {
                        // Add as a new row
                        DataRow newWebContentRow = webcontent_stats.NewRow();
                        newWebContentRow[0] = thisRow[0];
                        newWebContentRow[1] = thisRow[1];
                        webcontent_stats.Rows.Add(newWebContentRow);
                        webcontent_rows.Add(thisRow[0].ToString(), newWebContentRow);
                    }
                }
            }
        }
        /// <summary> Process the IIS web logs to SQL insert commands </summary>
        public void Process_IIS_Logs()
        {
            // **** READ THE LOOKUP TABLES FROM THE DATABASE **** //
            DataSet lookupTables = SobekCM_Database.Get_Statistics_Lookup_Tables();

            // ***** CODE BELOW READS ALL THE LOG FILES AND THEN WRITES THEM AS XML DATASETS *****//
            SobekCM_Log_Reader ufdc_log_reader = new SobekCM_Log_Reader(lookupTables.Tables[0], sobekcm_web_location);
            //string[] files = Directory.GetFiles(sobekcm_log_location, "*.log");
            //foreach (string thisFile in files)
            //{
            //    On_New_Status("Processing " + (new FileInfo(thisFile)).Name);
            //    ufdc_log_reader.Read_Log(thisFile).Write_XML(dataset_location);
            //}

            // ***** CODE BELOW READS ALL THE DAILY XML DATASETS AND COMBINES THEM INTO MONTHLY *****//
            // ***** DATASETS WHICH ARE SUBSEQUENTLY WRITTEN AS XML DATASETS AS WELL            *****//
            On_New_Status("Combining daily datasets into a monthly dataset");
            foreach (string year_month in year_months)
            {
                string[] year_month_files = Directory.GetFiles(dataset_location, year_month + "*.xml");
                if (year_month_files.Length > 0)
                {
                    SobekCM_Stats_DataSet combined = new SobekCM_Stats_DataSet();
                    foreach (string file in year_month_files)
                    {
                        if ((new FileInfo(file)).Name.IndexOf(year_month + ".xml") < 0)
                        {
                            SobekCM_Stats_DataSet daily = new SobekCM_Stats_DataSet();
                            daily.Read_XML(file);

                            combined.Merge(daily);
                        }
                    }

                    // Write the complete data set
                    combined.Write_XML(dataset_location, year_month + ".xml");

                    // Just write the highest users in a seperate, more readable, file
                    combined.Write_Highest_Users(dataset_location, "users_" + year_month + ".xml");
                }
            }

            //// ***** CODE BELOW READS THE MONTHLY DATASETS AND THEN WRITES THE SQL INSERTION SCRIPTS ***** //
            // Read all the data lists first for id lookups
            Dictionary<string, int> aggregationHash = Table_To_Hash(lookupTables.Tables[2]);
            Dictionary<string, int> bibHash = Table_To_Hash(lookupTables.Tables[1]);
            Dictionary<string, int> portalHash = new Dictionary<string, int>();
            foreach (DataRow thisRow in lookupTables.Tables[3].Rows)
            {
                if (!portalHash.ContainsKey(thisRow[2].ToString().ToUpper()))
                {
                    portalHash[thisRow[2].ToString().ToUpper()] = Convert.ToInt32(thisRow[0]);
                }
            }

            On_New_Status("Writing SQL insert commands");
            foreach (string yearmonth in year_months)
            {
                SobekCM_Stats_DataSet monthly;
                string thisFile = dataset_location + "\\" + yearmonth + ".xml";
                if (File.Exists(thisFile))
                {
                    monthly = new SobekCM_Stats_DataSet();
                    monthly.Read_XML(thisFile);
                    int year = Convert.ToInt32(yearmonth.Substring(0, 4));
                    int month = Convert.ToInt32(yearmonth.Substring(4));
                    monthly.Write_SQL_Inserts(sql_output_location + "\\" + yearmonth + ".sql", year, month, aggregationHash, bibHash, portalHash);
                }
            }

            On_New_Status("COMPLETE!");
        }
        /// <summary> Read a IIS web log, analyze completely, and return the corresponding <see cref="SobekCM_Stats_DataSet"/> object </summary>
        /// <param name="Log_File"> Location for the log file to read </param>
        /// <returns> Object with all the analyzed hits and sessions from the web log </returns>
        public SobekCM_Stats_DataSet Read_Log(string Log_File)
        {
            // Get list of ips which include dloc.com
            dloc_ips = new List<string>();

            // Create the list of hits
            hits = new SortedList<SobekCM_Hit, SobekCM_Hit>();

            // Create the list of sessions
            sessions = new Dictionary<string, SobekCM_Session>();

            // Create the return set
            SobekCM_Stats_DataSet returnValue = new SobekCM_Stats_DataSet();

            // Get the date of the log file
            FileInfo fileInfo = new FileInfo(Log_File);
            string name = fileInfo.Name.Replace(fileInfo.Extension, "");
            DateTime logDate = new DateTime(Convert.ToInt32("20" + name.Substring(4, 2)),
                                            Convert.ToInt32(name.Substring(6, 2)), Convert.ToInt32(name.Substring(8, 2)));
            returnValue.Date = logDate;

            // Open a connection to the log file and save each hit
            StreamReader reader = new StreamReader(Log_File);
            string line = reader.ReadLine();
            while (line != null)
            {
                parse_line(line);
                line = reader.ReadLine();
            }

            // Now, step through each hit in the list
            foreach (SobekCM_Hit hit in hits.Values)
            {
                if (hit.UFDC_URL.ToUpper().IndexOf(".ASPX") < 0)
                {
                    // Always increment the hits
                    returnValue.Increment_Hits();

                    // Add this IP hit
                    returnValue.Add_IP_Hit(hit.IP, hit.UserAgent);

                    // Shouldn't start with '/'
                    if (hit.UFDC_URL[0] == '/')
                    {
                        hit.UFDC_URL = hit.UFDC_URL.Substring(1);
                    }
                    hit.UFDC_URL = hit.UFDC_URL.ToLower();
                    if (hit.UFDC_URL.IndexOf("design/webcontent/") == 0)
                        hit.UFDC_URL = hit.UFDC_URL.Substring(18);

                    // Add this as a webcontent hit
                    returnValue.Add_WebContent_Hit(hit.UFDC_URL);
                }
                else
                {
                    // parse the url
                    string[] splitter = hit.Query_String.ToLower().Split("&".ToCharArray());
                    NameValueCollection queryStringCollection = new NameValueCollection();
                    foreach (string thisSplit in splitter)
                    {
                        int equals_index = thisSplit.IndexOf("=");
                        if ((equals_index > 0) && (equals_index < thisSplit.Length - 1))
                        {
                            string query_name = thisSplit.Substring(0, equals_index);
                            string query_value = thisSplit.Substring(equals_index + 1);
                            queryStringCollection[query_name] = query_value;

                            if (query_name.ToLower() == "portal")
                                hit.UFDC_URL = query_value;
                        }
                    }

                    // Now, get the navigation object using the standard SobekCM method

                    try
                    {
                        SobekCM_Navigation_Object currentMode = new SobekCM_Navigation_Object(queryStringCollection, hit.UFDC_URL,
                                                                                              new string[] {"en"}, Codes, Collection_Aliases,
                                                                                              ref Item_Lookup_Object, URL_Portals, null);
                        if (currentMode != null)
                            currentMode.Set_Robot_Flag(hit.UserAgent, hit.IP);
                        if ((currentMode != null) && (!currentMode.Is_Robot))
                        {
                            // Always increment the hits
                            returnValue.Increment_Hits();

                            // Add this IP hit
                            returnValue.Add_IP_Hit(hit.IP, hit.UserAgent);

                            // Increment the portal hits
                            returnValue.Add_Portal_Hit(currentMode.SobekCM_Instance_Name.ToUpper());

                            // Check for pre-existing session
                            SobekCM_Session thisSession;
                            if (sessions.ContainsKey(hit.IP))
                            {
                                SobekCM_Session possibleSession = sessions[hit.IP];
                                TimeSpan difference = hit.Time.Subtract(possibleSession.Last_Hit);
                                if (difference.TotalMinutes >= 60)
                                {
                                    thisSession = new SobekCM_Session(hit.IP, hit.Time);
                                    sessions[hit.IP] = thisSession;

                                    returnValue.Increment_Sessions();
                                }
                                else
                                {
                                    possibleSession.Last_Hit = hit.Time;
                                    thisSession = possibleSession;
                                }
                            }
                            else
                            {
                                thisSession = new SobekCM_Session(hit.IP, hit.Time);
                                sessions.Add(hit.IP, thisSession);

                                returnValue.Increment_Sessions();
                            }

                            if ((currentMode.Mode == Display_Mode_Enum.Item_Display) ||
                                (currentMode.Mode == Display_Mode_Enum.Item_Print))
                            {
                                if ((currentMode.ItemID_DEPRECATED > 0) ||
                                    ((currentMode.VID.Length > 0) && (currentMode.BibID.Length > 0)))
                                {
                                    if (currentMode.ItemID_DEPRECATED <= 0)
                                    {
                                        DataRow[] ufdc2_bib_select =
                                            itemList.Select("bibid = '" + currentMode.BibID + "' and vid='" +
                                                            currentMode.VID + "'");
                                        if (ufdc2_bib_select.Length > 0)
                                        {
                                            currentMode.ItemID_DEPRECATED =
                                                Convert.ToInt32(ufdc2_bib_select[0]["itemid"]);
                                        }
                                    }

                                    returnValue.Add_Item_Hit(currentMode.ItemID_DEPRECATED, currentMode.BibID,
                                                             currentMode.VID, currentMode.ViewerCode.ToUpper(),
                                                             currentMode.Text_Search, thisSession.SessionID);
                                }
                                else if (currentMode.BibID.Length > 0)
                                {
                                    returnValue.Add_Bib_Hit(currentMode.BibID.ToUpper(), thisSession.SessionID);
                                }
                            }
                            else
                            {
                                string code = currentMode.Aggregation;
                                string institution = String.Empty;
                                if ((code.Length > 0) && (code.ToUpper()[0] == 'I'))
                                {
                                    institution = code;
                                    code = String.Empty;
                                }

                                if ((institution.Length > 0) && (institution.ToUpper()[0] != 'I'))
                                    institution = "i" + institution;

                                // For some collections we are counting the institution hit and collection
                                // hit just so the full use of the site is recorded
                                if (code.Length > 0)
                                {
                                    returnValue.Add_Collection_Hit(code.ToLower(), currentMode.Mode,
                                                                   thisSession.SessionID);
                                }

                                // Was this an institutional level hit?
                                if (institution.Length > 0)
                                {
                                    returnValue.Add_Institution_Hit(institution.ToLower(), currentMode.Mode,
                                                                    thisSession.SessionID);
                                }

                                // Is this a static "webcontent" top-level page?
                                if (currentMode.Mode == Display_Mode_Enum.Simple_HTML_CMS)
                                {
                                    if ((currentMode.Info_Browse_Mode != "unknown") &&
                                        (currentMode.Info_Browse_Mode != "default"))
                                    {
                                        returnValue.Add_WebContent_Hit(currentMode.Info_Browse_Mode.ToLower());
                                    }
                                }

                                // Add the write type, if not normal HTML stuff
                                switch (currentMode.Writer_Type)
                                {
                                    case Writer_Type_Enum.DataSet:
                                    case Writer_Type_Enum.XML:
                                        returnValue.Add_XML_Hit();
                                        break;

                                    case Writer_Type_Enum.OAI:
                                        returnValue.Add_OAI_Hit();
                                        break;

                                    case Writer_Type_Enum.JSON:
                                        returnValue.Add_JSON_Hit();
                                        break;
                                }
                            }
                        }
                        else
                        {
                            if ((currentMode != null) && (currentMode.Is_Robot))
                                returnValue.Add_Robot_Hit();
                        }
                    }
                    catch (Exception ee)
                    {
                        bool error = true;
                    }

                    //System.Windows.Forms.MessageBox.Show("'" + hit.Time.ToString() + "' '" + hit.IP + "' '" + hit.UFDC_URL + "'");
                }
            }

            return returnValue;
        }