/// <summary> Process the IIS web logs to SQL insert commands </summary> public void Process_IIS_Logs() { // **** READ THE LOOKUP TABLES FROM THE DATABASE **** // DataSet lookupTables = SobekCM_Database.Get_Statistics_Lookup_Tables(); // ***** CODE BELOW READS ALL THE LOG FILES AND THEN WRITES THEM AS XML DATASETS *****// SobekCM_Log_Reader ufdc_log_reader = new SobekCM_Log_Reader(lookupTables.Tables[0], sobekcm_web_location); //string[] files = Directory.GetFiles(sobekcm_log_location, "*.log"); //foreach (string thisFile in files) //{ // On_New_Status("Processing " + (new FileInfo(thisFile)).Name); // ufdc_log_reader.Read_Log(thisFile).Write_XML(dataset_location); //} // ***** CODE BELOW READS ALL THE DAILY XML DATASETS AND COMBINES THEM INTO MONTHLY *****// // ***** DATASETS WHICH ARE SUBSEQUENTLY WRITTEN AS XML DATASETS AS WELL *****// On_New_Status("Combining daily datasets into a monthly dataset"); foreach (string year_month in year_months) { string[] year_month_files = Directory.GetFiles(dataset_location, year_month + "*.xml"); if (year_month_files.Length > 0) { SobekCM_Stats_DataSet combined = new SobekCM_Stats_DataSet(); foreach (string file in year_month_files) { if ((new FileInfo(file)).Name.IndexOf(year_month + ".xml") < 0) { SobekCM_Stats_DataSet daily = new SobekCM_Stats_DataSet(); daily.Read_XML(file); combined.Merge(daily); } } // Write the complete data set combined.Write_XML(dataset_location, year_month + ".xml"); // Just write the highest users in a seperate, more readable, file combined.Write_Highest_Users(dataset_location, "users_" + year_month + ".xml"); } } //// ***** CODE BELOW READS THE MONTHLY DATASETS AND THEN WRITES THE SQL INSERTION SCRIPTS ***** // // Read all the data lists first for id lookups Dictionary <string, int> aggregationHash = Table_To_Hash(lookupTables.Tables[2]); Dictionary <string, int> bibHash = Table_To_Hash(lookupTables.Tables[1]); Dictionary <string, int> portalHash = new Dictionary <string, int>(); foreach (DataRow thisRow in lookupTables.Tables[3].Rows) { if (!portalHash.ContainsKey(thisRow[2].ToString().ToUpper())) { portalHash[thisRow[2].ToString().ToUpper()] = Convert.ToInt32(thisRow[0]); } } On_New_Status("Writing SQL insert commands"); foreach (string yearmonth in year_months) { SobekCM_Stats_DataSet monthly; string thisFile = dataset_location + "\\" + yearmonth + ".xml"; if (File.Exists(thisFile)) { monthly = new SobekCM_Stats_DataSet(); monthly.Read_XML(thisFile); int year = Convert.ToInt32(yearmonth.Substring(0, 4)); int month = Convert.ToInt32(yearmonth.Substring(4)); monthly.Write_SQL_Inserts(sql_output_location + "\\" + yearmonth + ".sql", year, month, aggregationHash, bibHash, portalHash); } } On_New_Status("COMPLETE!"); }
/// <summary> Process the IIS web logs to SQL insert commands </summary> public void Process_IIS_Logs() { // **** READ THE LOOKUP TABLES FROM THE DATABASE **** // DataSet lookupTables = SobekCM_Database.Get_Statistics_Lookup_Tables(); // ***** CODE BELOW READS ALL THE LOG FILES AND THEN WRITES THEM AS XML DATASETS *****// SobekCM_Log_Reader ufdc_log_reader = new SobekCM_Log_Reader(lookupTables.Tables[0], sobekcm_web_location); //string[] files = Directory.GetFiles(sobekcm_log_location, "*.log"); //foreach (string thisFile in files) //{ // On_New_Status("Processing " + (new FileInfo(thisFile)).Name); // ufdc_log_reader.Read_Log(thisFile).Write_XML(dataset_location); //} // ***** CODE BELOW READS ALL THE DAILY XML DATASETS AND COMBINES THEM INTO MONTHLY *****// // ***** DATASETS WHICH ARE SUBSEQUENTLY WRITTEN AS XML DATASETS AS WELL *****// On_New_Status("Combining daily datasets into a monthly dataset"); foreach (string year_month in year_months) { string[] year_month_files = Directory.GetFiles(dataset_location, year_month + "*.xml"); if (year_month_files.Length > 0) { SobekCM_Stats_DataSet combined = new SobekCM_Stats_DataSet(); foreach (string file in year_month_files) { if ((new FileInfo(file)).Name.IndexOf(year_month + ".xml") < 0) { SobekCM_Stats_DataSet daily = new SobekCM_Stats_DataSet(); daily.Read_XML(file); combined.Merge(daily); } } // Write the complete data set combined.Write_XML(dataset_location, year_month + ".xml"); // Just write the highest users in a seperate, more readable, file combined.Write_Highest_Users(dataset_location, "users_" + year_month + ".xml"); } } //// ***** CODE BELOW READS THE MONTHLY DATASETS AND THEN WRITES THE SQL INSERTION SCRIPTS ***** // // Read all the data lists first for id lookups Dictionary<string, int> aggregationHash = Table_To_Hash(lookupTables.Tables[2]); Dictionary<string, int> bibHash = Table_To_Hash(lookupTables.Tables[1]); Dictionary<string, int> portalHash = new Dictionary<string, int>(); foreach (DataRow thisRow in lookupTables.Tables[3].Rows) { if (!portalHash.ContainsKey(thisRow[2].ToString().ToUpper())) { portalHash[thisRow[2].ToString().ToUpper()] = Convert.ToInt32(thisRow[0]); } } On_New_Status("Writing SQL insert commands"); foreach (string yearmonth in year_months) { SobekCM_Stats_DataSet monthly; string thisFile = dataset_location + "\\" + yearmonth + ".xml"; if (File.Exists(thisFile)) { monthly = new SobekCM_Stats_DataSet(); monthly.Read_XML(thisFile); int year = Convert.ToInt32(yearmonth.Substring(0, 4)); int month = Convert.ToInt32(yearmonth.Substring(4)); monthly.Write_SQL_Inserts(sql_output_location + "\\" + yearmonth + ".sql", year, month, aggregationHash, bibHash, portalHash); } } On_New_Status("COMPLETE!"); }