/// <summary> Process the IIS web logs to SQL insert commands </summary>
public void Process_IIS_Logs()
{
// **** READ THE LOOKUP TABLES FROM THE DATABASE **** //
DataSet lookupTables = SobekCM_Database.Get_Statistics_Lookup_Tables();
// ***** CODE BELOW READS ALL THE LOG FILES AND THEN WRITES THEM AS XML DATASETS *****//
SobekCM_Log_Reader ufdc_log_reader = new SobekCM_Log_Reader(lookupTables.Tables[0], sobekcm_web_location);
//string[] files = Directory.GetFiles(sobekcm_log_location, "*.log");
//foreach (string thisFile in files)
//{
// On_New_Status("Processing " + (new FileInfo(thisFile)).Name);
// ufdc_log_reader.Read_Log(thisFile).Write_XML(dataset_location);
//}
// ***** CODE BELOW READS ALL THE DAILY XML DATASETS AND COMBINES THEM INTO MONTHLY *****//
// ***** DATASETS WHICH ARE SUBSEQUENTLY WRITTEN AS XML DATASETS AS WELL *****//
On_New_Status("Combining daily datasets into a monthly dataset");
foreach (string year_month in year_months)
{
string[] year_month_files = Directory.GetFiles(dataset_location, year_month + "*.xml");
if (year_month_files.Length > 0)
{
SobekCM_Stats_DataSet combined = new SobekCM_Stats_DataSet();
foreach (string file in year_month_files)
{
if ((new FileInfo(file)).Name.IndexOf(year_month + ".xml") < 0)
{
SobekCM_Stats_DataSet daily = new SobekCM_Stats_DataSet();
daily.Read_XML(file);
combined.Merge(daily);
}
}
// Write the complete data set
combined.Write_XML(dataset_location, year_month + ".xml");
// Just write the highest users in a seperate, more readable, file
combined.Write_Highest_Users(dataset_location, "users_" + year_month + ".xml");
}
}
//// ***** CODE BELOW READS THE MONTHLY DATASETS AND THEN WRITES THE SQL INSERTION SCRIPTS ***** //
// Read all the data lists first for id lookups
Dictionary <string, int> aggregationHash = Table_To_Hash(lookupTables.Tables[2]);
Dictionary <string, int> bibHash = Table_To_Hash(lookupTables.Tables[1]);
Dictionary <string, int> portalHash = new Dictionary <string, int>();
foreach (DataRow thisRow in lookupTables.Tables[3].Rows)
{
if (!portalHash.ContainsKey(thisRow[2].ToString().ToUpper()))
{
portalHash[thisRow[2].ToString().ToUpper()] = Convert.ToInt32(thisRow[0]);
}
}
On_New_Status("Writing SQL insert commands");
foreach (string yearmonth in year_months)
{
SobekCM_Stats_DataSet monthly;
string thisFile = dataset_location + "\\" + yearmonth + ".xml";
if (File.Exists(thisFile))
{
monthly = new SobekCM_Stats_DataSet();
monthly.Read_XML(thisFile);
int year = Convert.ToInt32(yearmonth.Substring(0, 4));
int month = Convert.ToInt32(yearmonth.Substring(4));
monthly.Write_SQL_Inserts(sql_output_location + "\\" + yearmonth + ".sql", year, month, aggregationHash, bibHash, portalHash);
}
}
On_New_Status("COMPLETE!");
}