private bool CheckFile()
{
if (_fileName.StartsWith("--"))
{
List <string> files = IndihiangHelper.ParseFile(_fileName);
LogInfoEventArgs logInfo = null;
for (int i = 0; i < files.Count; i++)
{
if (!string.IsNullOrEmpty(files[i]))
{
if (!File.Exists(files[i]))
{
logInfo = new LogInfoEventArgs(
LogParserId.ToString(),
EnumLogFile.UNKNOWN,
LogProcessStatus.FAILED,
"LogParser.CheckFile()",
String.Format("{0} isn't found", files[i]));
_synContext.Post(OnAnalyzeLog, logInfo);
return(false);
}
}
}
logInfo = new LogInfoEventArgs(
LogParserId.ToString(),
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Process()",
string.Format("Total log files are {0} files", files.Count.ToString()));
_synContext.Post(OnAnalyzeLog, logInfo);
}
else
if (!File.Exists(_fileName))
{
LogInfoEventArgs logInfo = new LogInfoEventArgs(
LogParserId.ToString(),
EnumLogFile.UNKNOWN,
LogProcessStatus.FAILED,
"LogParser.CheckFile()",
String.Format("{0} isn't found", _fileName));
_synContext.Post(OnAnalyzeLog, logInfo);
return(false);
}
return(true);
}
private void DumpPartial(List <Indihiang.DomainObject.IPCountry> listIpCountry, Dictionary <string, string> cacheIpCountry)
{
if (!_dumpLogQueue.IsEmpty)
{
Debug.WriteLine(string.Format("Total dumo data: {0}", _dumpLogQueue.Count));
List <Indihiang.DomainObject.DumpData>[] list = _dumpLogQueue.ToArray();
for (int i = 0; i < list.Length; i++)
{
Debug.WriteLine(string.Format("Dump data: {0}:{1}", i + 1, _dumpLogQueue.Count));
for (int j = 0; j < list[i].Count; j++)
{
if (!string.IsNullOrEmpty(list[i][j].Client_IP))
{
if (!cacheIpCountry.ContainsKey(list[i][j].Client_IP))
{
double ip = IndihiangHelper.IPAddressToNumber(list[i][j].Client_IP);
for (int k = 0; k < listIpCountry.Count; k++)
{
if (ip >= listIpCountry[k].IpStart && ip <= listIpCountry[k].IpEnd)
{
Indihiang.DomainObject.DumpData obj = list[i][j];
obj.IPClientCountry = listIpCountry[k].CoutryName;
list[i][j] = obj;
cacheIpCountry.Add(list[i][j].Client_IP, listIpCountry[k].CoutryName);
break;
}
}
}
else
{
Indihiang.DomainObject.DumpData obj = list[i][j];
obj.IPClientCountry = cacheIpCountry[list[i][j].Client_IP];
list[i][j] = obj;
}
}
}
PerformDump(list[i]);
list[i].Clear();
}
}
cacheIpCountry.Clear();
listIpCountry.Clear();
_allDone = true;
}
private void PerformDump(List <Indihiang.DomainObject.DumpData> listDump)
{
if (listDump.Count > 0)
{
string file = string.Empty;
DataHelper helper = null;
file = IndihiangHelper.GetIndihiangFile(listDump[0].Year.ToString(), ParserID);
if (!File.Exists(file))
{
IndihiangHelper.CopyLogDB(file);
IndihiangHelper.InitialIndihiangFile(file, listDump[0].Year.ToString());
helper = new DataHelper(file);
}
else
{
helper = new DataHelper(file);
}
helper.InsertBulkDumpData(listDump);
}
}
protected void RunParse(string logFile)
{
List <Indihiang.DomainObject.DumpData> listDump = new List <Indihiang.DomainObject.DumpData>();
using (StreamReader sr = new StreamReader(File.Open(logFile, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
string line = sr.ReadLine();
List <string> currentHeader = new List <string>();
while (!string.IsNullOrEmpty(line))
{
if (IsLogHeader(line))
{
#region Parse Header
List <string> list1 = ParseHeader(line);
if (list1 != null)
{
currentHeader = new List <string>(list1);
}
#endregion
}
else
{
line = line.Replace('\0', ' ');
#region Parse Data
if (!string.IsNullOrEmpty(line))
{
string[] rows = line.Split(new char[] { ' ' });
if (rows != null)
{
if (rows.Length > 0)
{
string val = string.Empty;
Indihiang.DomainObject.DumpData dump = new Indihiang.DomainObject.DumpData();
dump.FullFileName = logFile;
for (int i = 0; i < currentHeader.Count; i++)
{
try
{
if (currentHeader[i].Equals("date"))
{
val = rows[i];
if (string.IsNullOrEmpty(val))
{
continue;
}
DateTime datetime = DateTime.Parse(val);
dump.Day = datetime.Day;
dump.Month = datetime.Month;
dump.Year = datetime.Year;
}
if (currentHeader[i].Equals("s-ip"))
{
dump.Server_IP = rows[i];
}
if (currentHeader[i].Equals("s-port"))
{
dump.Server_Port = rows[i];
}
if (currentHeader[i].Equals("cs-uri-stem"))
{
dump.Page_Access = rows[i];
}
if (currentHeader[i].Equals("cs-uri-query"))
{
dump.Query_Page_Access = rows[i];
}
if (currentHeader[i].Equals("cs-username"))
{
dump.Access_Username = rows[i];
}
if (currentHeader[i].Equals("c-ip"))
{
dump.Client_IP = rows[i];
}
if (currentHeader[i].Equals("cs(User-Agent)"))
{
dump.User_Agent = IndihiangHelper.CheckUserAgent(rows[i]);
}
if (currentHeader[i].Equals("sc-status"))
{
if (dump.Protocol_Status.Contains("."))
{
dump.Protocol_Status = string.Format("{0}{1}", rows[i], dump.Protocol_Status);
}
else
{
dump.Protocol_Status = rows[i];
}
}
if (currentHeader[i].Equals("sc-substatus"))
{
dump.Protocol_Status = string.Format("{0}.{1}", dump.Protocol_Status, rows[i]);
}
if (currentHeader[i].Equals("cs(Referer)"))
{
dump.Referer = rows[i];
dump.RefererClass = IndihiangHelper.GetRefererClass(rows[i]);
}
if (currentHeader[i].Equals("sc-bytes"))
{
if (!string.IsNullOrEmpty(rows[i]))
{
dump.Bytes_Sent = Convert.ToInt64(rows[i]);
}
else
{
dump.Bytes_Sent = 0;
}
}
if (currentHeader[i].Equals("cs-bytes"))
{
if (!string.IsNullOrEmpty(rows[i]))
{
dump.Bytes_Received = Convert.ToInt64(rows[i]);
}
else
{
dump.Bytes_Received = 0;
}
}
if (currentHeader[i].Equals("time-taken"))
{
if (!string.IsNullOrEmpty(rows[i]))
{
dump.TimeTaken = Convert.ToInt64(rows[i]);
}
else
{
dump.TimeTaken = 0;
}
}
}
catch (Exception err)
{
Console.WriteLine(err.Message);
}
}
if (dump.Day > 0 && dump.Month > 0)
{
listDump.Add(dump);
}
}
}
}
#endregion
}
line = sr.ReadLine();
if (!string.IsNullOrEmpty(line))
{
line = line.Trim();
}
}
if (listDump.Count > 0)
{
_dumpLogQueue.Enqueue(listDump);
}
}
}
private void DumpData()
{
List <Indihiang.DomainObject.DumpData> listDump;
DataHelper helper = new DataHelper(IndihiangHelper.GetIPCountryDb());
List <Indihiang.DomainObject.IPCountry> listIpCountry = helper.GetAllIpCountry();
Dictionary <string, string> cacheIpCountry = new Dictionary <string, string>();
cacheIpCountry.Add("127.0.0.1", "(Local)");
while (!_finish)
{
try
{
if (_dumpLogQueue.TryDequeue(out listDump))
{
for (int j = 0; j < listDump.Count; j++)
{
if (!string.IsNullOrEmpty(listDump[j].Client_IP))
{
if (!cacheIpCountry.ContainsKey(listDump[j].Client_IP))
{
double ip = IndihiangHelper.IPAddressToNumber(listDump[j].Client_IP);
for (int k = 0; k < listIpCountry.Count; k++)
{
if (ip >= listIpCountry[k].IpStart && ip <= listIpCountry[k].IpEnd)
{
Indihiang.DomainObject.DumpData obj = listDump[j];
obj.IPClientCountry = listIpCountry[k].CoutryName;
listDump[j] = obj;
cacheIpCountry.Add(listDump[j].Client_IP, listIpCountry[k].CoutryName);
break;
}
}
}
else
{
Indihiang.DomainObject.DumpData obj = listDump[j];
obj.IPClientCountry = cacheIpCountry[listDump[j].Client_IP];
listDump[j] = obj;
}
}
}
PerformDump(listDump);
listDump.Clear();
}
else
{
Thread.Sleep(10);
}
}
catch (Exception err)
{
Logger.Write(err.Message);
Logger.Write(err.StackTrace);
Debug.WriteLine(err.Message);
}
}
if (!_dumpLogQueue.IsEmpty)
{
Debug.WriteLine(string.Format("Total remain data: {0}", _dumpLogQueue.Count));
List <Indihiang.DomainObject.DumpData>[] list = _dumpLogQueue.ToArray();
for (int i = 0; i < list.Length; i++)
{
Debug.WriteLine(string.Format("Dump data: {0}:{1}", i + 1, _dumpLogQueue.Count));
for (int j = 0; j < list[i].Count; j++)
{
if (!string.IsNullOrEmpty(list[i][j].Client_IP))
{
if (!cacheIpCountry.ContainsKey(list[i][j].Client_IP))
{
double ip = IndihiangHelper.IPAddressToNumber(list[i][j].Client_IP);
for (int k = 0; k < listIpCountry.Count; k++)
{
if (ip >= listIpCountry[k].IpStart && ip <= listIpCountry[k].IpEnd)
{
Indihiang.DomainObject.DumpData obj = list[i][j];
obj.IPClientCountry = listIpCountry[k].CoutryName;
list[i][j] = obj;
cacheIpCountry.Add(list[i][j].Client_IP, listIpCountry[k].CoutryName);
break;
}
}
}
else
{
Indihiang.DomainObject.DumpData obj = list[i][j];
obj.IPClientCountry = cacheIpCountry[list[i][j].Client_IP];
list[i][j] = obj;
}
}
}
PerformDump(list[i]);
list[i].Clear();
}
}
cacheIpCountry.Clear();
listIpCountry.Clear();
_exitDump.Set();
}
protected void RunParse(string logFile)
{
List <Indihiang.DomainObject.DumpData> listDump = new List <Indihiang.DomainObject.DumpData>();
bool bRead = false;
DateTime dt = DateTime.Now;
string strError = "(unknown)";
retry:
if (((DateTime.Now - dt).TotalSeconds > 5) && (System.Windows.Forms.MessageBox.Show("Timed out trying to open log file:\n\n" + logFile + "\n\nError: " + strError + "\n\nTry again?", "Time out", System.Windows.Forms.MessageBoxButtons.YesNo, System.Windows.Forms.MessageBoxIcon.Warning, System.Windows.Forms.MessageBoxDefaultButton.Button1) != System.Windows.Forms.DialogResult.Yes))
{
return;
}
try
{
using (StreamReader sr = new StreamReader(File.Open(logFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) // FileShare.Read
{
bRead = true;
string line = sr.ReadLine();
List <string> currentHeader = new List <string>();
while (!string.IsNullOrEmpty(line))
{
if (IsLogHeader(line))
{
#region Parse Header
List <string> list1 = ParseHeader(line);
if (list1 != null)
{
currentHeader = new List <string>(list1);
}
#endregion
}
else
{
line = line.Replace('\0', ' ');
#region Parse Data
if (!string.IsNullOrEmpty(line))
{
string[] rows = line.Split(new char[] { ' ' });
if (rows != null)
{
if (rows.Length > 0)
{
string val = string.Empty;
Indihiang.DomainObject.DumpData dump = new Indihiang.DomainObject.DumpData();
dump.FullFileName = logFile;
for (int i = 0; i < currentHeader.Count; i++)
{
try
{
if (currentHeader[i].Equals("date"))
{
val = rows[i];
if (string.IsNullOrEmpty(val))
{
continue;
}
DateTime datetime = DateTime.Parse(val);
dump.Day = datetime.Day;
dump.Month = datetime.Month;
dump.Year = datetime.Year;
}
if (currentHeader[i].Equals("s-ip"))
{
dump.Server_IP = rows[i];
}
if (currentHeader[i].Equals("s-port"))
{
dump.Server_Port = rows[i];
}
if (currentHeader[i].Equals("cs-uri-stem"))
{
dump.Page_Access = rows[i];
}
if (currentHeader[i].Equals("cs-uri-query"))
{
dump.Query_Page_Access = rows[i];
}
if (currentHeader[i].Equals("cs-username"))
{
dump.Access_Username = rows[i];
}
if (currentHeader[i].Equals("c-ip"))
{
dump.Client_IP = rows[i];
}
if (currentHeader[i].Equals("cs(User-Agent)"))
{
dump.User_Agent = IndihiangHelper.CheckUserAgent(rows[i]);
}
if (currentHeader[i].Equals("sc-status"))
{
if (dump.Protocol_Status.Contains("."))
{
dump.Protocol_Status = string.Format("{0}{1}", rows[i], dump.Protocol_Status);
}
else
{
dump.Protocol_Status = rows[i];
}
}
if (currentHeader[i].Equals("sc-substatus"))
{
dump.Protocol_Status = string.Format("{0}.{1}", dump.Protocol_Status, rows[i]);
}
if (currentHeader[i].Equals("cs(Referer)"))
{
dump.Referer = rows[i];
dump.RefererClass = IndihiangHelper.GetRefererClass(rows[i]);
}
if (currentHeader[i].Equals("sc-bytes"))
{
if (!string.IsNullOrEmpty(rows[i]))
{
dump.Bytes_Sent = Convert.ToInt64(rows[i]);
}
else
{
dump.Bytes_Sent = 0;
}
}
if (currentHeader[i].Equals("cs-bytes"))
{
if (!string.IsNullOrEmpty(rows[i]))
{
dump.Bytes_Received = Convert.ToInt64(rows[i]);
}
else
{
dump.Bytes_Received = 0;
}
}
if (currentHeader[i].Equals("time-taken"))
{
if (!string.IsNullOrEmpty(rows[i]))
{
dump.TimeTaken = Convert.ToInt64(rows[i]);
}
else
{
dump.TimeTaken = 0;
}
}
}
catch (Exception err)
{
Console.WriteLine(err.Message);
}
}
if (dump.Day > 0 && dump.Month > 0)
{
listDump.Add(dump);
}
}
}
}
#endregion
}
line = sr.ReadLine();
if (!string.IsNullOrEmpty(line))
{
line = line.Trim();
}
}
if (listDump.Count > 0)
{
_dumpLogQueue.Enqueue(listDump);
}
}
}
/*catch (IOException)
* {
* //
* }*/
catch (Exception ex)
{
strError = ex.Message;
}
if (bRead == false)
{
goto retry;
}
}
private bool SequentialParse(bool success, List <string> listFiles)
{
try
{
//List<ManualResetEventSlim> resets = new List<ManualResetEventSlim>();
//_exitDump = new ManualResetEventSlim(false);
DataHelper helper = new DataHelper(IndihiangHelper.GetIPCountryDb());
List <Indihiang.DomainObject.IPCountry> listIpCountry = helper.GetAllIpCountry();
Dictionary <string, string> cacheIpCountry = new Dictionary <string, string>();
cacheIpCountry.Add("127.0.0.1", "(Local)");
for (int i = 0; i < listFiles.Count; i++)
{
LogInfoEventArgs logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Run Parse on file {0}", listFiles[i]));
_synContext.Post(OnParseLog, logInfo);
try
{
RunParse(listFiles[i]);
DumpPartial(listIpCountry, cacheIpCountry);
logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Run Parse : {0} was done", listFiles[i]));
_synContext.Post(OnParseLog, logInfo);
}
catch (Exception err)
{
logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Error occurred on file {0}==>{1}\r\n{2}", listFiles[i], err.Message, err.StackTrace));
_synContext.Post(OnParseLog, logInfo);
}
}
_finish = true;
//LogInfoEventArgs logInfo2 = new LogInfoEventArgs(
// ParserID,
// EnumLogFile.UNKNOWN,
// LogProcessStatus.SUCCESS,
// "Parse()",
// "Consolidating log files...");
//_synContext.Post(OnParseLog, logInfo2);
//try
//{
// //DumpData();
// //if (!_exitDump.IsSet)
// // _exitDump.Wait();
//}
//catch { }
LogInfoEventArgs logInfo2 = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
"Consolidated log file was done");
_synContext.Post(OnParseLog, logInfo2);
Thread.Sleep(100);
success = true;
}
catch (AggregateException err)
{
Logger.Write(err.Message);
Logger.Write(err.StackTrace);
#region Handle Exception
LogInfoEventArgs logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Internal Error: {0}", err.Message));
_synContext.Post(OnParseLog, logInfo);
logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Source Internal Error: {0}", err.Source));
_synContext.Post(OnParseLog, logInfo);
logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Detail Internal Error: {0}", err.StackTrace));
_synContext.Post(OnParseLog, logInfo);
logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Internal Exception Error: {0}", err.InnerException.Message));
_synContext.Post(OnParseLog, logInfo);
#endregion
}
catch (Exception err)
{
Logger.Write(err.Message);
Logger.Write(err.StackTrace);
#region Handle Exception
LogInfoEventArgs logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Internal Error: {0}", err.Message));
_synContext.Post(OnParseLog, logInfo);
logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Source Internal Error: {0}", err.Source));
_synContext.Post(OnParseLog, logInfo);
logInfo = new LogInfoEventArgs(
ParserID,
EnumLogFile.UNKNOWN,
LogProcessStatus.SUCCESS,
"Parse()",
String.Format("Detail Internal Error: {0}", err.StackTrace));
_synContext.Post(OnParseLog, logInfo);
#endregion
}
return(success);
}