private bool CheckFile() { if (_fileName.StartsWith("--")) { List <string> files = IndihiangHelper.ParseFile(_fileName); LogInfoEventArgs logInfo = null; for (int i = 0; i < files.Count; i++) { if (!string.IsNullOrEmpty(files[i])) { if (!File.Exists(files[i])) { logInfo = new LogInfoEventArgs( LogParserId.ToString(), EnumLogFile.UNKNOWN, LogProcessStatus.FAILED, "LogParser.CheckFile()", String.Format("{0} isn't found", files[i])); _synContext.Post(OnAnalyzeLog, logInfo); return(false); } } } logInfo = new LogInfoEventArgs( LogParserId.ToString(), EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Process()", string.Format("Total log files are {0} files", files.Count.ToString())); _synContext.Post(OnAnalyzeLog, logInfo); } else if (!File.Exists(_fileName)) { LogInfoEventArgs logInfo = new LogInfoEventArgs( LogParserId.ToString(), EnumLogFile.UNKNOWN, LogProcessStatus.FAILED, "LogParser.CheckFile()", String.Format("{0} isn't found", _fileName)); _synContext.Post(OnAnalyzeLog, logInfo); return(false); } return(true); }
private void DumpPartial(List <Indihiang.DomainObject.IPCountry> listIpCountry, Dictionary <string, string> cacheIpCountry) { if (!_dumpLogQueue.IsEmpty) { Debug.WriteLine(string.Format("Total dumo data: {0}", _dumpLogQueue.Count)); List <Indihiang.DomainObject.DumpData>[] list = _dumpLogQueue.ToArray(); for (int i = 0; i < list.Length; i++) { Debug.WriteLine(string.Format("Dump data: {0}:{1}", i + 1, _dumpLogQueue.Count)); for (int j = 0; j < list[i].Count; j++) { if (!string.IsNullOrEmpty(list[i][j].Client_IP)) { if (!cacheIpCountry.ContainsKey(list[i][j].Client_IP)) { double ip = IndihiangHelper.IPAddressToNumber(list[i][j].Client_IP); for (int k = 0; k < listIpCountry.Count; k++) { if (ip >= listIpCountry[k].IpStart && ip <= listIpCountry[k].IpEnd) { Indihiang.DomainObject.DumpData obj = list[i][j]; obj.IPClientCountry = listIpCountry[k].CoutryName; list[i][j] = obj; cacheIpCountry.Add(list[i][j].Client_IP, listIpCountry[k].CoutryName); break; } } } else { Indihiang.DomainObject.DumpData obj = list[i][j]; obj.IPClientCountry = cacheIpCountry[list[i][j].Client_IP]; list[i][j] = obj; } } } PerformDump(list[i]); list[i].Clear(); } } cacheIpCountry.Clear(); listIpCountry.Clear(); _allDone = true; }
private void PerformDump(List <Indihiang.DomainObject.DumpData> listDump) { if (listDump.Count > 0) { string file = string.Empty; DataHelper helper = null; file = IndihiangHelper.GetIndihiangFile(listDump[0].Year.ToString(), ParserID); if (!File.Exists(file)) { IndihiangHelper.CopyLogDB(file); IndihiangHelper.InitialIndihiangFile(file, listDump[0].Year.ToString()); helper = new DataHelper(file); } else { helper = new DataHelper(file); } helper.InsertBulkDumpData(listDump); } }
protected void RunParse(string logFile) { List <Indihiang.DomainObject.DumpData> listDump = new List <Indihiang.DomainObject.DumpData>(); using (StreamReader sr = new StreamReader(File.Open(logFile, FileMode.Open, FileAccess.Read, FileShare.Read))) { string line = sr.ReadLine(); List <string> currentHeader = new List <string>(); while (!string.IsNullOrEmpty(line)) { if (IsLogHeader(line)) { #region Parse Header List <string> list1 = ParseHeader(line); if (list1 != null) { currentHeader = new List <string>(list1); } #endregion } else { line = line.Replace('\0', ' '); #region Parse Data if (!string.IsNullOrEmpty(line)) { string[] rows = line.Split(new char[] { ' ' }); if (rows != null) { if (rows.Length > 0) { string val = string.Empty; Indihiang.DomainObject.DumpData dump = new Indihiang.DomainObject.DumpData(); dump.FullFileName = logFile; for (int i = 0; i < currentHeader.Count; i++) { try { if (currentHeader[i].Equals("date")) { val = rows[i]; if (string.IsNullOrEmpty(val)) { continue; } DateTime datetime = DateTime.Parse(val); dump.Day = datetime.Day; dump.Month = datetime.Month; dump.Year = datetime.Year; } if (currentHeader[i].Equals("s-ip")) { dump.Server_IP = rows[i]; } if (currentHeader[i].Equals("s-port")) { dump.Server_Port = rows[i]; } if (currentHeader[i].Equals("cs-uri-stem")) { dump.Page_Access = rows[i]; } if (currentHeader[i].Equals("cs-uri-query")) { dump.Query_Page_Access = rows[i]; } if (currentHeader[i].Equals("cs-username")) { dump.Access_Username = rows[i]; } if (currentHeader[i].Equals("c-ip")) { dump.Client_IP = rows[i]; } if (currentHeader[i].Equals("cs(User-Agent)")) { dump.User_Agent = IndihiangHelper.CheckUserAgent(rows[i]); } if (currentHeader[i].Equals("sc-status")) { if (dump.Protocol_Status.Contains(".")) { dump.Protocol_Status = string.Format("{0}{1}", rows[i], dump.Protocol_Status); } else { dump.Protocol_Status = rows[i]; } } if (currentHeader[i].Equals("sc-substatus")) { dump.Protocol_Status = string.Format("{0}.{1}", dump.Protocol_Status, rows[i]); } if (currentHeader[i].Equals("cs(Referer)")) { dump.Referer = rows[i]; dump.RefererClass = IndihiangHelper.GetRefererClass(rows[i]); } if (currentHeader[i].Equals("sc-bytes")) { if (!string.IsNullOrEmpty(rows[i])) { dump.Bytes_Sent = Convert.ToInt64(rows[i]); } else { dump.Bytes_Sent = 0; } } if (currentHeader[i].Equals("cs-bytes")) { if (!string.IsNullOrEmpty(rows[i])) { dump.Bytes_Received = Convert.ToInt64(rows[i]); } else { dump.Bytes_Received = 0; } } if (currentHeader[i].Equals("time-taken")) { if (!string.IsNullOrEmpty(rows[i])) { dump.TimeTaken = Convert.ToInt64(rows[i]); } else { dump.TimeTaken = 0; } } } catch (Exception err) { Console.WriteLine(err.Message); } } if (dump.Day > 0 && dump.Month > 0) { listDump.Add(dump); } } } } #endregion } line = sr.ReadLine(); if (!string.IsNullOrEmpty(line)) { line = line.Trim(); } } if (listDump.Count > 0) { _dumpLogQueue.Enqueue(listDump); } } }
private void DumpData() { List <Indihiang.DomainObject.DumpData> listDump; DataHelper helper = new DataHelper(IndihiangHelper.GetIPCountryDb()); List <Indihiang.DomainObject.IPCountry> listIpCountry = helper.GetAllIpCountry(); Dictionary <string, string> cacheIpCountry = new Dictionary <string, string>(); cacheIpCountry.Add("127.0.0.1", "(Local)"); while (!_finish) { try { if (_dumpLogQueue.TryDequeue(out listDump)) { for (int j = 0; j < listDump.Count; j++) { if (!string.IsNullOrEmpty(listDump[j].Client_IP)) { if (!cacheIpCountry.ContainsKey(listDump[j].Client_IP)) { double ip = IndihiangHelper.IPAddressToNumber(listDump[j].Client_IP); for (int k = 0; k < listIpCountry.Count; k++) { if (ip >= listIpCountry[k].IpStart && ip <= listIpCountry[k].IpEnd) { Indihiang.DomainObject.DumpData obj = listDump[j]; obj.IPClientCountry = listIpCountry[k].CoutryName; listDump[j] = obj; cacheIpCountry.Add(listDump[j].Client_IP, listIpCountry[k].CoutryName); break; } } } else { Indihiang.DomainObject.DumpData obj = listDump[j]; obj.IPClientCountry = cacheIpCountry[listDump[j].Client_IP]; listDump[j] = obj; } } } PerformDump(listDump); listDump.Clear(); } else { Thread.Sleep(10); } } catch (Exception err) { Logger.Write(err.Message); Logger.Write(err.StackTrace); Debug.WriteLine(err.Message); } } if (!_dumpLogQueue.IsEmpty) { Debug.WriteLine(string.Format("Total remain data: {0}", _dumpLogQueue.Count)); List <Indihiang.DomainObject.DumpData>[] list = _dumpLogQueue.ToArray(); for (int i = 0; i < list.Length; i++) { Debug.WriteLine(string.Format("Dump data: {0}:{1}", i + 1, _dumpLogQueue.Count)); for (int j = 0; j < list[i].Count; j++) { if (!string.IsNullOrEmpty(list[i][j].Client_IP)) { if (!cacheIpCountry.ContainsKey(list[i][j].Client_IP)) { double ip = IndihiangHelper.IPAddressToNumber(list[i][j].Client_IP); for (int k = 0; k < listIpCountry.Count; k++) { if (ip >= listIpCountry[k].IpStart && ip <= listIpCountry[k].IpEnd) { Indihiang.DomainObject.DumpData obj = list[i][j]; obj.IPClientCountry = listIpCountry[k].CoutryName; list[i][j] = obj; cacheIpCountry.Add(list[i][j].Client_IP, listIpCountry[k].CoutryName); break; } } } else { Indihiang.DomainObject.DumpData obj = list[i][j]; obj.IPClientCountry = cacheIpCountry[list[i][j].Client_IP]; list[i][j] = obj; } } } PerformDump(list[i]); list[i].Clear(); } } cacheIpCountry.Clear(); listIpCountry.Clear(); _exitDump.Set(); }
protected void RunParse(string logFile) { List <Indihiang.DomainObject.DumpData> listDump = new List <Indihiang.DomainObject.DumpData>(); bool bRead = false; DateTime dt = DateTime.Now; string strError = "(unknown)"; retry: if (((DateTime.Now - dt).TotalSeconds > 5) && (System.Windows.Forms.MessageBox.Show("Timed out trying to open log file:\n\n" + logFile + "\n\nError: " + strError + "\n\nTry again?", "Time out", System.Windows.Forms.MessageBoxButtons.YesNo, System.Windows.Forms.MessageBoxIcon.Warning, System.Windows.Forms.MessageBoxDefaultButton.Button1) != System.Windows.Forms.DialogResult.Yes)) { return; } try { using (StreamReader sr = new StreamReader(File.Open(logFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) // FileShare.Read { bRead = true; string line = sr.ReadLine(); List <string> currentHeader = new List <string>(); while (!string.IsNullOrEmpty(line)) { if (IsLogHeader(line)) { #region Parse Header List <string> list1 = ParseHeader(line); if (list1 != null) { currentHeader = new List <string>(list1); } #endregion } else { line = line.Replace('\0', ' '); #region Parse Data if (!string.IsNullOrEmpty(line)) { string[] rows = line.Split(new char[] { ' ' }); if (rows != null) { if (rows.Length > 0) { string val = string.Empty; Indihiang.DomainObject.DumpData dump = new Indihiang.DomainObject.DumpData(); dump.FullFileName = logFile; for (int i = 0; i < currentHeader.Count; i++) { try { if (currentHeader[i].Equals("date")) { val = rows[i]; if (string.IsNullOrEmpty(val)) { continue; } DateTime datetime = DateTime.Parse(val); dump.Day = datetime.Day; dump.Month = datetime.Month; dump.Year = datetime.Year; } if (currentHeader[i].Equals("s-ip")) { dump.Server_IP = rows[i]; } if (currentHeader[i].Equals("s-port")) { dump.Server_Port = rows[i]; } if (currentHeader[i].Equals("cs-uri-stem")) { dump.Page_Access = rows[i]; } if (currentHeader[i].Equals("cs-uri-query")) { dump.Query_Page_Access = rows[i]; } if (currentHeader[i].Equals("cs-username")) { dump.Access_Username = rows[i]; } if (currentHeader[i].Equals("c-ip")) { dump.Client_IP = rows[i]; } if (currentHeader[i].Equals("cs(User-Agent)")) { dump.User_Agent = IndihiangHelper.CheckUserAgent(rows[i]); } if (currentHeader[i].Equals("sc-status")) { if (dump.Protocol_Status.Contains(".")) { dump.Protocol_Status = string.Format("{0}{1}", rows[i], dump.Protocol_Status); } else { dump.Protocol_Status = rows[i]; } } if (currentHeader[i].Equals("sc-substatus")) { dump.Protocol_Status = string.Format("{0}.{1}", dump.Protocol_Status, rows[i]); } if (currentHeader[i].Equals("cs(Referer)")) { dump.Referer = rows[i]; dump.RefererClass = IndihiangHelper.GetRefererClass(rows[i]); } if (currentHeader[i].Equals("sc-bytes")) { if (!string.IsNullOrEmpty(rows[i])) { dump.Bytes_Sent = Convert.ToInt64(rows[i]); } else { dump.Bytes_Sent = 0; } } if (currentHeader[i].Equals("cs-bytes")) { if (!string.IsNullOrEmpty(rows[i])) { dump.Bytes_Received = Convert.ToInt64(rows[i]); } else { dump.Bytes_Received = 0; } } if (currentHeader[i].Equals("time-taken")) { if (!string.IsNullOrEmpty(rows[i])) { dump.TimeTaken = Convert.ToInt64(rows[i]); } else { dump.TimeTaken = 0; } } } catch (Exception err) { Console.WriteLine(err.Message); } } if (dump.Day > 0 && dump.Month > 0) { listDump.Add(dump); } } } } #endregion } line = sr.ReadLine(); if (!string.IsNullOrEmpty(line)) { line = line.Trim(); } } if (listDump.Count > 0) { _dumpLogQueue.Enqueue(listDump); } } } /*catch (IOException) * { * // * }*/ catch (Exception ex) { strError = ex.Message; } if (bRead == false) { goto retry; } }
private bool SequentialParse(bool success, List <string> listFiles) { try { //List<ManualResetEventSlim> resets = new List<ManualResetEventSlim>(); //_exitDump = new ManualResetEventSlim(false); DataHelper helper = new DataHelper(IndihiangHelper.GetIPCountryDb()); List <Indihiang.DomainObject.IPCountry> listIpCountry = helper.GetAllIpCountry(); Dictionary <string, string> cacheIpCountry = new Dictionary <string, string>(); cacheIpCountry.Add("127.0.0.1", "(Local)"); for (int i = 0; i < listFiles.Count; i++) { LogInfoEventArgs logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Run Parse on file {0}", listFiles[i])); _synContext.Post(OnParseLog, logInfo); try { RunParse(listFiles[i]); DumpPartial(listIpCountry, cacheIpCountry); logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Run Parse : {0} was done", listFiles[i])); _synContext.Post(OnParseLog, logInfo); } catch (Exception err) { logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Error occurred on file {0}==>{1}\r\n{2}", listFiles[i], err.Message, err.StackTrace)); _synContext.Post(OnParseLog, logInfo); } } _finish = true; //LogInfoEventArgs logInfo2 = new LogInfoEventArgs( // ParserID, // EnumLogFile.UNKNOWN, // LogProcessStatus.SUCCESS, // "Parse()", // "Consolidating log files..."); //_synContext.Post(OnParseLog, logInfo2); //try //{ // //DumpData(); // //if (!_exitDump.IsSet) // // _exitDump.Wait(); //} //catch { } LogInfoEventArgs logInfo2 = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", "Consolidated log file was done"); _synContext.Post(OnParseLog, logInfo2); Thread.Sleep(100); success = true; } catch (AggregateException err) { Logger.Write(err.Message); Logger.Write(err.StackTrace); #region Handle Exception LogInfoEventArgs logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Internal Error: {0}", err.Message)); _synContext.Post(OnParseLog, logInfo); logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Source Internal Error: {0}", err.Source)); _synContext.Post(OnParseLog, logInfo); logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Detail Internal Error: {0}", err.StackTrace)); _synContext.Post(OnParseLog, logInfo); logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Internal Exception Error: {0}", err.InnerException.Message)); _synContext.Post(OnParseLog, logInfo); #endregion } catch (Exception err) { Logger.Write(err.Message); Logger.Write(err.StackTrace); #region Handle Exception LogInfoEventArgs logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Internal Error: {0}", err.Message)); _synContext.Post(OnParseLog, logInfo); logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Source Internal Error: {0}", err.Source)); _synContext.Post(OnParseLog, logInfo); logInfo = new LogInfoEventArgs( ParserID, EnumLogFile.UNKNOWN, LogProcessStatus.SUCCESS, "Parse()", String.Format("Detail Internal Error: {0}", err.StackTrace)); _synContext.Post(OnParseLog, logInfo); #endregion } return(success); }