/// <summary> /// Write analyzed log to internal database /// </summary> /// <param name="fileName">Log file name</param> /// <param name="logStream">Stream of log file</param> /// <param name="enc">Log file encoding</param> public void Write(string fileName, Stream logStream, Encoding enc) { log.In(logStream); // Reset line count lineCount = 0; using (SQLiteTransaction trans = connection.BeginTransaction()) { StreamReader reader = new StreamReader(new BufferedStream(logStream), enc); FileValue file = InsertFileName(fileName); ParsedLogLine prevParsedLog = null; while (true) { string line = reader.ReadLine(); if (line == null) { break; } ParsedLogLine logLine = ParseLine(file, line, regex, columnDefList); if (logLine != null) { // If log line has error then set previous datetime value for sorting if (logLine.HasError) { ParsedLogColumn col = GetDateTimeColumn(prevParsedLog); if (col != null) { logLine.ColumnList.Add(col); } } else { prevParsedLog = logLine; } lineCount++; logLine.LineNumber = lineCount; InsertLogLine(logLine); } } trans.Commit(); } log.Out(); }
/// <summary> /// Read log lines from database /// </summary> /// <param name="criteria">Search condition</param> /// <param name="colDefList">Column definition list</param> /// <param name="highlightDefList">Highlight definition list</param> /// <param name="offset">Read start position</param> /// <param name="limit">Read log lines limit</param> /// <returns>Parsed log lines</returns> public ParsedLog ReadLines(SearchCriteria criteria, ColumnDefinitionList colDefList, HighlightDefinitionList highlightDefList, int offset, int limit) { log.In(criteria, offset, limit); SQLiteCommand comm = connection.CreateCommand(); SetSelectStatement(comm, colDefList, criteria, offset, limit); ParsedLog parsedLog = new ParsedLog(); List <ParsedLogLine> lineList = new List <ParsedLogLine>(); using (SQLiteDataReader reader = comm.ExecuteReader()) { while (reader.Read()) { ParsedLogLine logLine = new ParsedLogLine(); logLine.File = new FileValue(); logLine.File.Name = GetStringValue(reader, IntermediateLogConstants.COL_NAME_LF_FILE_NAME); logLine.LineNumber = GetIntValue(reader, IntermediateLogConstants.COL_NAME_PL_LINE_NUM); logLine.HasError = GetBooleanValue(reader, IntermediateLogConstants.COL_NAME_PL_HAS_ERROR); logLine.NotParsedLog = GetStringValue(reader, IntermediateLogConstants.COL_NAME_PL_NOT_PARSED_LINE); foreach (ColumnDefinition colDef in colDefList) { ParsedLogColumn col = new ParsedLogColumn(); col.ColumnDefinition = colDef; col.Value = GetStringValue(reader, colDef.ColumnName); if (highlightDefList != null) { SetHighlights(col, highlightDefList); } logLine.ColumnList.Add(col); } lineList.Add(logLine); } } parsedLog.TotalLineCount = GetTotalLineCount(); parsedLog.TargetLineCount = GetLineCount(criteria); parsedLog.LogLineList = lineList; log.Out(lineList); return(parsedLog); }
/// <summary> /// Insert Parsed log line to ParsedLog table /// </summary> /// <param name="logLine">Target log line</param> private void InsertLogLine(ParsedLogLine logLine) { log.InPrivate(logLine); using (SQLiteCommand comm = connection.CreateCommand()) { comm.CommandText = "insert into " + IntermediateLogConstants.TABLE_NAME_PARSED_LOGS + "(" + CreateColumnsString(logLine) + ")"; SetColumnValues(comm, logLine); comm.ExecuteNonQuery(); } log.OutPrivate(); }
/// <summary> /// Find and get DateTime column from parsed log line /// </summary> /// <param name="logLine">Parsed log line</param> /// <returns>DateTime log column</returns> private ParsedLogColumn GetDateTimeColumn(ParsedLogLine logLine) { if (logLine == null) { return(null); } foreach (ParsedLogColumn col in logLine.ColumnList) { if (col.ColumnDefinition.IsDateTimeField) { return(col); } } return(null); }
/// <summary> /// Add parameters for insert column values /// </summary> /// <param name="comm">Command</param> /// <param name="logLine">Target log line</param> private void SetColumnValues(SQLiteCommand comm, ParsedLogLine logLine) { log.InPrivate(comm, logLine); comm.CommandText += " values("; comm.CommandText += "@LogLine, "; comm.Parameters.Add("LogLine", System.Data.DbType.Int64); comm.Parameters["LogLine"].Value = logLine.LineNumber; comm.CommandText += "@FileID, "; comm.Parameters.Add("FileID", System.Data.DbType.Int64); comm.Parameters["FileID"].Value = logLine.File.ID; comm.CommandText += "@HasError, "; comm.Parameters.Add("HasError", System.Data.DbType.Int64); comm.Parameters["HasError"].Value = logLine.HasError; comm.CommandText += "@NotParsedLog, "; comm.Parameters.Add("NotParsedLog", System.Data.DbType.String); comm.Parameters["NotParsedLog"].Value = logLine.NotParsedLog; foreach (ParsedLogColumn col in logLine.ColumnList) { string colName = col.ColumnDefinition.ColumnName; comm.CommandText += "@" + colName + ", "; if (col.ColumnDefinition.IsDateTimeField || col.ColumnDefinition.Hashable) { comm.Parameters.Add(colName, System.Data.DbType.Int64); } else { comm.Parameters.Add(colName, System.Data.DbType.String); } comm.Parameters[colName].Value = col.Value; } comm.CommandText = comm.CommandText.Substring(0, comm.CommandText.Length - 2) + ")"; log.OutPrivate(); }
/// <summary> /// Create column name list string value for SELECT statement. /// </summary> /// <param name="logLine">Target log line</param> /// <returns>Column name list string</returns> private string CreateColumnsString(ParsedLogLine logLine) { log.InPrivate(logLine); string cols = IntermediateLogConstants.COL_NAME_PL_LINE_NUM + "," + IntermediateLogConstants.COL_NAME_PL_FILE_ID + "," + IntermediateLogConstants.COL_NAME_PL_HAS_ERROR + "," + IntermediateLogConstants.COL_NAME_PL_NOT_PARSED_LINE + ","; foreach (ParsedLogColumn col in logLine.ColumnList) { cols += col.ColumnDefinition.ColumnName + ","; } string val = cols.Substring(0, cols.Length - 1); log.OutPrivate(val); return(val); }
/// <summary> /// Parse log line /// </summary> /// <param name="file">Log file name</param> /// <param name="line">Target log line</param> /// <param name="regex">Analyzing regex</param> /// <param name="colDefList">Column definition list</param> /// <returns>Parsed log line</returns> private ParsedLogLine ParseLine(FileValue file, string line, Regex regex, ColumnDefinitionList colDefList) { log.InPrivate(line, regex, colDefList); ParsedLogLine logLine = new ParsedLogLine(); logLine.File = file; logLine.ColumnList = new List <ParsedLogColumn>(); string[] lineElems = regex.Split(line); lineElems = RemoveFirstAndLastEmptyElems(lineElems); if (lineElems.Length != colDefList.Count) { log.Warn("Log element size is different.Skip this line. Expected=" + colDefList.Count.ToString() + " Actual=" + lineElems.Length); logLine.HasError = true; logLine.NotParsedLog = line; log.OutPrivate(logLine); return(logLine); } int index = 0; foreach (string lineElem in lineElems) { ColumnDefinition colDef = colDefList[index]; ParsedLogColumn col = new ParsedLogColumn(); col.ColumnDefinition = colDef; if (colDef.IsDateTimeField) { if (SetDataTimeValue(lineElem, col, colDef) == false) { logLine.HasError = true; log.Warn("Could not parse DateTime value. Value=" + lineElem); } } else if (colDef.Hashable) { SetHashedValue(lineElem, col); } else { col.Value = lineElem; } logLine.ColumnList.Add(col); index++; } if (logLine.HasError || line.Trim() == "") { logLine.ColumnList.Clear(); logLine.HasError = true; logLine.NotParsedLog = line; } log.OutPrivate(logLine); return(logLine); }