protected override void on_new_lines(string new_lines) {
int line_count = 0;
last_lines_string_.set_lines(new_lines, ref line_count);
if (line_count < 1)
return;
int start_idx = 0;
if (has_header_line_)
lock (this)
// if at least one entry - can't read column names
if (column_names_.Count < 1 && entries_.Count == 0) {
column_names_ = split.to_list(last_lines_string_.line_at(0), separator_);
start_idx = 1;
}
List<log_entry_line> entries_now = new List<log_entry_line>();
var column_names = this.column_names;
for (int i = start_idx; i < line_count; ++i) {
var list = split.to_list(last_lines_string_.line_at(i), separator_);
log_entry_line entry = new log_entry_line();
for ( int j = 0; j < column_names.Count; ++j)
entry.add( column_names[j], list.Count > j ? list[j] : "");
entries_now.Add(entry);
}
lock (this) {
foreach ( var entry in entries_now)
string_.add_preparsed_line(entry.ToString());
entries_.AddRange(entries_now);
}
}
protected override void on_new_lines(string new_lines)
{
int line_count = 0;
last_lines_string_.set_lines(new_lines, ref line_count);
if (line_count < 1)
{
return;
}
int start_idx = 0;
if (has_header_line_)
{
lock (this)
// if at least one entry - can't read column names
if (this.column_names.Count < 1 && entries_.Count == 0)
{
this.column_names = parse_csv(last_lines_string_.line_at(0));
start_idx = 1;
}
}
List <log_entry_line> entries_now = new List <log_entry_line>();
var column_names = this.column_names;
string before = "";
for (int i = start_idx; i < line_count; ++i)
{
var cur_line = last_lines_string_.line_at(i);
var list = parse_csv(before + cur_line);
if (list.Count < column_names.Count)
{
before += cur_line + "\r\n";
continue;
}
before = "";
if (list.Count > column_names.Count)
{
logger.Warn("invalid csv line, too many cells: " + list.Count + " , instead of " + column_names.Count);
}
log_entry_line entry = new log_entry_line();
for (int j = 0; j < column_names.Count; ++j)
{
entry.add(column_names[j], list[j]);
}
entries_now.Add(entry);
}
lock (this) {
foreach (var entry in entries_now)
{
string_.add_preparsed_line(entry.ToString());
}
entries_.AddRange(entries_now);
}
}
protected override void on_new_lines(string new_lines)
{
int line_count = 0;
last_lines_string_.set_lines(new_lines, ref line_count);
if (line_count < 1)
{
return;
}
int start_idx = 0;
if (has_header_line_)
{
lock (this)
// if at least one entry - can't read column names
if (this.column_names.Count < 1 && entries_.Count == 0)
{
this.column_names = split.to_list(last_lines_string_.line_at(0), separator_);
start_idx = 1;
}
}
List <log_entry_line> entries_now = new List <log_entry_line>();
var column_names = this.column_names;
for (int i = start_idx; i < line_count; ++i)
{
var list = split.to_list(last_lines_string_.line_at(i), separator_);
log_entry_line entry = new log_entry_line();
for (int j = 0; j < column_names.Count; ++j)
{
entry.add(column_names[j], list.Count > j ? list[j] : "");
}
entries_now.Add(entry);
}
lock (this) {
foreach (var entry in entries_now)
{
string_.add_preparsed_line(entry.ToString());
}
entries_.AddRange(entries_now);
}
}
protected override void on_new_lines(string new_lines)
{
foreach (var c in new_lines.ToCharArray())
{
sb_.Append(c);
if (c == '{')
{
open_count_++;
}
else if (c == '}')
{
open_count_--;
if (open_count_ == 0)
{
// Full object in buffer
var obj = JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(sb_.ToString());
var line = new log_entry_line();
foreach (var entry in obj)
{
var value = entry.Value.ToString();
if (entry.Value.GetType() == typeof(DateTime))
{
value = ((DateTime)entry.Value).ToString("o");
}
line.analyze_and_add(entry.Key, value);
}
lock (this) {
entries_.Add(line);
string_.add_preparsed_line(line.ToString());
}
sb_.Clear();
}
}
}
}
protected override void on_new_lines(string next) {
string now = "";
string delimeter;
bool needs_set_column_names;
lock (this) {
last_ += next;
if (last_.Length < MIN_LEN && delimeter_name_ == "")
return;
last_ = last_.TrimStart();
if (delimeter_name_ == "") {
if (last_.StartsWith("<?xml ")) {
// we need to ignore xml prefix when searching for delimeter
int ignore = last_.IndexOf(">");
last_ = last_.Substring(ignore + 1).TrimStart();
}
int delimeter_idx = last_.IndexOfAny( new []{'>',' ', '\n', '\r', '\t'});
delimeter_name_ = last_.Substring(1, delimeter_idx - 1);
logger.Debug("[parse] parsing xml by " + delimeter_name_);
}
string end = "/" + delimeter_name_;
int last_idx = last_.LastIndexOf(end);
if (last_idx >= 0) {
// we can fully parse at least one entry
int xml_end = last_.IndexOf('>', last_idx);
if (xml_end > 0) {
now = last_.Substring(0, xml_end + 1);
last_ = last_.Substring(xml_end + 1);
}
}
if ( now == "")
// there's not enought text to parse a single log entry
return;
delimeter = delimeter_name_;
needs_set_column_names = column_names.Count < 1;
}
XmlTextReader reader = new XmlTextReader(now, XmlNodeType.Element, xml_parse_context_) { Namespaces = false };
// FIXME read all attributes , and save them as name.attr_name ; if name contains "xxx:", ignore that
// timestamp -> date + time
log_entry_line entry = new log_entry_line();
string last_element = "";
List<string> column_names_now = null;
try {
while (reader.Read()) {
if (reader.NodeType == XmlNodeType.Element) {
string element_name = reader.Name;
last_element = simple_element_name(element_name);
// read all its attributes
for (int i = 0; i < reader.AttributeCount; ++i) {
reader.MoveToAttribute(i);
string name = last_element + "." + reader.Name;
string text = (reader.Value ?? "").Trim();
if (ignore_delimeter_name_on_log_entry_)
if (element_name == delimeter)
name = reader.Name;
entry.analyze_and_add(name, text);
}
} else if (reader.NodeType == XmlNodeType.Text) {
Debug.Assert(last_element != "");
string text = (reader.Value ?? "").Trim();
entry.analyze_and_add(last_element, text);
} else if (reader.NodeType == XmlNodeType.EndElement) {
if (reader.Name == delimeter) {
// we read a full object
if (needs_set_column_names && column_names_now == null)
column_names_now = entry.names;
lock (this) {
entries_.Add(entry);
string_.add_preparsed_line(entry.ToString());
}
entry = new log_entry_line();
last_element = "";
}
}
}
if ( column_names_now != null)
column_names = column_names_now;
} catch (Exception e) {
logger.Fatal("[parse] could not parse xml: " + e);
}
}
protected override void on_new_lines(string new_lines) {
int line_count = 0;
last_lines_string_.set_lines(new_lines, ref line_count);
if (line_count < 1)
return;
int start_idx = 0;
if (has_header_line_)
lock (this)
// if at least one entry - can't read column names
if (this.column_names.Count < 1 && entries_.Count == 0) {
this.column_names = parse_csv( last_lines_string_.line_at(0));
start_idx = 1;
}
List<log_entry_line> entries_now = new List<log_entry_line>();
var column_names = this.column_names;
string before = "";
for (int i = start_idx; i < line_count; ++i) {
var cur_line = last_lines_string_.line_at(i);
var list = parse_csv(before + cur_line);
if (list.Count < column_names.Count) {
before += cur_line + "\r\n";
continue;
}
before = "";
if ( list.Count > column_names.Count)
logger.Warn("invalid csv line, too many cells: " + list.Count + " , instead of " + column_names.Count);
log_entry_line entry = new log_entry_line();
for ( int j = 0; j < column_names.Count; ++j)
entry.add( column_names[j], list[j]);
entries_now.Add(entry);
}
lock (this) {
foreach ( var entry in entries_now)
string_.add_preparsed_line(entry.ToString());
entries_.AddRange(entries_now);
}
}
protected override void on_new_lines(string new_lines)
{
int line_count = 0;
last_lines_string_.set_lines(new_lines, ref line_count);
if (line_count < 1)
{
return;
}
int start_idx = 0;
if (has_header_line_)
{
lock (this)
// if at least one entry - can't read column names
if (this.column_names.Count < 1 && entries_.Count == 0)
{
start_idx = try_parse_header(parse_csv(last_lines_string_.line_at(0))) ? 1 : 0;
}
}
List <log_entry_line> entries_now = new List <log_entry_line>();
var column_names = this.column_names;
string before = before_unprocessed_;
for (int i = start_idx; i < line_count; ++i)
{
var cur_line = last_lines_string_.line_at(i);
var list = parse_csv(before + cur_line);
if (list.Count < column_names.Count)
{
before += cur_line + "\r\n";
continue;
}
if (list.Count > column_names.Count)
{
if (list.Count == column_names.Count + 1 && !has_appended_message_column_)
{
has_appended_message_column_ = true;
var new_column_names = column_names.ToList();
new_column_names.Add("msg");
column_names = this.column_names = new_column_names;
}
}
if (list.Count > column_names.Count)
{
logger.Warn("invalid csv line" + (i + line_offset_) + " too many cells: " + list.Count + " , instead of " + column_names.Count);
reader.add_error("Bad CSV Line at " + (i + line_offset_ + 1) + ". Expected " + column_names.Count + " cells, got " + list.Count, error_list_keeper.level_type.warning);
}
log_entry_line entry = new log_entry_line();
for (int j = 0; j < column_names.Count; ++j)
{
entry.analyze_and_add(column_names[j], list[j]);
}
entries_now.Add(entry);
before = "";
}
line_offset_ += line_count;
before_unprocessed_ = before;
lock (this) {
foreach (var entry in entries_now)
{
string_.add_preparsed_line(entry.ToString());
}
entries_.AddRange(entries_now);
}
}
protected override void on_new_lines(string new_lines) {
int line_count = 0;
last_lines_string_.set_lines(new_lines, ref line_count);
if (line_count < 1)
return;
int start_idx = 0;
if (has_header_line_)
lock (this)
// if at least one entry - can't read column names
if (this.column_names.Count < 1 && entries_.Count == 0)
start_idx = try_parse_header( parse_csv( last_lines_string_.line_at(0))) ? 1 : 0;
List<log_entry_line> entries_now = new List<log_entry_line>();
var column_names = this.column_names;
string before = before_unprocessed_;
for (int i = start_idx; i < line_count; ++i) {
var cur_line = last_lines_string_.line_at(i);
var list = parse_csv(before + cur_line);
if (list.Count < column_names.Count) {
before += cur_line + "\r\n";
continue;
}
if ( list.Count > column_names.Count)
if (list.Count == column_names.Count + 1 && !has_appended_message_column_) {
has_appended_message_column_ = true;
var new_column_names = column_names.ToList();
new_column_names.Add("msg");
column_names = this.column_names = new_column_names;
}
if (list.Count > column_names.Count) {
logger.Warn("invalid csv line" + (i+line_offset_) + " too many cells: " + list.Count + " , instead of " + column_names.Count);
reader.add_error("Bad CSV Line at " + (i+line_offset_+1) + ". Expected " + column_names.Count + " cells, got " + list.Count, error_list_keeper.level_type.warning);
}
log_entry_line entry = new log_entry_line();
for ( int j = 0; j < column_names.Count; ++j)
entry.analyze_and_add( column_names[j], list[j]);
entries_now.Add(entry);
before = "";
}
line_offset_ += line_count;
before_unprocessed_ = before;
lock (this) {
foreach ( var entry in entries_now)
string_.add_preparsed_line(entry.ToString());
entries_.AddRange(entries_now);
}
}
protected override void on_new_lines(string next)
{
string now = "";
string delimeter;
bool needs_set_column_names;
lock (this) {
last_ += next;
if (last_.Length < MIN_LEN && delimeter_name_ == "")
{
return;
}
last_ = last_.TrimStart();
if (delimeter_name_ == "")
{
if (last_.StartsWith("<?xml "))
{
// we need to ignore xml prefix when searching for delimeter
int ignore = last_.IndexOf(">");
last_ = last_.Substring(ignore + 1).TrimStart();
}
int delimeter_idx = last_.IndexOfAny(new [] { '>', ' ', '\n', '\r', '\t' });
delimeter_name_ = last_.Substring(1, delimeter_idx - 1);
logger.Debug("[parse] parsing xml by " + delimeter_name_);
}
string end = "/" + delimeter_name_;
int last_idx = last_.LastIndexOf(end);
if (last_idx >= 0)
{
// we can fully parse at least one entry
int xml_end = last_.IndexOf('>', last_idx);
if (xml_end > 0)
{
now = last_.Substring(0, xml_end + 1);
last_ = last_.Substring(xml_end + 1);
}
}
if (now == "")
{
// there's not enought text to parse a single log entry
return;
}
delimeter = delimeter_name_;
needs_set_column_names = column_names.Count < 1;
}
XmlTextReader reader = new XmlTextReader(now, XmlNodeType.Element, xml_parse_context_)
{
Namespaces = false
};
// FIXME read all attributes , and save them as name.attr_name ; if name contains "xxx:", ignore that
// timestamp -> date + time
log_entry_line entry = new log_entry_line();
string last_element = "";
List <string> column_names_now = null;
try {
while (reader.Read())
{
if (reader.NodeType == XmlNodeType.Element)
{
string element_name = reader.Name;
last_element = simple_element_name(element_name);
// read all its attributes
for (int i = 0; i < reader.AttributeCount; ++i)
{
reader.MoveToAttribute(i);
string name = last_element + "." + reader.Name;
string text = (reader.Value ?? "").Trim();
if (ignore_delimeter_name_on_log_entry_)
{
if (element_name == delimeter)
{
name = reader.Name;
}
}
entry.analyze_and_add(name, text);
}
}
else if (reader.NodeType == XmlNodeType.Text)
{
Debug.Assert(last_element != "");
string text = (reader.Value ?? "").Trim();
entry.analyze_and_add(last_element, text);
}
else if (reader.NodeType == XmlNodeType.EndElement)
{
if (reader.Name == delimeter)
{
// we read a full object
if (needs_set_column_names && column_names_now == null)
{
column_names_now = entry.names;
}
lock (this) {
entries_.Add(entry);
string_.add_preparsed_line(entry.ToString());
}
entry = new log_entry_line();
last_element = "";
}
}
}
if (column_names_now != null)
{
column_names = column_names_now;
}
} catch (Exception e) {
logger.Fatal("[parse] could not parse xml: " + e);
}
}
