private void Schematize(LogItemSchema schema, IEnumerable <LogField> items)
{
foreach (var item in items)
{
// is this a new item we haven't seen before?
if (!schema.HasChild(item.Name))
{
var typeName = item.Name;
if (item is LogEntry)
{
// recurrse down the structure
var s = new LogItemSchema()
{
Name = item.Name, Type = typeName
};
schema.AddChild(s);
Schematize(s, ((LogEntry)item).GetFields());
}
else
{
// leaf node
typeName = item.Value.GetType().Name;
var leaf = new LogItemSchema()
{
Name = item.Name, Type = typeName
};
schema.AddChild(leaf);
}
}
}
}
public IEnumerable<DataValue> GetDataValues(LogItemSchema schema, DateTime startTime, TimeSpan duration)
{
if (data != null)
{
bool allValues = (startTime == DateTime.MinValue && duration == TimeSpan.MaxValue);
DateTime endTime = allValues ? DateTime.MaxValue : startTime + duration;
string xname = XmlConvert.EncodeLocalName(schema.Name);
int defaultX = 0;
foreach (var e in data.Root.Elements(xname))
{
int? us = GetTimeMicroseconds(e);
int x = us.HasValue ? us.Value : defaultX++;
DateTime time = this.startTime.AddMilliseconds(x / 1000);
if (allValues || (us.HasValue && time > startTime && time < endTime))
{
string value = e.Value;
double d = 0;
if (double.TryParse(value, out d))
{
yield return new DataValue() { X = x, Y = d };
}
}
}
}
}
internal void Combine(LogItemSchema s)
{
Dictionary <string, LogItemSchema> index = new Dictionary <string, Model.LogItemSchema>();
if (this.HasChildren)
{
lock (this.childItems)
{
foreach (var i in this.childItems)
{
index[i.Name] = i;
}
}
}
if (s.HasChildren)
{
lock (s.childItems)
{
foreach (var child in s.childItems)
{
LogItemSchema found = null;
if (index.TryGetValue(child.Name, out found))
{
found.Combine(child);
}
else
{
LogItemSchema copy = child.Clone();
copy.Parent = this;
this.AddChild(child);
}
}
}
}
}
private void CreateSchema(PX4BinaryLog log)
{
LogItemSchema schema = new LogItemSchema()
{
Name = "Px4DataLog", Type = "Root"
};
foreach (var fmt in log.GetFormats())
{
LogItemSchema element = new LogItemSchema()
{
Name = fmt.Name
};
int i = 0;
foreach (var c in fmt.Columns)
{
LogItemSchema column = new LogItemSchema()
{
Name = c
};
if (i < fmt.FormatString.Length)
{
column.Type = GetTypeName(fmt.FormatString[i]);
}
i++;
element.AddChild(column);
}
schema.AddChild(element);
}
this.schema = schema;
}
private void Schematize(LogItemSchema schema, IEnumerable <LogField> items)
{
foreach (var item in items)
{
// is this a new item we haven't seen before?
if (!(from c in schema.ChildItems where c.Name == item.Name select c).Any())
{
var typeName = item.Name;
if (item is LogEntry)
{
// recurrse down the structure
var s = new LogItemSchema()
{
Name = item.Name, Type = typeName, ChildItems = new List <Model.LogItemSchema>(), Parent = schema
};
schema.ChildItems.Add(s);
Schematize(s, ((LogEntry)item).GetFields());
}
else
{
// leaf node
typeName = item.Value.GetType().Name;
var leaf = new LogItemSchema()
{
Name = item.Name, Type = typeName, Parent = schema
};
schema.ChildItems.Add(leaf);
}
}
}
}
internal MavlinkQueryEnumerator(MavlinkLog log, LogItemSchema schema, CancellationToken token)
{
this.log = log;
this.schema = schema;
this.token = token;
this.log.AddQuery(this);
}
public IEnumerable <DataValue> GetDataValues(LogItemSchema schema, DateTime startTime, TimeSpan duration)
{
if (data != null)
{
bool allValues = (startTime == DateTime.MinValue && duration == TimeSpan.MaxValue);
DateTime endTime = allValues ? DateTime.MaxValue : startTime + duration;
string xname = XmlConvert.EncodeLocalName(schema.Name);
int defaultX = 0;
foreach (var e in data.Root.Elements(xname))
{
int? us = GetTimeMicroseconds(e);
int x = us.HasValue ? us.Value : defaultX++;
DateTime time = this.startTime.AddMilliseconds(x / 1000);
if (allValues || (us.HasValue && time > startTime && time < endTime))
{
string value = e.Value;
double d = 0;
if (double.TryParse(value, out d))
{
yield return(new DataValue()
{
X = x, Y = d
});
}
}
}
}
}
private void CreateSchema(object message)
{
Type t = message.GetType();
if (schemaCache.ContainsKey(t))
{
return;
}
string name = t.Name;
if (name.StartsWith("mavlink_"))
{
name = name.Substring(8);
}
else if (name.StartsWith("mavlink"))
{
name = name.Substring(7);
}
LogItemSchema item = new LogItemSchema()
{
Name = name, Type = t.Name
};
foreach (FieldInfo fi in t.GetFields(BindingFlags.Public | BindingFlags.Instance))
{
var field = new LogItemSchema()
{
Name = fi.Name, Type = fi.FieldType.Name
};
item.AddChild(field);
object value = fi.GetValue(message);
// byte[] array is special, we return that as binhex encoded binary data.
if (fi.FieldType.IsArray && fi.FieldType != typeof(byte[]))
{
field.IsArray = true;
Type itemType = fi.FieldType.GetElementType();
Array a = (Array)value;
for (int i = 0, n = a.Length; i < n; i++)
{
field.AddChild(new LogItemSchema()
{
Name = i.ToString(), Type = itemType.Name
});
}
}
}
schemaCache[t] = item;
schema.AddChild(item);
if (SchemaChanged != null)
{
SchemaChanged(this, EventArgs.Empty);
}
}
internal void Clear()
{
this.data = new List <Message>();
schema = new LogItemSchema()
{
Name = "MavlinkLog", Type = "Root"
};
schemaCache = new Dictionary <Type, LogItemSchema>();
}
public IEnumerable <DataValue> GetDataValues(LogItemSchema schema, DateTime startTime, TimeSpan duration)
{
if (data != null && schema.Parent != null)
{
foreach (LogEntry entry in GetRows(schema.Parent.Name, startTime, duration))
{
yield return(entry.GetDataValue(schema.Name));
}
}
}
internal void AddChild(LogItemSchema item)
{
if (this.childItems == null)
{
this.childItems = new List <LogItemSchema>();
}
item.Parent = this;
lock (this.childItems)
{
this.childItems.Add(item);
}
}
private void CreateSchema(object message)
{
Type t = message.GetType();
if (schemaCache.ContainsKey(t))
{
return;
}
LogItemSchema item = new LogItemSchema()
{
Name = t.Name, Type = t.Name, ChildItems = new List <LogItemSchema>(), Parent = this.schema
};
foreach (FieldInfo fi in t.GetFields(BindingFlags.Public | BindingFlags.Instance))
{
var field = new LogItemSchema()
{
Name = fi.Name, Type = fi.FieldType.Name, Parent = item
};
item.ChildItems.Add(field);
object value = fi.GetValue(message);
// byte[] array is special, we return that as binhex encoded binary data.
if (fi.FieldType.IsArray && fi.FieldType != typeof(byte[]))
{
field.ChildItems = new List <Model.LogItemSchema>();
Type itemType = fi.FieldType.GetElementType();
Array a = (Array)value;
for (int i = 0, n = a.Length; i < n; i++)
{
field.ChildItems.Add(new LogItemSchema()
{
Name = i.ToString(), Type = itemType.Name, Parent = field
});
}
}
}
schemaCache[t] = item;
if (schema.ChildItems == null)
{
schema.ChildItems = new List <LogItemSchema>();
}
schema.ChildItems.Add(item);
if (SchemaChanged != null)
{
SchemaChanged(this, EventArgs.Empty);
}
}
public IEnumerable <DataValue> GetDataValues(LogItemSchema schema, DateTime startTime, TimeSpan duration)
{
if (schema != null)
{
foreach (var msg in GetMessages(startTime, duration))
{
DataValue value = GetDataValue(schema, msg);
if (value != null)
{
yield return(value);
}
}
}
}
private LogItemSchema Clone()
{
LogItemSchema copy = new LogItemSchema()
{
Id = this.Id, Name = this.Name, Type = this.Type
};
if (this.HasChildren)
{
lock (this.childItems)
{
foreach (var child in this.childItems)
{
var childClone = child.Clone();
copy.AddChild(childClone);
}
}
}
return(copy);
}
private LogItemSchema Clone()
{
LogItemSchema copy = new LogItemSchema()
{
Id = this.Id, Name = this.Name, Type = this.Type
};
if (this.HasChildren)
{
List <LogItemSchema> copyChildren = new List <Model.LogItemSchema>();
foreach (var child in this.ChildItems.ToArray())
{
var childClone = child.Clone();
childClone.Parent = copy;
copyChildren.Add(childClone);
}
copy.ChildItems = copyChildren;
}
return(copy);
}
private void CreateSchema(PX4BinaryLog log)
{
LogItemSchema schema = new LogItemSchema()
{
Name = "Px4DataLog", Type = "Root", ChildItems = new List <Model.LogItemSchema>()
};
foreach (var fmt in log.GetFormats())
{
LogItemSchema element = new LogItemSchema()
{
Name = fmt.Name, Parent = schema
};
int i = 0;
foreach (var c in fmt.Columns)
{
LogItemSchema column = new LogItemSchema()
{
Name = c, Parent = element
};
if (i < fmt.FormatString.Length)
{
column.Type = GetTypeName(fmt.FormatString[i]);
}
i++;
if (element.ChildItems == null)
{
element.ChildItems = new List <Model.LogItemSchema>();
}
element.ChildItems.Add(column);
}
schema.ChildItems.Add(element);
}
this.schema = schema;
}
public IEnumerable <DataValue> GetDataValues(LogItemSchema schema, DateTime startTime, TimeSpan duration)
{
if (schema != null)
{
bool first = true;
ulong startTicks = 0;
ulong previousTicks = 0;
foreach (var msg in GetMessages(startTime, duration))
{
if (first)
{
startTicks = previousTicks = msg.Ticks;
first = false;
}
previousTicks = msg.Ticks;
DataValue value = GetDataValue(schema, msg, startTicks);
if (value != null)
{
yield return(value);
}
}
}
}
private void GraphItem(LogItemSchema schema)
{
if (schema.IsNumeric)
{
ChartStack.Visibility = Visibility.Visible;
ChartStack.UpdateLayout();
SimpleLineChart chart = new SimpleLineChart();
chart.Margin = defaultChartMargin;
chart.Focusable = false;
chart.Closed += OnChartClosed;
chart.LineColor = HlsColor.GetRandomColor();
chart.StrokeThickness = 1;
chart.Tag = schema;
if (currentFlightLog != null && schema.Root == currentFlightLog.Schema)
{
List<DataValue> values = new List<DataValue>(currentFlightLog.GetDataValues(schema, DateTime.MinValue, TimeSpan.MaxValue));
InitializeChartData(schema, chart, values);
// now turn on live scrolling...
chart.LiveScrolling = true;
// now start watching the live update for new values that need to be added to this chart.
Task.Run(() =>
{
LiveUpdate(chart, currentFlightLog, schema);
});
}
else
{
List<DataValue> values = new List<DataValue>(GetSelectedDataValues(schema));
InitializeChartData(schema, chart, values);
}
if (chartGroup != null)
{
SimpleLineChart last = null;
if (chartGroup.Children.Count > 0)
{
last = chartGroup.Children[chartGroup.Children.Count - 1] as SimpleLineChart;
}
chartGroup.Children.Add(chart);
if (chartGroup.Parent == null)
{
ChartStack.AddChart(chartGroup);
}
if (last != null)
{
last.Next = chart;
}
}
else
{
ChartStack.AddChart(chart);
}
LayoutCharts();
Messages.Visibility = Visibility.Collapsed;
}
else
{
Paragraph p = new Paragraph();
foreach (var value in GetSelectedDataValues(schema))
{
p.Inlines.Add(new Run(value.Label));
p.Inlines.Add(new LineBreak());
}
var doc = Messages.Document;
doc.Blocks.Add(p);
Messages.Visibility = Visibility.Visible;
Messages.Focus();
}
}
internal MavlinkQuery(MavlinkLog log, LogItemSchema schema, CancellationToken token)
{
this.log = log;
this.schema = schema;
this.token = token;
}
public IEnumerable<DataValue> LiveQuery(LogItemSchema schema, CancellationToken token)
{
throw new NotImplementedException("LiveQuery");
}
private void UnselectCategory(LogItemSchema item)
{
if (CategoryList.SelectedItems.Contains(item))
{
CategoryList.SelectedItems.Remove(item);
}
else
{
// might be a child category item...
foreach (var childList in childLists)
{
if (childList.SelectedItems.Contains(item))
{
childList.SelectedItems.Remove(item);
}
}
}
}
public IEnumerable <DataValue> LiveQuery(LogItemSchema schema, CancellationToken token)
{
return(new MavlinkQuery(this, schema, token));
}
public async Task Load(string fileName, ProgressUtility progress)
{
flights.Clear();
// CSV doesn't have realtime clock, so go with the file date instead.
this.startTime = File.GetLastWriteTime(fileName);
// time (ms)
long min = long.MaxValue;
long max = long.MinValue;
await Task.Run(() =>
{
timeElementName = null;
using (Stream s = File.OpenRead(fileName))
{
Dictionary <string, LogItemSchema> map = new Dictionary <string, LogItemSchema>();
XmlNameTable nametable = new NameTable();
using (XmlCsvReader reader = new XmlCsvReader(s, System.Text.Encoding.UTF8, new Uri(fileName), nametable))
{
reader.FirstRowHasColumnNames = true;
reader.ColumnsAsAttributes = true;
while (reader.Read())
{
progress.ShowProgress(0, s.Length, s.Position);
if (this.schema == null)
{
// create the schema
this.schema = new LogItemSchema()
{
Name = "CsvLog", Type = "Root"
};
LogItemSchema row = null;
foreach (String name in reader.ColumnNames)
{
if (timeElementName == null && (name.ToLower().Contains("time") || name.ToLower().Contains("ticks")))
{
timeElementName = name;
}
if (name.Contains(":"))
{
// then we have sub-parts.
int pos = name.IndexOf(":");
string key = name.Substring(0, pos);
string field = name.Substring(pos + 1);
LogItemSchema group = null;
if (!map.ContainsKey(key))
{
group = new LogItemSchema()
{
Name = key, Type = key
};
this.schema.AddChild(group);
map[key] = group;
}
else
{
group = map[key];
}
var leaf = new LogItemSchema()
{
Name = field, Type = "Double"
};
group.AddChild(leaf);
map[name] = leaf;
}
else
{
if (row == null)
{
row = new LogItemSchema()
{
Name = "Other", Type = "Other"
};
this.schema.AddChild(row);
}
var leaf = new LogItemSchema()
{
Name = name, Type = "Double"
};
row.AddChild(leaf);
map[name] = leaf;
}
}
}
if (reader.NodeType == XmlNodeType.Element && reader.LocalName == "row")
{
// read a row
long time = GetTicks(reader);
min = Math.Min(min, time);
max = Math.Max(max, time);
LogEntry row = new Model.LogEntry()
{
Name = "Other", Timestamp = (ulong)time
};
log.Add(row);
Dictionary <string, LogEntry> groups = new Dictionary <string, LogEntry>();
if (reader.MoveToFirstAttribute())
{
do
{
string name = XmlConvert.DecodeName(reader.LocalName);
LogItemSchema itemSchema = map[name];
LogEntry e = row;
if (name.Contains(":"))
{
// then we have sub-parts.
int pos = name.IndexOf(":");
string key = name.Substring(0, pos);
string field = name.Substring(pos + 1);
if (!groups.ContainsKey(key))
{
e = new LogEntry()
{
Name = key, Timestamp = (ulong)time
};
groups[key] = e;
log.Add(e);
}
else
{
e = groups[key];
}
name = field;
}
string value = reader.Value;
double d = 0;
if (double.TryParse(value, out d))
{
e.SetField(name, d);
}
else
{
if (!string.IsNullOrEmpty(value))
{
// not a number.
itemSchema.Type = "String";
e.SetField(name, value);
}
}
}while (reader.MoveToNextAttribute());
reader.MoveToElement();
}
}
}
}
}
});
// this log has no absolute UTC time, only ticks since board was booted, so we make up a start time.
DateTime end = this.startTime.AddMilliseconds((max - min) / 1000);
var flight = new Flight()
{
Log = this, StartTime = this.startTime, Duration = end - this.startTime
};
this.duration = end - this.startTime;
this.flights.Add(flight);
}
public IEnumerable <DataValue> GetDataValues(LogItemSchema schema, DateTime startTime, TimeSpan duration)
{
throw new NotImplementedException();
}
internal DataValue GetDataValue(LogItemSchema schema, Message msg)
{
// we support 3 levels of nesting, so schema could be the row, the column or an array item.
LogItemSchema rowSchema = schema;
LogItemSchema columnSchema = schema;
LogItemSchema arraySchema = schema;
if (rowSchema.Parent != this.schema)
{
rowSchema = rowSchema.Parent;
}
if (rowSchema.Parent != this.schema)
{
columnSchema = rowSchema;
rowSchema = rowSchema.Parent;
}
StringBuilder textBuilder = new StringBuilder();
var row = msg.TypedValue;
if (row != null && row.GetType().Name == rowSchema.Type)
{
// get a time value for this message.
FieldInfo fi = row.GetType().GetField(columnSchema.Name, BindingFlags.Public | BindingFlags.Instance);
if (fi != null)
{
object value = fi.GetValue(row);
DataValue data = new DataValue()
{
X = msg.Timestamp.Ticks / 10, UserData = msg
}; // microseconds (Ticks are in 100 nanoseconds).
// byte array is special (we treat this like text).
if (value is byte[])
{
textBuilder.Length = 0;
byte[] text = (byte[])value;
bool binary = false;
// see if this is binary or text.
for (int i = 0, n = text.Length; i < n; i++)
{
byte b = text[i];
if (b != 0 && b < 0x20 || b > 0x80)
{
binary = true;
}
}
for (int i = 0, n = text.Length; i < n; i++)
{
byte b = text[i];
if (b != 0)
{
if (binary)
{
textBuilder.Append(b.ToString("x2"));
textBuilder.Append(" ");
}
else
{
char ch = Convert.ToChar(b);
textBuilder.Append(ch);
}
}
}
data.Label = textBuilder.ToString();
}
else if (value is string)
{
data.Label = (string)value;
}
else
{
if (fi.FieldType.IsArray)
{
// then we are expecting an array selector as well...
Array a = (Array)value;
int index = 0;
int.TryParse(arraySchema.Name, out index);
if (a.Length > index)
{
value = a.GetValue(index);
}
}
data.Y = ConvertToNumeric(value);
data.Label = GetLabel(row, columnSchema.Name);
};
return(data);
}
}
return(null);
}
IEnumerable<DataValue> GetSelectedDataValues(LogItemSchema schema)
{
List<DataValue> combined = new List<DataValue>();
List<Flight> selected = GetSelectedFlights();
if (selected.Count == 0)
{
// show everything.
selected.Add(new Flight() { StartTime = DateTime.MinValue, Duration = TimeSpan.MaxValue });
}
foreach (IDataLog log in this.logs)
{
if (log != null)
{
foreach (var flight in selected)
{
if (flight.Log == null || flight.Log == log)
{
combined.AddRange(log.GetDataValues(schema, flight.StartTime, flight.Duration));
}
}
}
}
return combined;
}
public async Task Load(string fileName, ProgressUtility progress)
{
flights.Clear();
// CSV doesn't have realtime clock, so go with the file date instead.
this.startTime = File.GetLastWriteTime(fileName);
// time (us)
int min = int.MaxValue;
int max = int.MinValue;
await Task.Run(() =>
{
using (Stream s = File.OpenRead(fileName))
{
XmlNameTable nametable = new NameTable();
using (XmlCsvReader reader = new XmlCsvReader(s, System.Text.Encoding.UTF8, new Uri(fileName), nametable))
{
progress.ShowProgress(0, s.Length, s.Position);
reader.FirstRowHasColumnNames = true;
data = XDocument.Load(reader);
this.schema = new LogItemSchema()
{
Name = "CsvDataLog", Type = "Root"
};
// create the schema
List <LogItemSchema> children = new List <Model.LogItemSchema>();
foreach (String name in reader.ColumnNames)
{
children.Add(new LogItemSchema()
{
Name = name, Parent = this.schema
});
}
this.schema.ChildItems = children;
progress.ShowProgress(0, s.Length, s.Position);
}
}
foreach (var e in data.Root.Elements())
{
int?i = GetTimeMicroseconds(e);
if (i.HasValue)
{
if (i.Value < min)
{
min = i.Value;
}
if (i > max)
{
max = i.Value;
}
}
}
});
// this log has no absolute UTC time, only ticks since board was booted, so we make up a start time.
DateTime end = this.startTime.AddMilliseconds((max - min) / 1000);
var flight = new Flight()
{
Log = this, StartTime = this.startTime, Duration = end - this.startTime
};
this.duration = end - this.startTime;
this.flights.Add(flight);
}
// bugbug: should concatenate data from selected logs - then sort by X...
// bugbug: give them unique X values that are smooth over time. Alternatively we could
// give the entry Timestamp, but that seems to bunch up the values, so timeclock is not
// updating smoothly for some reason...
private void InitializeChartData(LogItemSchema schema, SimpleLineChart chart, List<DataValue> values)
{
int x = 0;
foreach (var d in values)
{
d.X = x++;
}
chart.SetData(new Model.DataSeries()
{
Name = schema.Name,
Values = values
});
}
private void LiveUpdate(SimpleLineChart chart, MavlinkLog currentFlightLog, LogItemSchema schema)
{
// this method is running on a background task and it's job is to read an infinite stream of
// data values from the log and show them in the live scrolling chart.
CancellationTokenSource canceller = new CancellationTokenSource();
chart.Closed += (s,e) =>
{
canceller.Cancel();
};
var query = currentFlightLog.LiveQuery(schema, canceller.Token);
foreach (DataValue item in query)
{
if (item == null)
{
return;
}
chart.SetCurrentValue(item);
}
}
public IEnumerable <DataValue> LiveQuery(LogItemSchema schema, CancellationToken token)
{
throw new NotImplementedException();
}
public async Task Load(string fileName, ProgressUtility progress)
{
flights.Clear();
// CSV doesn't have realtime clock, so go with the file date instead.
this.startTime = File.GetLastWriteTime(fileName);
// time (us)
int min = int.MaxValue;
int max = int.MinValue;
await Task.Run(() =>
{
using (Stream s = File.OpenRead(fileName))
{
XmlNameTable nametable = new NameTable();
using (XmlCsvReader reader = new XmlCsvReader(s, System.Text.Encoding.UTF8, new Uri(fileName), nametable))
{
progress.ShowProgress(0, s.Length, s.Position);
reader.FirstRowHasColumnNames = true;
data = XDocument.Load(reader);
this.schema = new LogItemSchema() { Name = "CsvDataLog", Type = "Root" };
// create the schema
List<LogItemSchema> children = new List<Model.LogItemSchema>();
foreach (String name in reader.ColumnNames)
{
children.Add(new LogItemSchema() { Name = name, Parent = this.schema });
}
this.schema.ChildItems = children;
progress.ShowProgress(0, s.Length, s.Position);
}
}
foreach (var e in data.Root.Elements())
{
int? i = GetTimeMicroseconds(e);
if (i.HasValue)
{
if (i.Value < min)
{
min = i.Value;
}
if (i > max)
{
max = i.Value;
}
}
}
});
// this log has no absolute UTC time, only ticks since board was booted, so we make up a start time.
DateTime end = this.startTime.AddMilliseconds((max - min) / 1000);
var flight = new Flight() { Log = this, StartTime = this.startTime, Duration = end - this.startTime };
this.duration = end - this.startTime;
this.flights.Add(flight);
}