public override Expression Evaluate(EvalStore store)
{
Table data = store.Get<Table>("$DATA");
if (data == null || data.Rows == 0) return ConstExpr.Empty;
int limit;
int offset=0;
Expression e = _expr.Evaluate(store);
if (e.ExpressionType != Language.ExpressionType.CONST) throw new Exception("LIMIT number must be evaluated to an integer.");
Element l = (e as ConstExpr).Value;
if (l.DataType != ElementType.LONG) throw new Exception("LIMIT number must be evaluated to an integer.");
limit = l.ToInt32(null);
if (limit < 1) throw new Exception("LIMIT must be an positive integer greater than 1.");
Element o = null;
if (this.Offset != null) {
e = _expr.Evaluate(store);
if (e.ExpressionType != Language.ExpressionType.CONST) throw new Exception("OFFSET number must be evaluated to an integer.");
o = (e as ConstExpr).Value;
if (o.DataType != ElementType.LONG) throw new Exception("OFFSET number must be evaluated to an integer.");
offset = o.ToInt32(null);
if (offset < 0) throw new Exception("OFFSET must be an positive integer.");
}
int end = offset + limit;
if (end > data.Rows) end = data.Rows;
List<Element[]> rows = new List<Element[]>();
for (int i = offset; i < end; i++) {
rows.Add(data.Row(i));
}
Table r = new Table(data.Name, data.ColumnDefinitions, rows);
store.Put<Table>("$DATA", r);
return new ConstExpr(r);
}
public TableIndexerSort(Table src, int col)
{
_entries = new TableIndexEntry[src.Rows];
for (int i = 0; i < src.Rows; i++) {
_entries[i] = new TableIndexEntry(src[i, col], i);
}
Array.Sort<TableIndexEntry>(_entries);
}
public TableIndexerHash(Table src, int col)
{
_entries = new Dictionary<string, List<int>>();
for (int i = 0; i < src.Rows; i++) {
string key = src[i, col].ToString();
this.Add(key, i);
}
}
public TableIndexerHash(Table src, int[] col)
{
_entries = new Dictionary<string, List<int>>();
string[] keys = new string[col.Length];
for (int i = 0; i < src.Rows; i++) {
for (int j = 0; j < col.Length; j++) {
keys[j] = (string)src[i, col[j]];
}
string key = string.Join(",", keys);
this.Add(key, i);
}
}
public TableIndexerSort(Table src, int[] col)
{
_entries = new TableIndexEntry[src.Rows];
string[] key = new string[col.Length];
for (int i = 0; i < src.Rows; i++) {
for (int j = 0; j < col.Length; j++) {
key[j] = (string)src[i, col[j]];
}
_entries[i] = new TableIndexEntry(new Element(string.Join(",", key)), i);
}
Array.Sort<TableIndexEntry>(_entries);
}
public string CreateResourceFilters(Table sel)
{
HashSet<string> resIds = new HashSet<string>();
int resCol = sel.ColumnDefinitions.IndexOf("ResID");
for (int i = 0; i < sel.Rows; i++) {
resIds.Add((string)sel[i, resCol]);
}
if (resIds.Count == 0) return null;
string[] ids = new string[resIds.Count];
resIds.CopyTo(ids);
if (ids.Length == 1) return "ResID=" + ids[0];
else return "ResID in (" + string.Join(",", ids) + ")";
}
public override Expression Evaluate(EvalStore store)
{
var e = _expr.To<IdentifierExpr>();
object o = store.Get<object>(e.Name);
if (o is Table) {
return new ConstExpr(o as Table);
}
else if (o is Element) {
Element v = o as Element;
List<TableColumn> cols = new List<TableColumn>();
cols.Add(new TableColumn(null,v.DataType, false));
List<Element[]> row = new List<Element[]>();
row.Add(new Element[1]);
row[0][0]=v;
Table t = new Table(null, cols, row);
return new ConstExpr(t);
}
return ConstExpr.Empty;
}
private Dictionary<string, Table> SelectFactTableRows(Dictionary<string, Table> fromList, Table facts)
{
Dictionary<string, Table> selFacts = new Dictionary<string, Table>();
foreach (var item in fromList) {
Table s = item.Value;
for (int i = 0; i < s.Rows; i++) {
var attrId = s[i, "AttrID"];
var f = facts.Select(new Predicate("AttrID", Predicate.PredicateOperator.EQ, attrId));
if (!selFacts.ContainsKey(item.Key)) {
f.Name = item.Key;
selFacts.Add(item.Key, f);
}
else {
selFacts[item.Key].UnionWith(f);
}
}
}
return selFacts;
}
private void ExecuteQuery()
{
var lines = queryCode.Lines;
string query = string.Join("\n", lines);
Expression s = null;
try {
Parser p = new Parser();
s = p.Parse(query);
}
catch (Exception ex) {
Trace.WriteLine("Invalid syntax: " + ex.Message);
DialogResult result = MessageBox.Show(ex.Message, "Error parsing query", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
Expression r = null;
try {
toolStripButtonRun.Enabled = false;
Stopwatch watch = Stopwatch.StartNew();
_store.Enter();
r = s.Evaluate(_store);
watch.Stop();
Trace.WriteLine("QueryExecutionTime: " + watch.ElapsedMilliseconds + "ms");
if (r.Value.DataType == ElementType.TABLE) {
_data = (Table)r.Value;
Trace.WriteLine("Number of Rows: " + _data.Rows);
PopulateListView(_data);
toolStripStatusLabel1.Text = "Elapsed time: " + watch.ElapsedMilliseconds + "ms, Rows: " + _data.Rows;
}
else {
toolStripStatusLabel1.Text = "Elapsed time: " + watch.ElapsedMilliseconds + "ms";
}
}
catch (Exception ex) {
Trace.WriteLine("Error evaluating statement: " + ex.Message);
DialogResult result = MessageBox.Show(ex.Message, "Error executing query", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally {
_store.Exit();
toolStripButtonRun.Enabled = true;
}
}
private void PopulateListView(Table data)
{
Stopwatch watch = Stopwatch.StartNew();
resultList.BeginUpdate();
resultList.Items.Clear();
resultList.Columns.Clear();
resultList.View = View.Details;
resultList.LabelEdit = false;
resultList.FullRowSelect = true;
resultList.GridLines = true;
resultList.Sorting = System.Windows.Forms.SortOrder.None;
resultList.AutoResizeColumns(ColumnHeaderAutoResizeStyle.ColumnContent);
resultList.Columns.Add("", -2, HorizontalAlignment.Right);
for (int i = 0; i < data.Cols; i++) {
switch (data.ColumnDefinitions[i].DataType) {
case ElementType.DOUBLE:
case ElementType.LONG:
resultList.Columns.Add(data.ColumnDefinitions[i].Name, -2, HorizontalAlignment.Right);
break;
default:
resultList.Columns.Add(data.ColumnDefinitions[i].Name, -2, HorizontalAlignment.Left);
break;
}
}
_items = new ListViewItem[data.Rows];
for (int i = 0; i < data.Rows; i++) {
_items[i] = new ListViewItem((i + 1).ToString());
for (int j = 0; j < data.Cols; j++) {
_items[i].SubItems.Add(FormatValue(data[i, j]));
}
}
resultList.Items.AddRange(_items);
resultList.AutoResizeColumns(ColumnHeaderAutoResizeStyle.ColumnContent);
AutoResizeColumns(resultList);
resultList.EndUpdate();
watch.Stop();
Trace.WriteLine("PopulateListView: " + watch.ElapsedMilliseconds + "ms");
}
private Table CreateMasterDataTable(Table data, Table facts, HashSet<string> selIds)
{
if (data == null) return new Table();
if (data.Rows == 0) return data;
HashSet<int> factCols = new HashSet<int>();
foreach (string id in selIds) {
int col = data.ColumnDefinitions.IndexOf(id);
if (col < 0) {
col = facts.ColumnDefinitions.IndexOf(id);
if (col < 0) {
throw new Exception("No such column: " + id);
}
factCols.Add(col);
}
}
if (factCols.Count == 0) return data;
int dataCols = data.Cols;
foreach (int c in factCols) {
data.ColumnDefinitions.Add(facts.ColumnDefinitions[c]);
}
TableIndex idx = facts.GetIndex(new string[3] { "LogTypeID", "ResID", "AttrID" });
for (int i = 0; i < data.Rows; i++) {
Element[] row = data.Row(i);
string key = data[i, "LogTypeID"].ToString() + "," + data[i, "ResID"].ToString() + "," + data[i, "AttrID"].ToString();
int m = idx.FindEqual(new Element(key))[0];
Element[] r = facts.Row(m);
int j = 0;
foreach (int c in factCols) {
row[dataCols + j] = r[c];
j++;
}
}
return data;
}
private string TableToJSON(Table t)
{
if (t == null) return "[]";
if (t.Rows == 0) return "[]";
List<string> rows = new List<string>();
for (int i = 0; i < t.Rows; i++) {
List<string> col = new List<string>();
for (int j = 0; j < t.Cols; j++) {
string name = Util.ToJSONString(t.ColumnDefinitions[j].Name);
string value;
Element e = t[i, j];
switch (e.DataType) {
case ElementType.DATETIME:
value = Util.DateTimeToUnixTimestamp(e.DateTimeValue()).ToString();
break;
case ElementType.DOUBLE:
value = FormatDouble(e);
break;
default:
value = Util.ToJSONString(e.ToString());
break;
}
col.Add(name + ":" + value);
}
rows.Add("{" + string.Join(",", col) + "}");
}
return "[" + string.Join(",", rows) + "]";
}
List<List<int>> Partition(Table data, List<int> source, List<Mapper> mappers)
{
List<List<int>> s = new List<List<int>>();
s.Add(source);
foreach (Mapper m in mappers) {
Dictionary<string, List<int>> groups = new Dictionary<string, List<int>>();
foreach (List<int> q in s) {
foreach (int row in q) {
string key = m.Map(data.Row(row));
List<int> l;
if (!groups.TryGetValue(key.ToString(), out l)) {
l = new List<int>();
groups.Add(key.ToString(), l);
}
l.Add(row);
}
}
s.Clear();
foreach (var p in groups) {
s.Add(p.Value);
}
}
return s;
}
private List<List<int>> Group(Table data, List<int> source, Mapper mapper)
{
Dictionary<string, List<int>> groups = new Dictionary<string, List<int>>();
foreach (int p in source) {
string key = mapper.Map(data.Row(p));
if (!groups.ContainsKey(key)) groups.Add(key, new List<int>());
groups[key].Add(p);
}
List<List<int>> r = new List<List<int>>();
foreach (var g in groups.Values) {
r.Add(g);
}
return r;
}
private List<List<int>> CreateGroups(Table data, List<int> source, List<Mapper> mappers)
{
List<List<int>> s = new List<List<int>>();
s.Add(source);
foreach (Mapper m in mappers) {
List<List<int>> q = new List<List<int>>();
foreach (List<int> g in s) {
List<List<int>> z = Group(data, g, m);
foreach (var n in z) {
q.Add(n);
}
}
s = q;
}
return s;
}
public override Expression Evaluate(EvalStore store)
{
Stopwatch watch;
store.Remove("$DATA");
var facts = store.Get<Table>("Facts");
var fromList = store.Get<Dictionary<string, Table>>("$FROM_LIST");
if (fromList == null || fromList.Count == 0) {
return ConstExpr.Empty;
}
HashSet<string> refIds = store.Get<HashSet<string>>("$REFIDS");
var selFacts = SelectFactTableRows(fromList, facts);
for (int i = 0; i < _exprs.Count; i++) {
_exprs[i] = Reduce(store, _exprs[i]);
}
Dictionary<int, List<Expression>> exprGroups;
Dictionary<string, Expression> idExprMap;
NormalizeExpressions(selFacts, _exprs, out exprGroups, out idExprMap);
//find out the time range
if (exprGroups[TIME_FILTER].Count == 0) {
throw new Exception("Time range expression is not specified.");
}
DateTime min = DateTime.MaxValue;
DateTime max = DateTime.MinValue;
DateTime[] range = ExtraTimeRange(store, exprGroups[TIME_FILTER][0]);
if (range != null) {
if (range[0] < min) min = range[0];
if (range[1] > max) max = range[1];
}
else {
throw new Exception("Time range is not specified.");
}
//find a list of data log by applying filters of various properties from the fact table
Table sel = null;
using (EvalScope scope = new EvalScope(store, new EvalContext(EvalContext.ContextType.SEARCH))) {
if (exprGroups[ATTR_FILTER].Count == 0) {
sel = (Table) selFacts["_"];
}
else {
store.Put<Dictionary<string, Table>>("$SEL_FACTS", selFacts);
foreach (Expression expr in exprGroups[ATTR_FILTER]) {
Expression e = expr.Evaluate(store);
if (e.ExpressionType != Language.ExpressionType.CONST) {
throw new Exception("The expression should be evaluated to a value. " + e.ToString());
}
ConstExpr c = e as ConstExpr;
if (c.Value.DataType == ElementType.TABLE) {
Table s = c.Value.GetObject<Table>();
if (sel == null) sel = s;
else sel.IntersectWith(s);
}
else {
if (c.Value.DataType != ElementType.EMPTY) {
throw new Exception("The expression should be evaluated to a set of rows or an empty set. " + e.ToString());
}
}
}
}
}
if (sel == null || sel.Rows == 0) return ConstExpr.Empty;
//creating additional filters for the metric loading queries
string resFilter = CreateResourceFilters(sel);
string ctxFilter = CreateContextFilter(store, exprGroups);
string filters;
if (!string.IsNullOrEmpty(resFilter) && !string.IsNullOrEmpty(ctxFilter)) {
filters = (resFilter + " AND " + ctxFilter);
}
else if (string.IsNullOrEmpty(resFilter)) {
filters = ctxFilter;
}
else {
filters = resFilter;
}
MetricsDataLoader dataLoader = new MetricsDataLoader();
Table data = dataLoader.Load(sel, min, max, filters, refIds);
Trace.WriteLine("WhereClase.Evaluate.TotalRows: " + data.Rows);
//applies filters that involve result columns
watch = Stopwatch.StartNew();
if (exprGroups[OTHER_FILTER].Count > 0) {
using (EvalScope scope = new EvalScope(store, new EvalContext(EvalContext.ContextType.FILTER))) {
store.Put<Table>("$DATA", data);
HashSet<int> selRows = new HashSet<int>();
for (int i = 0; i < data.Rows; i++) {
store.Put<Element[]>("$ROW", data.Row(i));
bool include = true;
foreach (Expression expr in exprGroups[OTHER_FILTER]) {
ConstExpr c = expr.Evaluate(store) as ConstExpr;
if (!((bool)c.Value)) {
include = false;
break;
}
}
if (include) {
selRows.Add(i);
}
}
data = data.Select(selRows);
}
}
if (data == null) data = new Table();
watch.Stop();
Trace.WriteLine("WhereClase.Evaluate.Filter: " + watch.ElapsedMilliseconds + "ms");
Trace.WriteLine("WhereClase.Evaluate.SelectedRows: " + data.Rows);
store.Put<Table>("$DATA", data);
return new ConstExpr(data);
}
public Table Load(Table selSeries, DateTime start, DateTime end, string filters, HashSet<string> refIds)
{
Stopwatch watch = Stopwatch.StartNew();
Dictionary<int, Dictionary<int, HashSet<int>>> loadSpecs = new Dictionary<int, Dictionary<int, HashSet<int>>>();
loadSpecs.Add(STORE_VALUE, new Dictionary<int, HashSet<int>>());
loadSpecs.Add(STORE_INST, new Dictionary<int, HashSet<int>>());
ElementType valueType = (ElementType) selSeries[0, selSeries.ColumnDefinitions.IndexOf("AttrDataType")].LongValue();
for (int i = 0; i < selSeries.Rows; i++) {
int attrID = selSeries[i, selSeries.ColumnDefinitions.IndexOf("AttrID")].ToInt32(null);
int logID = selSeries[i, selSeries.ColumnDefinitions.IndexOf("LogTypeID")].ToInt32(null);
int storeType = selSeries[i, selSeries.ColumnDefinitions.IndexOf("StoreType")].ToInt32(null);
if (!loadSpecs[storeType].ContainsKey(logID)) {
loadSpecs[storeType].Add(logID, new HashSet<int>());
}
loadSpecs[storeType][logID].Add(attrID);
}
//create the table schema
List<TableColumn> s = new List<TableColumn>();
s.Add(new TableColumn("time", ElementType.DATETIME, false));
s.Add(new TableColumn("LogTypeID", ElementType.LONG, false));
s.Add(new TableColumn("ResID", ElementType.LONG, false));
s.Add(new TableColumn("AttrID", ElementType.LONG, false));
s.Add(new TableColumn("Context", ElementType.STRING, false));
s.Add(new TableColumn("value", valueType, false));
s.Add(new TableColumn("severity", ElementType.LONG, false));
TableColumns schema = new TableColumns(s);
int numCols = schema.Count;
foreach (string col in refIds) {
if (schema.IndexOf(col) < 0) numCols++;
}
List<Element[]> rows = new List<Element[]>();
using (SqlConnection connection = new SqlConnection(this.GetConnectionString())) {
connection.Open();
foreach (KeyValuePair<int, Dictionary<int, HashSet<int>>> ls in loadSpecs) {
foreach (KeyValuePair<int, HashSet<int>> l in ls.Value) {
string cmd = CreateQueryString(l.Key, l.Value, ls.Key, filters);
using (SqlCommand command = new SqlCommand(cmd, connection)) {
command.Parameters.AddWithValue("@startTime", start.ToUniversalTime());
command.Parameters.AddWithValue("@endTime", end.ToUniversalTime());
SqlDataReader reader = command.ExecuteReader();
while (reader.Read()) {
DateTime time = reader.GetDateTime(0);
string context = reader.GetString(1);
int resID = reader.GetInt32(2);
try {
int[] attrIDs = new int[l.Value.Count];
l.Value.CopyTo(attrIDs);
for (int i = 0; i < attrIDs.Length; i++) {
Element[] row = new Element[numCols];
row[0] = new Element(time); //time
row[1] = new Element(l.Key); //log ID
row[2] = new Element(resID); //resource ID
row[3] = new Element(attrIDs[i]); //attribute ID
row[4] = new Element(context); //context
try {
row[5] = GetCellValue(reader, VALUE_COL_START + i * 2, valueType); //value
row[6] = new Element(reader.GetInt16(VALUE_COL_START + i * 2 + 1)); //severity
}
catch (Exception ex) {
Trace.Write(ex);
}
rows.Add(row);
}
}
catch (Exception ex) {
Trace.WriteLine("Error: " + ex.Message);
}
}
reader.Close();
}
}
}
}
Table data = new Table("$ATTR_DATA", schema, rows);
watch.Stop();
Trace.WriteLine("MetricsDataLoader.Load : " + watch.ElapsedMilliseconds + "ms");
return data;
}
public virtual void UnionWith(Table b)
{
if (this.Cols != b.Cols) {
throw new Exception("Table columns must agree.");
}
for (int i = 0; i < this.Cols; i++) {
if (b.ColumnDefinitions[i].DataType != b.ColumnDefinitions[i].DataType) {
throw new Exception("Table column definitions must agree.");
}
}
Dictionary<string, Element[]> rows = new Dictionary<string, Element[]>();
for (int i = 0; i < this.Rows; i++) {
Element[] row = this.Row(i);
string key = MakeRowKey(row);
if (!rows.ContainsKey(key)) {
rows.Add(key, row);
}
}
for (int i = 0; i < b.Rows; i++) {
Element[] row = b.Row(i);
string key = MakeRowKey(row);
if (!rows.ContainsKey(key)) {
rows.Add(key, row);
}
}
List<Element[]> newRows = new List<Element[]>();
foreach (Element[] row in rows.Values) {
newRows.Add(row);
}
_rows = newRows;
}
