public override Expression Evaluate(EvalStore store) { Table data = store.Get<Table>("$DATA"); if (data == null || data.Rows == 0) return ConstExpr.Empty; int limit; int offset=0; Expression e = _expr.Evaluate(store); if (e.ExpressionType != Language.ExpressionType.CONST) throw new Exception("LIMIT number must be evaluated to an integer."); Element l = (e as ConstExpr).Value; if (l.DataType != ElementType.LONG) throw new Exception("LIMIT number must be evaluated to an integer."); limit = l.ToInt32(null); if (limit < 1) throw new Exception("LIMIT must be an positive integer greater than 1."); Element o = null; if (this.Offset != null) { e = _expr.Evaluate(store); if (e.ExpressionType != Language.ExpressionType.CONST) throw new Exception("OFFSET number must be evaluated to an integer."); o = (e as ConstExpr).Value; if (o.DataType != ElementType.LONG) throw new Exception("OFFSET number must be evaluated to an integer."); offset = o.ToInt32(null); if (offset < 0) throw new Exception("OFFSET must be an positive integer."); } int end = offset + limit; if (end > data.Rows) end = data.Rows; List<Element[]> rows = new List<Element[]>(); for (int i = offset; i < end; i++) { rows.Add(data.Row(i)); } Table r = new Table(data.Name, data.ColumnDefinitions, rows); store.Put<Table>("$DATA", r); return new ConstExpr(r); }
public TableIndexerSort(Table src, int col) { _entries = new TableIndexEntry[src.Rows]; for (int i = 0; i < src.Rows; i++) { _entries[i] = new TableIndexEntry(src[i, col], i); } Array.Sort<TableIndexEntry>(_entries); }
public TableIndexerHash(Table src, int col) { _entries = new Dictionary<string, List<int>>(); for (int i = 0; i < src.Rows; i++) { string key = src[i, col].ToString(); this.Add(key, i); } }
public TableIndexerHash(Table src, int[] col) { _entries = new Dictionary<string, List<int>>(); string[] keys = new string[col.Length]; for (int i = 0; i < src.Rows; i++) { for (int j = 0; j < col.Length; j++) { keys[j] = (string)src[i, col[j]]; } string key = string.Join(",", keys); this.Add(key, i); } }
public TableIndexerSort(Table src, int[] col) { _entries = new TableIndexEntry[src.Rows]; string[] key = new string[col.Length]; for (int i = 0; i < src.Rows; i++) { for (int j = 0; j < col.Length; j++) { key[j] = (string)src[i, col[j]]; } _entries[i] = new TableIndexEntry(new Element(string.Join(",", key)), i); } Array.Sort<TableIndexEntry>(_entries); }
public string CreateResourceFilters(Table sel) { HashSet<string> resIds = new HashSet<string>(); int resCol = sel.ColumnDefinitions.IndexOf("ResID"); for (int i = 0; i < sel.Rows; i++) { resIds.Add((string)sel[i, resCol]); } if (resIds.Count == 0) return null; string[] ids = new string[resIds.Count]; resIds.CopyTo(ids); if (ids.Length == 1) return "ResID=" + ids[0]; else return "ResID in (" + string.Join(",", ids) + ")"; }
public override Expression Evaluate(EvalStore store) { var e = _expr.To<IdentifierExpr>(); object o = store.Get<object>(e.Name); if (o is Table) { return new ConstExpr(o as Table); } else if (o is Element) { Element v = o as Element; List<TableColumn> cols = new List<TableColumn>(); cols.Add(new TableColumn(null,v.DataType, false)); List<Element[]> row = new List<Element[]>(); row.Add(new Element[1]); row[0][0]=v; Table t = new Table(null, cols, row); return new ConstExpr(t); } return ConstExpr.Empty; }
private Dictionary<string, Table> SelectFactTableRows(Dictionary<string, Table> fromList, Table facts) { Dictionary<string, Table> selFacts = new Dictionary<string, Table>(); foreach (var item in fromList) { Table s = item.Value; for (int i = 0; i < s.Rows; i++) { var attrId = s[i, "AttrID"]; var f = facts.Select(new Predicate("AttrID", Predicate.PredicateOperator.EQ, attrId)); if (!selFacts.ContainsKey(item.Key)) { f.Name = item.Key; selFacts.Add(item.Key, f); } else { selFacts[item.Key].UnionWith(f); } } } return selFacts; }
private void ExecuteQuery() { var lines = queryCode.Lines; string query = string.Join("\n", lines); Expression s = null; try { Parser p = new Parser(); s = p.Parse(query); } catch (Exception ex) { Trace.WriteLine("Invalid syntax: " + ex.Message); DialogResult result = MessageBox.Show(ex.Message, "Error parsing query", MessageBoxButtons.OK, MessageBoxIcon.Error); return; } Expression r = null; try { toolStripButtonRun.Enabled = false; Stopwatch watch = Stopwatch.StartNew(); _store.Enter(); r = s.Evaluate(_store); watch.Stop(); Trace.WriteLine("QueryExecutionTime: " + watch.ElapsedMilliseconds + "ms"); if (r.Value.DataType == ElementType.TABLE) { _data = (Table)r.Value; Trace.WriteLine("Number of Rows: " + _data.Rows); PopulateListView(_data); toolStripStatusLabel1.Text = "Elapsed time: " + watch.ElapsedMilliseconds + "ms, Rows: " + _data.Rows; } else { toolStripStatusLabel1.Text = "Elapsed time: " + watch.ElapsedMilliseconds + "ms"; } } catch (Exception ex) { Trace.WriteLine("Error evaluating statement: " + ex.Message); DialogResult result = MessageBox.Show(ex.Message, "Error executing query", MessageBoxButtons.OK, MessageBoxIcon.Error); } finally { _store.Exit(); toolStripButtonRun.Enabled = true; } }
private void PopulateListView(Table data) { Stopwatch watch = Stopwatch.StartNew(); resultList.BeginUpdate(); resultList.Items.Clear(); resultList.Columns.Clear(); resultList.View = View.Details; resultList.LabelEdit = false; resultList.FullRowSelect = true; resultList.GridLines = true; resultList.Sorting = System.Windows.Forms.SortOrder.None; resultList.AutoResizeColumns(ColumnHeaderAutoResizeStyle.ColumnContent); resultList.Columns.Add("", -2, HorizontalAlignment.Right); for (int i = 0; i < data.Cols; i++) { switch (data.ColumnDefinitions[i].DataType) { case ElementType.DOUBLE: case ElementType.LONG: resultList.Columns.Add(data.ColumnDefinitions[i].Name, -2, HorizontalAlignment.Right); break; default: resultList.Columns.Add(data.ColumnDefinitions[i].Name, -2, HorizontalAlignment.Left); break; } } _items = new ListViewItem[data.Rows]; for (int i = 0; i < data.Rows; i++) { _items[i] = new ListViewItem((i + 1).ToString()); for (int j = 0; j < data.Cols; j++) { _items[i].SubItems.Add(FormatValue(data[i, j])); } } resultList.Items.AddRange(_items); resultList.AutoResizeColumns(ColumnHeaderAutoResizeStyle.ColumnContent); AutoResizeColumns(resultList); resultList.EndUpdate(); watch.Stop(); Trace.WriteLine("PopulateListView: " + watch.ElapsedMilliseconds + "ms"); }
private Table CreateMasterDataTable(Table data, Table facts, HashSet<string> selIds) { if (data == null) return new Table(); if (data.Rows == 0) return data; HashSet<int> factCols = new HashSet<int>(); foreach (string id in selIds) { int col = data.ColumnDefinitions.IndexOf(id); if (col < 0) { col = facts.ColumnDefinitions.IndexOf(id); if (col < 0) { throw new Exception("No such column: " + id); } factCols.Add(col); } } if (factCols.Count == 0) return data; int dataCols = data.Cols; foreach (int c in factCols) { data.ColumnDefinitions.Add(facts.ColumnDefinitions[c]); } TableIndex idx = facts.GetIndex(new string[3] { "LogTypeID", "ResID", "AttrID" }); for (int i = 0; i < data.Rows; i++) { Element[] row = data.Row(i); string key = data[i, "LogTypeID"].ToString() + "," + data[i, "ResID"].ToString() + "," + data[i, "AttrID"].ToString(); int m = idx.FindEqual(new Element(key))[0]; Element[] r = facts.Row(m); int j = 0; foreach (int c in factCols) { row[dataCols + j] = r[c]; j++; } } return data; }
private string TableToJSON(Table t) { if (t == null) return "[]"; if (t.Rows == 0) return "[]"; List<string> rows = new List<string>(); for (int i = 0; i < t.Rows; i++) { List<string> col = new List<string>(); for (int j = 0; j < t.Cols; j++) { string name = Util.ToJSONString(t.ColumnDefinitions[j].Name); string value; Element e = t[i, j]; switch (e.DataType) { case ElementType.DATETIME: value = Util.DateTimeToUnixTimestamp(e.DateTimeValue()).ToString(); break; case ElementType.DOUBLE: value = FormatDouble(e); break; default: value = Util.ToJSONString(e.ToString()); break; } col.Add(name + ":" + value); } rows.Add("{" + string.Join(",", col) + "}"); } return "[" + string.Join(",", rows) + "]"; }
List<List<int>> Partition(Table data, List<int> source, List<Mapper> mappers) { List<List<int>> s = new List<List<int>>(); s.Add(source); foreach (Mapper m in mappers) { Dictionary<string, List<int>> groups = new Dictionary<string, List<int>>(); foreach (List<int> q in s) { foreach (int row in q) { string key = m.Map(data.Row(row)); List<int> l; if (!groups.TryGetValue(key.ToString(), out l)) { l = new List<int>(); groups.Add(key.ToString(), l); } l.Add(row); } } s.Clear(); foreach (var p in groups) { s.Add(p.Value); } } return s; }
private List<List<int>> Group(Table data, List<int> source, Mapper mapper) { Dictionary<string, List<int>> groups = new Dictionary<string, List<int>>(); foreach (int p in source) { string key = mapper.Map(data.Row(p)); if (!groups.ContainsKey(key)) groups.Add(key, new List<int>()); groups[key].Add(p); } List<List<int>> r = new List<List<int>>(); foreach (var g in groups.Values) { r.Add(g); } return r; }
private List<List<int>> CreateGroups(Table data, List<int> source, List<Mapper> mappers) { List<List<int>> s = new List<List<int>>(); s.Add(source); foreach (Mapper m in mappers) { List<List<int>> q = new List<List<int>>(); foreach (List<int> g in s) { List<List<int>> z = Group(data, g, m); foreach (var n in z) { q.Add(n); } } s = q; } return s; }
public override Expression Evaluate(EvalStore store) { Stopwatch watch; store.Remove("$DATA"); var facts = store.Get<Table>("Facts"); var fromList = store.Get<Dictionary<string, Table>>("$FROM_LIST"); if (fromList == null || fromList.Count == 0) { return ConstExpr.Empty; } HashSet<string> refIds = store.Get<HashSet<string>>("$REFIDS"); var selFacts = SelectFactTableRows(fromList, facts); for (int i = 0; i < _exprs.Count; i++) { _exprs[i] = Reduce(store, _exprs[i]); } Dictionary<int, List<Expression>> exprGroups; Dictionary<string, Expression> idExprMap; NormalizeExpressions(selFacts, _exprs, out exprGroups, out idExprMap); //find out the time range if (exprGroups[TIME_FILTER].Count == 0) { throw new Exception("Time range expression is not specified."); } DateTime min = DateTime.MaxValue; DateTime max = DateTime.MinValue; DateTime[] range = ExtraTimeRange(store, exprGroups[TIME_FILTER][0]); if (range != null) { if (range[0] < min) min = range[0]; if (range[1] > max) max = range[1]; } else { throw new Exception("Time range is not specified."); } //find a list of data log by applying filters of various properties from the fact table Table sel = null; using (EvalScope scope = new EvalScope(store, new EvalContext(EvalContext.ContextType.SEARCH))) { if (exprGroups[ATTR_FILTER].Count == 0) { sel = (Table) selFacts["_"]; } else { store.Put<Dictionary<string, Table>>("$SEL_FACTS", selFacts); foreach (Expression expr in exprGroups[ATTR_FILTER]) { Expression e = expr.Evaluate(store); if (e.ExpressionType != Language.ExpressionType.CONST) { throw new Exception("The expression should be evaluated to a value. " + e.ToString()); } ConstExpr c = e as ConstExpr; if (c.Value.DataType == ElementType.TABLE) { Table s = c.Value.GetObject<Table>(); if (sel == null) sel = s; else sel.IntersectWith(s); } else { if (c.Value.DataType != ElementType.EMPTY) { throw new Exception("The expression should be evaluated to a set of rows or an empty set. " + e.ToString()); } } } } } if (sel == null || sel.Rows == 0) return ConstExpr.Empty; //creating additional filters for the metric loading queries string resFilter = CreateResourceFilters(sel); string ctxFilter = CreateContextFilter(store, exprGroups); string filters; if (!string.IsNullOrEmpty(resFilter) && !string.IsNullOrEmpty(ctxFilter)) { filters = (resFilter + " AND " + ctxFilter); } else if (string.IsNullOrEmpty(resFilter)) { filters = ctxFilter; } else { filters = resFilter; } MetricsDataLoader dataLoader = new MetricsDataLoader(); Table data = dataLoader.Load(sel, min, max, filters, refIds); Trace.WriteLine("WhereClase.Evaluate.TotalRows: " + data.Rows); //applies filters that involve result columns watch = Stopwatch.StartNew(); if (exprGroups[OTHER_FILTER].Count > 0) { using (EvalScope scope = new EvalScope(store, new EvalContext(EvalContext.ContextType.FILTER))) { store.Put<Table>("$DATA", data); HashSet<int> selRows = new HashSet<int>(); for (int i = 0; i < data.Rows; i++) { store.Put<Element[]>("$ROW", data.Row(i)); bool include = true; foreach (Expression expr in exprGroups[OTHER_FILTER]) { ConstExpr c = expr.Evaluate(store) as ConstExpr; if (!((bool)c.Value)) { include = false; break; } } if (include) { selRows.Add(i); } } data = data.Select(selRows); } } if (data == null) data = new Table(); watch.Stop(); Trace.WriteLine("WhereClase.Evaluate.Filter: " + watch.ElapsedMilliseconds + "ms"); Trace.WriteLine("WhereClase.Evaluate.SelectedRows: " + data.Rows); store.Put<Table>("$DATA", data); return new ConstExpr(data); }
public Table Load(Table selSeries, DateTime start, DateTime end, string filters, HashSet<string> refIds) { Stopwatch watch = Stopwatch.StartNew(); Dictionary<int, Dictionary<int, HashSet<int>>> loadSpecs = new Dictionary<int, Dictionary<int, HashSet<int>>>(); loadSpecs.Add(STORE_VALUE, new Dictionary<int, HashSet<int>>()); loadSpecs.Add(STORE_INST, new Dictionary<int, HashSet<int>>()); ElementType valueType = (ElementType) selSeries[0, selSeries.ColumnDefinitions.IndexOf("AttrDataType")].LongValue(); for (int i = 0; i < selSeries.Rows; i++) { int attrID = selSeries[i, selSeries.ColumnDefinitions.IndexOf("AttrID")].ToInt32(null); int logID = selSeries[i, selSeries.ColumnDefinitions.IndexOf("LogTypeID")].ToInt32(null); int storeType = selSeries[i, selSeries.ColumnDefinitions.IndexOf("StoreType")].ToInt32(null); if (!loadSpecs[storeType].ContainsKey(logID)) { loadSpecs[storeType].Add(logID, new HashSet<int>()); } loadSpecs[storeType][logID].Add(attrID); } //create the table schema List<TableColumn> s = new List<TableColumn>(); s.Add(new TableColumn("time", ElementType.DATETIME, false)); s.Add(new TableColumn("LogTypeID", ElementType.LONG, false)); s.Add(new TableColumn("ResID", ElementType.LONG, false)); s.Add(new TableColumn("AttrID", ElementType.LONG, false)); s.Add(new TableColumn("Context", ElementType.STRING, false)); s.Add(new TableColumn("value", valueType, false)); s.Add(new TableColumn("severity", ElementType.LONG, false)); TableColumns schema = new TableColumns(s); int numCols = schema.Count; foreach (string col in refIds) { if (schema.IndexOf(col) < 0) numCols++; } List<Element[]> rows = new List<Element[]>(); using (SqlConnection connection = new SqlConnection(this.GetConnectionString())) { connection.Open(); foreach (KeyValuePair<int, Dictionary<int, HashSet<int>>> ls in loadSpecs) { foreach (KeyValuePair<int, HashSet<int>> l in ls.Value) { string cmd = CreateQueryString(l.Key, l.Value, ls.Key, filters); using (SqlCommand command = new SqlCommand(cmd, connection)) { command.Parameters.AddWithValue("@startTime", start.ToUniversalTime()); command.Parameters.AddWithValue("@endTime", end.ToUniversalTime()); SqlDataReader reader = command.ExecuteReader(); while (reader.Read()) { DateTime time = reader.GetDateTime(0); string context = reader.GetString(1); int resID = reader.GetInt32(2); try { int[] attrIDs = new int[l.Value.Count]; l.Value.CopyTo(attrIDs); for (int i = 0; i < attrIDs.Length; i++) { Element[] row = new Element[numCols]; row[0] = new Element(time); //time row[1] = new Element(l.Key); //log ID row[2] = new Element(resID); //resource ID row[3] = new Element(attrIDs[i]); //attribute ID row[4] = new Element(context); //context try { row[5] = GetCellValue(reader, VALUE_COL_START + i * 2, valueType); //value row[6] = new Element(reader.GetInt16(VALUE_COL_START + i * 2 + 1)); //severity } catch (Exception ex) { Trace.Write(ex); } rows.Add(row); } } catch (Exception ex) { Trace.WriteLine("Error: " + ex.Message); } } reader.Close(); } } } } Table data = new Table("$ATTR_DATA", schema, rows); watch.Stop(); Trace.WriteLine("MetricsDataLoader.Load : " + watch.ElapsedMilliseconds + "ms"); return data; }
public virtual void UnionWith(Table b) { if (this.Cols != b.Cols) { throw new Exception("Table columns must agree."); } for (int i = 0; i < this.Cols; i++) { if (b.ColumnDefinitions[i].DataType != b.ColumnDefinitions[i].DataType) { throw new Exception("Table column definitions must agree."); } } Dictionary<string, Element[]> rows = new Dictionary<string, Element[]>(); for (int i = 0; i < this.Rows; i++) { Element[] row = this.Row(i); string key = MakeRowKey(row); if (!rows.ContainsKey(key)) { rows.Add(key, row); } } for (int i = 0; i < b.Rows; i++) { Element[] row = b.Row(i); string key = MakeRowKey(row); if (!rows.ContainsKey(key)) { rows.Add(key, row); } } List<Element[]> newRows = new List<Element[]>(); foreach (Element[] row in rows.Values) { newRows.Add(row); } _rows = newRows; }