public GroupedTable(Database.Table table, ArrayRange range, int colToGroupFirst, int colToGroupLast, int[] colGroupOrder, SortOrder[] sortOrder)
: base(table.Schema)
{
m_table = table;
if (m_table is ExpandTable)
{
m_expandTable = (ExpandTable)m_table;
}
m_Meta = m_table.GetMetaData();
m_GroupedColumnFirst = colToGroupFirst;
m_GroupedColumnLast = colToGroupLast;
m_ColumnOrder = colGroupOrder;
m_SortOrder = sortOrder;
int sourceGroupedColumnIndex = m_ColumnOrder[colToGroupFirst];
using (Profiling.GetMarker(Profiling.MarkerId.GroupedTable).Auto())
{
var col = m_table.GetColumnByIndex(sourceGroupedColumnIndex);
var metaCol = m_Meta.GetColumnByIndex(sourceGroupedColumnIndex);
if (metaCol.DefaultGroupAlgorithm != null)
{
m_GroupCollection = metaCol.DefaultGroupAlgorithm.Group(col, range, m_SortOrder[colToGroupFirst]);
}
else
{
throw new Exception("Trying to group a column without grouping algorithm. Column '" + metaCol.Name + "' from table '" + m_table.GetName() + "'");
}
InitializeFromGroupCollection(col, sourceGroupedColumnIndex);
}
}
protected virtual long[] GetSortIndex(System.Collections.Generic.IComparer <DataT> comparer, ArrayRange indices, bool relativeIndex)
{
using (Profiling.GetMarker(Profiling.MarkerId.Sort).Auto())
{
Update();
long count = indices.Count;
DataT[] keys = new DataT[count];
//create index array
long[] index = new long[count];
if (relativeIndex)
{
for (long i = 0; i != count; ++i)
{
index[i] = i;
keys[i] = GetRowValue(indices[i]);
}
}
else
{
for (long i = 0; i != count; ++i)
{
index[i] = indices[i];
keys[i] = GetRowValue(indices[i]);
}
}
using (Profiling.GetMarker(Profiling.MarkerId.ArraySort).Auto())
{
System.Array.Sort(keys, index, comparer);
}
return(index);
}
}
void SetSnapshot(DataRenderer dataRenderer, PackedMemorySnapshot snapshot)
{
if (snapshot == null)
{
m_RawSnapshot = null;
m_RawSchema = null;
SchemaToDisplay = null;
UpdateTableSelectionNames();
return;
}
m_RawSnapshot = snapshot;
ProgressBarDisplay.ShowBar(string.Format("Opening snapshot: {0}", System.IO.Path.GetFileNameWithoutExtension(snapshot.filePath)));
var cachedSnapshot = new CachedSnapshot(snapshot);
using (Profiling.GetMarker(Profiling.MarkerId.CrawlManagedData).Auto())
{
var crawling = Crawler.Crawl(cachedSnapshot);
crawling.MoveNext(); //start execution
var status = crawling.Current as EnumerationStatus;
float progressPerStep = 1.0f / status.StepCount;
while (crawling.MoveNext())
{
ProgressBarDisplay.UpdateProgress(status.CurrentStep * progressPerStep, status.StepStatus);
}
}
ProgressBarDisplay.ClearBar();
m_RawSchema = new RawSchema();
m_RawSchema.SetupSchema(cachedSnapshot, dataRenderer);
SchemaToDisplay = m_RawSchema;
if (k_DefaultViewFilePath.Length > 0)
{
using (ScopeDebugContext.Func(() => { return("File '" + k_DefaultViewFilePath + "'"); }))
{
Database.View.ViewSchema.Builder builder = null;
using (Profiling.GetMarker(Profiling.MarkerId.LoadViewDefinitionFile).Auto())
{
builder = Database.View.ViewSchema.Builder.LoadFromXMLFile(k_DefaultViewFilePath);
}
if (builder != null)
{
using (Profiling.GetMarker(Profiling.MarkerId.BuildViewDefinitionFile).Auto())
{
ViewSchema = builder.Build(m_RawSchema);
}
if (ViewSchema != null)
{
SchemaToDisplay = ViewSchema;
}
}
}
}
UpdateTableSelectionNames();
}
protected virtual void OnGUI(int elementIndex)
{
using (Profiling.GetMarker(Profiling.MarkerId.MemoryProfiler).Auto())
{
try
{
using (new Service <IDebugContextService> .ScopeService(new DebugContextService()))
{
var rect = m_VisualElements[elementIndex].contentRect;
if (float.IsNaN(rect.width) || float.IsNaN(rect.height))
{
rect = new Rect(0, 0, 1, 1);
}
m_VisualElementsOnGUICalls[elementIndex](rect);
}
}
catch (ExitGUIException)
{
throw;
}
catch (Exception e)
{
throw new Exception(DebugUtility.GetExceptionHelpMessage(e));
}
}
}
protected override IComparable GroupTyped(ColumnTyped <DataT> sourceColumn, ArrayRange sourceIndices)
{
using (Profiling.GetMarker(Profiling.MarkerId.MergeMedian).Auto())
{
long count = sourceIndices.indexCount;
DataT[] d = new DataT[count];
int i = 0;
foreach (var v in sourceColumn.VisitRows(sourceIndices))
{
d[i] = v;
++i;
}
Array.Sort(d, Comparer.Ascending <DataT>());
DataT median;
long mid = count / 2;
if (count % 2 == 0)
{
median = math.Div(math.Add(d[mid - 1], d[mid]), math.Make(2));
}
else
{
median = d[mid];
}
return(median);
}
}
// colToGroup is an index from aNewColumnOrder.
public GroupedTable(Database.Table table, ArrayRange range, int colToGroup, SortOrder sortOrder, FnCreateGroupTable subTable)
: base(table.Schema)
{
m_table = table;
if (m_table is ExpandTable)
{
m_expandTable = (ExpandTable)m_table;
}
m_Meta = m_table.GetMetaData();
m_GroupedColumnFirst = 0;
m_GroupedColumnLast = 0;
m_ColumnOrder = new int[] { colToGroup };// colGroupOrder;
m_SortOrder = new SortOrder[1] {
sortOrder
};
m_createGroupTable = subTable;
using (Profiling.GetMarker(Profiling.MarkerId.GroupedTable).Auto())
{
var col = m_table.GetColumnByIndex(colToGroup);
var metaCol = m_Meta.GetColumnByIndex(colToGroup);
if (metaCol.DefaultGroupAlgorithm != null)
{
m_GroupCollection = metaCol.DefaultGroupAlgorithm.Group(col, range, sortOrder);
}
else
{
throw new Exception("Trying to group a column without grouping algorithm. Column '" + metaCol.Name + "' from table '" + m_table.GetName() + "'");
}
InitializeFromGroupCollection(col, colToGroup);
}
}
protected override IComparable GroupTyped(ColumnTyped <DataT> sourceColumn, ArrayRange sourceIndices)
{
using (Profiling.GetMarker(Profiling.MarkerId.MergeMax).Auto())
{
DataT result = math.MinValue();
foreach (var v in sourceColumn.VisitRows(sourceIndices))
{
result = math.Max(result, v);
}
return(result);
}
}
protected override IComparable GroupTyped(ColumnTyped <DataT> sourceColumn, ArrayRange sourceIndices)
{
using (Profiling.GetMarker(Profiling.MarkerId.MergeAverage).Auto())
{
DataT result = math.Zero();
foreach (var v in sourceColumn.VisitRows(sourceIndices))
{
result = math.Add(result, v);
}
result = math.Div(result, math.Make(sourceIndices.indexCount));
return(result);
}
}
protected override IComparable GroupTyped(ColumnTyped <DataT> sourceColumn, ArrayRange sourceIndices)
{
using (Profiling.GetMarker(Profiling.MarkerId.MergeSumPositive).Auto())
{
DataT result = math.Zero();
DataT z = math.Zero();
foreach (var v in sourceColumn.VisitRows(sourceIndices))
{
if (math.Compare(v, z) >= 0)
{
result = math.Add(result, v);
}
}
return(result);
}
}
public override BaseMode BuildViewSchemaClone(Database.View.ViewSchema.Builder builder)
{
Database.View.ViewSchema vs;
using (Profiling.GetMarker(Profiling.MarkerId.BuildViewDefinitionFile).Auto())
{
vs = builder.Build(m_RawSchema);
}
if (vs != null)
{
SnapshotMode copy = new SnapshotMode(this);
copy.ViewSchema = vs;
copy.SchemaToDisplay = vs;
copy.UpdateTableSelectionNames();
return(copy);
}
return(null);
}
public DataChunk <DataT> GetChunk(long chunkIndex)
{
if (m_DataChunk[chunkIndex] == null)
{
using (Profiling.GetMarker(Profiling.MarkerId.NewDataChunk).Auto())
{
long indexFirst = chunkIndex * m_DataSet.ChunkSize;
long indexLast = indexFirst + m_DataSet.ChunkSize;
if (indexLast > m_DataSet.DataCount)
{
indexLast = m_DataSet.DataCount;
}
m_DataChunk[chunkIndex] = new DataChunk <DataT>(indexLast - indexFirst);
m_DataSource.Get(Range.FirstToLast(indexFirst, indexLast), ref m_DataChunk[chunkIndex].m_Data);
}
}
return(m_DataChunk[chunkIndex]);
}
public int m_ColumnIndexFirst; //index into m_SortColumn
public SortedTable(Database.Table table, int[] sortColumn, SortOrder[] sortOrder, int columnIndexFirst, ArrayRange indices)
: base(table.Schema)
{
m_SourceTable = table;
m_Meta = m_SourceTable.GetMetaData();
m_SortColumn = sortColumn;
m_SortOrder = sortOrder;
m_ColumnIndexFirst = columnIndexFirst;
using (Profiling.GetMarker(Profiling.MarkerId.SortedTable).Auto())
{
this.indices = SortRange(indices, sortColumn, sortOrder, m_ColumnIndexFirst);
}
//create columns
CreateColumn();
}
public long[] GetMatchingIndices(long row)
{
if (cacheMatchingIndicesRow != row)
{
if (HasWhereCondition())
{
using (Profiling.GetMarker(Profiling.MarkerId.Select).Auto())
{
cacheMatchingIndices = where.GetMatchingIndices(row);
cacheMatchingIndicesRow = row;
if (MaxRow >= 0)
{
System.Array.Resize(ref cacheMatchingIndices, MaxRow);
}
}
}
}
return(cacheMatchingIndices);
}
protected override IComparable GroupTyped(ColumnTyped <DataT> sourceColumn, ArrayRange sourceIndices)
{
using (Profiling.GetMarker(Profiling.MarkerId.MergeDeviation).Auto())
{
DataT count = math.Make(sourceIndices.indexCount);
DataT avg = math.Zero();
foreach (var v in sourceColumn.VisitRows(sourceIndices))
{
avg = math.Add(avg, v);
}
avg = math.Div(avg, count);
DataT dev = math.Zero();
foreach (var v in sourceColumn.VisitRows(sourceIndices))
{
dev = math.Add(dev, math.SubAbs(avg, v));
}
dev = math.Div(dev, count);
return(dev);
}
}
void ComputeAllValues()
{
long rowCount = m_ViewColumnNode.viewTable.GetGroupCount();
entries = new DataT[rowCount];
for (long row = 0; row != rowCount; ++row)
{
using (Profiling.GetMarker(Profiling.MarkerId.MergeColumn).Auto())
{
var subTable = m_ViewColumnNode.viewTable.CreateGroupTable(row);
if (subTable != null)
{
var subColumn = subTable.GetColumnByIndex(m_ViewColumnNode.metaColumn.Index);
while (subColumn is IColumnDecorator)
{
subColumn = (subColumn as IColumnDecorator).GetBaseColumn();
}
m_ViewColumnNode.metaColumn.DefaultMergeAlgorithm.Merge(this, row, subColumn, new ArrayRange(0, subColumn.GetRowCount()));
}
}
}
mbDirty = false;
}
Grouping.GroupCollection Grouping.IGroupAlgorithm.Group(Column sourceColumn, ArrayRange indices, SortOrder order)
{
using (Profiling.GetMarker(Profiling.MarkerId.GroupByDuplicate).Auto())
{
DupCollection coll = new DupCollection();
if (order == SortOrder.None)
{
order = SortOrder.Ascending;
}
coll.m_sortedIndex = sourceColumn.GetSortIndex(order, indices, false);
System.Collections.Generic.List <DupGroup> groups = new System.Collections.Generic.List <DupGroup>();
int iGroupFirst = 0;
for (int i = 1, j = 0; i < coll.m_sortedIndex.Length; j = i++)
{
if (sourceColumn.CompareRow(coll.m_sortedIndex[j], coll.m_sortedIndex[i]) != 0)
{
//create group;
Range range = Range.FirstLast(iGroupFirst, i);
groups.Add(new DupGroup(range));
iGroupFirst = i;
}
}
Range rangeLast = Range.FirstLast(iGroupFirst, coll.m_sortedIndex.Length);
if (rangeLast.length > 0)
{
groups.Add(new DupGroup(rangeLast));
}
coll.m_Groups = groups.ToArray();
#if (PROFILER_DEBUG_TEST)
{
UnityEngine.Debug.LogWarning("Testing GroupByDuplicate result...");
//test if grouping data is good
foreach (var g in coll.m_Groups)
{
Debug.Assert(g.range.first >= 0);
Debug.Assert(g.range.first < coll.m_sortedIndex.Length);
Debug.Assert(g.range.last > 0);
Debug.Assert(g.range.last <= coll.m_sortedIndex.Length);
//indices before should all be -1 compared to group's first index
for (long i = 0; i != g.range.first; ++i)
{
int c = sourceColumn.CompareRow(coll.m_sortedIndex[i], coll.m_sortedIndex[g.range.first]);
if (c != -1)
{
Debug.Assert(c == -1, "index " + i + " should be -1 compared to group starting at index " + g.range.first + "."
+ "\n Result=" + c
+ "\n val[" + i + "]=" + sourceColumn.GetRowValueString(coll.m_sortedIndex[i])
+ "\n val[" + g.range.first + "]=" + sourceColumn.GetRowValueString(coll.m_sortedIndex[g.range.first])
);
}
}
//indices inside group should all be 0 compared to group's first index
for (long i = g.range.first; i != g.range.last; ++i)
{
int c = sourceColumn.CompareRow(coll.m_sortedIndex[i], coll.m_sortedIndex[g.range.first]);
if (c != 0)
{
Debug.Assert(c == 0, "index " + i + " should be 0 compared to group starting at index " + g.range.first + "."
+ "\n Result=" + c
+ "\n val[" + i + "]=" + sourceColumn.GetRowValueString(coll.m_sortedIndex[i])
+ "\n val[" + g.range.first + "]=" + sourceColumn.GetRowValueString(coll.m_sortedIndex[g.range.first])
);
}
}
//indices after group should all be 1 compared to group's first index
for (long i = g.range.last; i != coll.m_sortedIndex.Length; ++i)
{
int c = sourceColumn.CompareRow(coll.m_sortedIndex[i], coll.m_sortedIndex[g.range.first]);
if (c != 1)
{
Debug.Assert(c == 1, "index " + i + " should be 1 compared to group starting at index " + g.range.first + "."
+ "\n Result=" + c
+ "\n val[" + i + "]=" + sourceColumn.GetRowValueString(coll.m_sortedIndex[i])
+ "\n val[" + g.range.first + "]=" + sourceColumn.GetRowValueString(coll.m_sortedIndex[g.range.first])
);
}
}
}
UnityEngine.Debug.LogWarning("Testing GroupByDuplicate result done");
}
#endif
return(coll);
}
}
public override long[] GetMatchIndex(ArrayRange rowRange, Operation.Operator operation, Operation.Expression expression, long expressionRowFirst, bool rowToRow)
{
using (Profiling.GetMarker(Profiling.MarkerId.ColumnMatchQuery).Auto())
{
Update();
long count = rowRange.Count;
long[] matchedIndices = new long[count];
long indexOflastMatchedIndex = 0;
Operation.Operation.ComparableComparator comparator = Operation.Operation.GetComparator(type, expression.type);
if (rowToRow)
{
for (long i = 0; i != count; ++i)
{
var leftValue = GetRowValue(rowRange[i]);
if (Operation.Operation.Match(operation, comparator, leftValue, expression, expressionRowFirst + i))
{
matchedIndices[indexOflastMatchedIndex] = rowRange[i];
++indexOflastMatchedIndex;
}
}
}
else
{
if (Operation.Operation.IsOperatorOneToMany(operation))
{
for (int i = 0; i != count; ++i)
{
var leftValue = GetRowValue(rowRange[i]);
if (Operation.Operation.Match(operation, comparator, leftValue, expression, expressionRowFirst))
{
matchedIndices[indexOflastMatchedIndex] = rowRange[i];
++indexOflastMatchedIndex;
}
}
}
else
{
var valueRight = expression.GetComparableValue(expressionRowFirst);
for (int i = 0; i != count; ++i)
{
var leftValue = GetRowValue(rowRange[i]);
if (Operation.Operation.Match(operation, comparator, leftValue, valueRight))
{
matchedIndices[indexOflastMatchedIndex] = rowRange[i];
++indexOflastMatchedIndex;
}
}
}
}
if (indexOflastMatchedIndex != count)
{
System.Array.Resize(ref matchedIndices, (int)indexOflastMatchedIndex);
}
#if MEMPROFILER_DEBUG_INFO
Algorithm.DebugLog("GetMatchIndex : indexCount " + rowRange.Count
+ " op:" + Operation.Operation.OperatorToString(operation)
+ " Exp():" + expression.GetValueString(expressionRowFirst, DefaultDataFormatter.Instance)
+ " expRowFirst:" + expressionRowFirst
+ " Column:" + this.GetDebugString(rowRange[0])
+ " Expression:" + expression.GetDebugString(expressionRowFirst)
+ " Result Count:" + (matchedIndices != null ? matchedIndices.Length : 0));
#endif
return(matchedIndices);
}
}
public override long[] GetMatchIndex(ArrayRange rowRange, Operation.Operator operation, Operation.Expression expression, long expressionRowFirst, bool rowToRow)
{
if (expression.type != type)
{
return(base.GetMatchIndex(rowRange, operation, expression, expressionRowFirst, rowToRow));
}
using (Profiling.GetMarker(Profiling.MarkerId.ColumnMatchQueryInt).Auto())
{
Update();
Operation.TypedExpression <int> typedExpression = expression as Operation.TypedExpression <int>;
long count = rowRange.Count;
var matchedIndices = new List <long>(128);
if (rowToRow)
{
for (long i = 0; i != count; ++i)
{
var lhs = m_Cache[rowRange[i]];
var rhs = typedExpression.GetValue(expressionRowFirst + i);
if (Operation.Operation.Match(operation, lhs, rhs))
{
matchedIndices.Add(rowRange[i]);
}
}
}
else
{
if (Operation.Operation.IsOperatorOneToMany(operation))
{
for (int i = 0; i != count; ++i)
{
var leftValue = m_Cache[rowRange[i]];
if (Operation.Operation.Match(operation, leftValue, typedExpression, expressionRowFirst))
{
matchedIndices.Add(rowRange[i]);
}
}
}
else
{
var valueRight = typedExpression.GetValue(expressionRowFirst);
//Optimization for equal operation when querying on all data
if (rowRange.IsSequence && operation == Operation.Operator.Equal)
{
//use the sorted index to trim down invalid values
long[] sortedIndex = GetSortIndexAsc();
int lowerIndexIndex = LowerBoundIndex(sortedIndex, (int)rowRange.Sequence.First, (int)rowRange.Count, valueRight);
int upperIndexIndex = (int)rowRange.Sequence.Last;
for (int i = lowerIndexIndex; i < upperIndexIndex; ++i)
{
if (m_Cache[sortedIndex[i]] == valueRight)
{
matchedIndices.Add(sortedIndex[i]);
}
else
{
break;
}
}
#if PROFILER_DEBUG_TEST
{
//Test to validate if optimization works
int matchCount = 0;
for (int i = 0; i != count; ++i)
{
var leftValue = m_Cache[rowRange[i]];
if (Operation.Operation.Match(operation, leftValue, valueRight))
{
++matchCount;
if (matchedIndices.FindIndex(x => x == rowRange[i]) < 0)
{
UnityEngine.Debug.LogError("GetMatchIndex Optimization failure on index " + rowRange[i]);
}
}
}
if (matchCount != matchedIndices.Count)
{
UnityEngine.Debug.LogError("GetMatchIndex Optimization failure on match count " + matchCount + " != " + matchedIndices.Count);
}
}
#endif
}
else
{
for (int i = 0; i != count; ++i)
{
var leftValue = m_Cache[rowRange[i]];
if (Operation.Operation.Match(operation, leftValue, valueRight))
{
matchedIndices.Add(rowRange[i]);
}
}
}
}
}
var matchedIndicesArray = matchedIndices.ToArray();
#if MEMPROFILER_DEBUG_INFO
Algorithm.DebugLog("GetMatchIndex : indexCount " + rowRange.Count
+ " op:" + Operation.Operation.OperatorToString(operation)
+ " Exp():" + expression.GetValueString(expressionRowFirst, DefaultDataFormatter.Instance)
+ " expRowFirst:" + expressionRowFirst
+ " Column:" + this.GetDebugString(rowRange[0])
+ " Expression:" + expression.GetDebugString(expressionRowFirst)
+ " Result Count:" + (matchedIndices != null ? matchedIndices.Length : 0));
#endif
return(matchedIndicesArray);
}
}
public override long GetFirstMatchIndex(Operation.Operator operation, Operation.Expression expression, long expRowFirst)
{
using (Profiling.GetMarker(Profiling.MarkerId.ColumnFirstMatchQuery).Auto())
{
Update();
long[] sortedIndex = GetSortIndexAsc();
Operation.Operation.ComparableComparator comparator = Operation.Operation.GetComparator(type, expression.type);
var val2 = expression.GetComparableValue(expRowFirst);
long firstMatchIndex = -1;
switch (operation)
{
case Operation.Operator.Less:
{
long iFirst = sortedIndex[0];
var val1 = GetRowValue(iFirst);
int result = comparator(val1, val2);
if (result < 0)
{
firstMatchIndex = iFirst;
}
break;
}
case Operation.Operator.LessEqual:
{
long iFirst = sortedIndex[0];
var val1 = GetRowValue(iFirst);
int result = comparator(val1, val2);
if (result <= 0)
{
firstMatchIndex = iFirst;
}
break;
}
case Operation.Operator.Equal:
{
long iFirstGreaterEqual = LowerBound(val2, comparator);
if (iFirstGreaterEqual < sortedIndex.Length)
{
long index = sortedIndex[iFirstGreaterEqual];
var val1 = GetRowValue(index);
int comparisonResult = comparator(val1, val2);
if (comparisonResult == 0)
{
firstMatchIndex = index;
}
}
break;
}
case Operation.Operator.GreaterEqual:
{
long iFirstGreaterEqual = LowerBound(val2, comparator);
if (iFirstGreaterEqual < sortedIndex.Length)
{
firstMatchIndex = sortedIndex[iFirstGreaterEqual];
}
break;
}
case Operation.Operator.Greater:
{
long iFirstGreater = UpperBound(val2, comparator);
if (iFirstGreater < sortedIndex.Length)
{
firstMatchIndex = sortedIndex[iFirstGreater];
}
break;
}
}
#if (PROFILER_DEBUG_TEST)
{
long count = sortedIndex.Length;
long resultTest = -1;
for (long i = 0; i != count; ++i)
{
long index = sortedIndex[i];
var val = GetRowValue(index);
int comparisonResult = comparator(val, val2);
bool matchResult = Operation.Operation.Match(operation, comparisonResult);
if (matchResult)
{
resultTest = index;
break;
}
}
UnityEngine.Debug.Assert(resultTest == firstMatchIndex);
if (resultTest >= 0)
{
var val1 = GetRowValue(resultTest);
int comparisonResult = comparator(val1, val2);
bool resultMatch = Operation.Operation.Match(operation, comparisonResult);
UnityEngine.Debug.Assert(resultMatch);
}
}
#endif
#if MEMPROFILER_DEBUG_INFO
Algorithm.DebugLog("GetFirstMatchIndex :"
+ " op:" + Operation.Operation.OperatorToString(operation)
+ " Exp():" + expression.GetValueString(expRowFirst, DefaultDataFormatter.Instance)
+ " expRowFirst:" + expRowFirst
+ " Column:" + this.GetDebugString(0)
+ " Expression:" + expression.GetDebugString(expRowFirst)
+ " Result:" + firstMatchIndex);
#endif
return(firstMatchIndex);
}
}