/// <summary>
/// Defines one aspect of aggregation config
/// </summary>
/// <param name="aggregationAspect">The aggregation aspect to be declared.</param>
/// <param name="types">The types to be set for the aspect</param>
public imbAttribute(dataPointAggregationAspect aggregationAspect, dataPointAggregationType types)
{
nameEnum = imbAttributeName.reporting_aggregation;
name = nameof(imbAttributeName.reporting_aggregation);
objExtra = aggregationAspect;
objMsg = types;
msg = types.ToString();
}
/// <summary>Performs post-processing of data collected by the workload plugin</summary>
/// <remarks><para>Loads all saved DataTables, groups rows in averages for each measure group and creates summary table with all experiments</para></remarks>
/// <param name="searchPattern">pattern used to select input files</param>
/// <param name="groupColumn">column name used for row grouping</param>
/// <param name="overviewColumns">columns to include in overview table</param>
/// <seealso cref="aceOperationSetExecutorBase"/>
public void aceOperation_runWorkloadData(
[Description("pattern used to select input files")] string searchPattern = "results*.xml",
[Description("column name used for row grouping")] string groupColumn = "measureGroup")
// [Description("columns to include in overview table")] String overviewColumns = "DataLoad,CrawlerIterations,ContentPages,dlcMaximum")
{
aceOperation_selectFiles(searchPattern, "index\\workload", true);
folder = folder["index\\workload"];
List <DataTable> tables = new List <DataTable>();
dataPointAggregationType aggType = dataPointAggregationType.avg;
int ci = 1;
int c = selectedFiles.Count();
output.log("[" + c + "] DataTable in the cue.");
List <DataTable> allTables = new List <DataTable>();
DataSet dSet = new DataSet();
aceDictionarySet <string, DataTable> byCrawler = new aceDictionarySet <string, DataTable>();
aceDictionarySet <string, DataTableForStatistics> byCrawlerRT = new aceDictionarySet <string, DataTableForStatistics>();
DataTableForStatistics rt = null;
foreach (FileInfo fi in selectedFiles)
{
try
{
objectTable <reportPlugIn_workloadEntry> workloadEntry = new objectTable <reportPlugIn_workloadEntry>(fi.FullName, true, "EntryID", "");
objectTable <reportPlugIn_workloadEntry> workloadGrouped = new objectTable <reportPlugIn_workloadEntry>("EntryID", "aggregated");
aceDictionarySet <int, reportPlugIn_workloadEntry> workloadGroups = workloadEntry.GetGroups <int>(groupColumn, "terminationWarning = 0");
collectionAggregationResultSet <reportPlugIn_workloadEntry> aggregateSet = new collectionAggregationResultSet <reportPlugIn_workloadEntry>();
foreach (var set in workloadGroups)
{
collectionAggregationResult <reportPlugIn_workloadEntry> aggregates = null;
aggregates = set.Value.GetAggregates(aggType);
var aggregate = aggregates[aggType];
aggregate.measureGroup = set.Key;
aggregate.EntryID = set.Key.ToString("D5") + "_" + aggType.ToString();
workloadGrouped.AddOrUpdate(aggregate);
aggregateSet.Add(aggregate.EntryID + "_" + fi.Name, aggregates);
}
string filename = (fi.Name + "_" + groupColumn + "_" + aggType.ToString()).getFilename();
string n = reportPlugIn_workload_state.ExtractEntryID(aggregateSet.lastItem.EntryID) + dSet.Tables.Count.ToString("D2");
DataTable dt = workloadGrouped.GetDataTable(dSet, n);
dt.SetDescription("Collection of [" + aggregateSet.recordType.Name + "] records grouped by [" + groupColumn + "]");
dt.SetAggregationAspect(dataPointAggregationAspect.subSetOfRows);
dt.SetAggregationOriginCount(aggregateSet.Count);
dt.SetAdditionalInfoEntry("Aggregation Type:", aggType);
dt.SetAdditionalInfoEntry("Data source file:", fi.Name);
dt.SetAdditionalInfoEntries("Last", aggregateSet.lastItem, typeof(string));
dt.SetTitle(n);
byCrawler.Add(aggregateSet.firstItem.Crawler, dt);
// dt.TableName = n;
// dSet.AddTable(dt);
rt = dt.GetReportAndSave(folder, imbWEMManager.authorNotation, n.getFilename(), true);
byCrawlerRT.Add(aggregateSet.firstItem.Crawler, rt);
response.AppendLine("[" + ci + " / " + c + "] DataTable [" + fi.Name + "] had [" + workloadGroups.Keys.Count() + "] groups. Result saved as: " + filename);
ci++;
} catch (Exception ex)
{
output.log("[" + ci + " / " + c + "] DataTable [" + fi.FullName + "] failed.");
output.log(ex.Message);
}
}
output.log("[" + c + "] DataTable processed.");
dSet.serializeDataSet("workload_all", folder, dataTableExportEnum.excel, imbWEMManager.authorNotation);
foreach (string key in byCrawler.Keys)
{
string filename = key.getFilename();
DataSet sd = new DataSet(key);
foreach (DataTable dti in byCrawler[key])
{
sd.AddTable(dti.Copy());
}
sd.AddTable(byCrawlerRT[key].First().RenderLegend());
sd.serializeDataSet(filename, folder, dataTableExportEnum.excel, imbWEMManager.authorNotation);
}
}
public DataColumnInReportDefinition Add(DataColumnInReportTypeEnum columnType, DataColumn column, dataPointAggregationType aggregation, string unit = "")
{
DataColumnInReportDefinition output = new DataColumnInReportDefinition();
output.columnType = columnType;
output.aggregation = aggregation;
output.columnSourceName = column.ColumnName;
output.columnPriority = column.GetPriority();
output.format = column.GetFormat();
Type valueType = typeof(string);
string name = ""; string description = ""; string letter = "";
switch (aggregation)
{
default:
case dataPointAggregationType.max:
case dataPointAggregationType.min:
case dataPointAggregationType.sum:
case dataPointAggregationType.firstEntry:
case dataPointAggregationType.lastEntry:
case dataPointAggregationType.range:
valueType = column.DataType;
break;
case dataPointAggregationType.avg:
case dataPointAggregationType.stdev:
case dataPointAggregationType.var:
case dataPointAggregationType.entropy:
valueType = typeof(double);
if (output.format.isNullOrEmpty())
{
output.format = "F5";
}
break;
case dataPointAggregationType.count:
valueType = typeof(int);
break;
}
letter = column.GetLetter();
if (columnType == DataColumnInReportTypeEnum.dataSummed)
{
if (!letter.isNullOrEmpty())
{
letter = aggregation.ToString() + "(" + letter + ")";
}
output.columnLetter = letter;
output.columnDescription = "(" + aggregation.ToString() + ") of " + column.ColumnName + ". " + column.GetDesc();
}
output.columnName = column.ColumnName + " (" + aggregation.ToString() + ")";
output.columnSourceName = column.ColumnName;
output.importance = column.GetImportance();
output.columnUnit = column.GetUnit();
output.columnValueType = valueType;
output.columnDefault = valueType.GetDefaultValue();
output.columnGroup = column.GetGroup();
output.spe = column.GetSPE();
Add(column.ColumnName, output);
return(output);
}