static void Measure(QueryMode mode, GrouperAlias gA, SorterAlias sA, string query, int threadCount)
{
WarmUp(mode, gA, sA, query, threadCount);
Console.WriteLine();
Console.WriteLine($"Starting measurements for {mode} / {gA} / {sA} / {query} / {threadCount}.");
double[] times = new double[repetitions];
for (int i = 0; i < repetitions; i++)
{
CleanGC();
var q = Query.Create(mode, query, graph, threadCount, PrinterType.Console, FormaterType.Simple, verticesPerThread, null, gA, sA, fixedArraySize, false);
timer.Restart();
q.Compute();
timer.Stop();
times[i] = timer.ElapsedMilliseconds;
Console.WriteLine($"Finished repetition {i} with {times[i]}");
}
ProcessResults(times, $"Measurements for {mode} / {gA} / {sA} / {query} / {threadCount}.");
}
static void WarmUp(QueryMode mode, GrouperAlias gA, SorterAlias sA, string query, int threadCount)
{
Console.WriteLine($"Starting warm ups for {mode} / {gA} / {sA} / {query} / {threadCount}.");
for (int i = 0; i < warmUps; i++)
{
CleanGC();
var q = Query.Create(mode, query, graph, threadCount, PrinterType.Console, FormaterType.Simple, verticesPerThread, null, gA, sA, fixedArraySize, false);
q.Compute();
Console.WriteLine($"Finished warm up no. {i}.");
}
}
public static void CheckAliases(GrouperAlias grouperAlias, SorterAlias sorterAlias, QueryMode mode)
{
if (mode == QueryMode.HalfStreamed)
{
if (!Aliases.HalfStreamedGroupers.Contains(grouperAlias))
{
throw new ArgumentException("Query HS, invalid grouper alias.");
}
else if (!Aliases.HalfStreamedSorters.Contains(sorterAlias))
{
throw new ArgumentException("Query HS, invalid sorter alias.");
}
else
{
}
}
else if (mode == QueryMode.Streamed)
{
if (!Aliases.StreamedGroupers.Contains(grouperAlias))
{
throw new ArgumentException("Query S, invalid grouper alias.");
}
else if (!Aliases.StreamedSorters.Contains(sorterAlias))
{
throw new ArgumentException("Query S, invalid sorter alias.");
}
else
{
}
}
else if (mode == QueryMode.Normal)
{
if (!Aliases.NormalGroupers.Contains(grouperAlias))
{
throw new ArgumentException("Query N, invalid grouper alias.");
}
else if (!Aliases.NormalSorters.Contains(sorterAlias))
{
throw new ArgumentException("Query N, invalid sorter alias.");
}
else
{
}
}
else
{
throw new ArgumentException("Query, invalid mode type.");
}
}
/// <summary>
/// A factory method.
/// Creates an appropriate group by solution based on the provided alias.
/// </summary>
public static Grouper Factory(GrouperAlias grouperAlias, Aggregate[] aggs, ExpressionHolder[] hashes, IGroupByExecutionHelper helper)
{
if (grouperAlias == GrouperAlias.RefB)
{
return(new GroupByWithBuckets(aggs, hashes, helper));
}
else if (grouperAlias == GrouperAlias.RefL)
{
return(new GroupByWithLists(aggs, hashes, helper));
}
else if (grouperAlias == GrouperAlias.GlobalB)
{
return(new GlobalGroupByBucket(aggs, hashes, helper, true));
}
else if (grouperAlias == GrouperAlias.GlobalL)
{
return(new GlobalGroupByArray(aggs, hashes, helper, false));
}
else if (grouperAlias == GrouperAlias.LocalB)
{
return(new LocalGroupByLocalTwoWayMergeBucket(aggs, hashes, helper, true));
}
else if (grouperAlias == GrouperAlias.LocalL)
{
return(new LocalGroupByLocalTwoWayMergeList(aggs, hashes, helper, false));
}
else if (grouperAlias == GrouperAlias.TwoStepB)
{
return(new TwoStepGroupByBucket(aggs, hashes, helper, true));
}
else if (grouperAlias == GrouperAlias.TwoStepL)
{
return(new TwoStepGroupByListBucket(aggs, hashes, helper, false));
}
else
{
throw new ArgumentException("Grouper, trying to create an unknown grouper.");
}
}
public QueryExecutionHelper(int threadCount, PrinterType printer, FormaterType formater, int verticesPerThread, int arraySize, string fileName, string ppmName, string stpmName, string patternName, GrouperAlias grouperName, SorterAlias sorterName)
{
this.ThreadCount = threadCount;
this.Printer = printer;
this.Formater = formater;
this.VerticesPerThread = verticesPerThread;
this.FixedArraySize = arraySize;
this.FileName = fileName;
this.ParallelPatternMatcherName = ppmName;
this.SingleThreadPatternMatcherName = stpmName;
this.PatternName = patternName;
this.GrouperAlias = grouperName;
this.SorterAlias = sorterName;
}
private static Query CreateInternalL(QueryMode mode, List <Token> tokens, Graph graph, int threadCount, PrinterType printer, FormaterType formater, int verticesPerThread, string fileName, GrouperAlias grouperAlias, SorterAlias sorterAlias, int fixedArraySize, bool allowPrint)
{
CheckArgs(tokens, graph, threadCount, printer, formater, verticesPerThread, fileName, fixedArraySize);
CheckAliases(grouperAlias, sorterAlias, mode);
if (mode == QueryMode.HalfStreamed)
{
return(new Query(tokens, graph, allowPrint, threadCount, printer, formater, verticesPerThread, fileName, grouperAlias, sorterAlias, fixedArraySize, false));
}
else if (mode == QueryMode.Streamed)
{
return(new Query(tokens, graph, allowPrint, threadCount, printer, formater, verticesPerThread, fileName, grouperAlias, sorterAlias, fixedArraySize, true));
}
else
{
return(new Query(tokens, graph, allowPrint, threadCount, printer, formater, verticesPerThread, fileName, grouperAlias, sorterAlias, fixedArraySize));
}
}
public static Query Create(QueryMode mode, TextReader inputQuery, Graph graph, int threadCount, PrinterType printer, FormaterType formater, int verticesPerThread, string fileName, GrouperAlias grouperAlias, SorterAlias sorterAlias, int fixedArraySize, bool allowPrint)
{
return(CreateInternalL(mode, Tokenizer.Tokenize(inputQuery), graph, threadCount, printer, formater, verticesPerThread, fileName, grouperAlias, sorterAlias, fixedArraySize, allowPrint));
}
/// <summary>
/// Builds a streamed version of a query from an input.
/// Cannot contain order by and group by simultaneously.
/// Called by static create method.
/// </summary>
/// <param name="tokens"> An input query.</param>
/// <param name="graph"> A graph to compute the query on.</param>
/// <param name="threadCount"> Maximum number of threads available to use.</param>
/// <param name="printer"> A printer to use. </param>
/// <param name="formater"> A formater to use by printer. </param>
/// <param name="verticesPerThread"> A number of vertices distributed to threads during parallel computation of the query.</param>
/// <param name="fileName"> A file to store results into if set, otherwise null. </param>
/// <param name="isStreamed"> A flag to distinguish a normal construtor from streamed constructor.</param>
/// <param name="allowPrint"> Whether to allow printing. Mainly for benchmarking purposes.</param>
/// <param name="fixedArraySize"> The size of the arrays used for storing results of the matcher.</param>
/// <param name="grouperAlias"> A grouper to use when specifying group by.</param>
/// <param name="sorterAlias"> A sorter to use when specifying order by.</param>
private Query(List <Token> tokens, Graph graph, bool allowPrint, int threadCount, PrinterType printer, FormaterType formater, int verticesPerThread, string fileName, GrouperAlias grouperAlias, SorterAlias sorterAlias, int fixedArraySize, bool isStreamed)
{
this.graph = graph;
this.variableMap = new VariableMap();
this.qEhelper = new QueryExecutionHelper(threadCount, printer, formater, verticesPerThread, fixedArraySize, fileName, "DFSParallelStreamed", "DFSSingleThreadStreamed", "SIMPLE", grouperAlias, sorterAlias);
// Parse input query.
var parsedClauses = Parser.Parse(tokens);
if (parsedClauses.ContainsKey(Parser.Clause.OrderBy) && parsedClauses.ContainsKey(Parser.Clause.GroupBy))
{
throw new ArgumentException($"{this.GetType()}, the streamed version of the query cannot contain group by and order by at the same time.");
}
// MATCH is always leaf.
MatchObjectStreamed match = (MatchObjectStreamed)QueryObject.Factory
(typeof(MatchObjectStreamed), graph, qEhelper, variableMap, parsedClauses[Parser.Clause.Match], null);
// GROUP BY and obtain the aggregates and hashes -> the all necessary info is in the expressionInfo class.
if (parsedClauses.ContainsKey(Parser.Clause.GroupBy))
{
this.exprInfo = new QueryExpressionInfo(true);
GroupByResultProcessor.ParseGroupBy(graph, variableMap, qEhelper, (GroupByNode)parsedClauses[Parser.Clause.GroupBy], exprInfo);
}
else
{
this.exprInfo = new QueryExpressionInfo(false);
}
// SELECT is the last one to process the resuls.
this.query = QueryObject.Factory
(typeof(SelectObject), graph, qEhelper, variableMap, parsedClauses[Parser.Clause.Select], exprInfo);
((SelectObject)this.query).allowPrint = allowPrint;
SetSingleGroupFlags();
// ORDER BY
if (parsedClauses.ContainsKey(Parser.Clause.OrderBy))
{
var comps = OrderByResultProcessor.ParseOrderBy(graph, variableMap, qEhelper, (OrderByNode)parsedClauses[Parser.Clause.OrderBy], exprInfo, variableMap.GetCount());
var orderByProc = OrderByResultProcessor.Factory(this.exprInfo, comps, qEhelper, variableMap.GetCount(), this.exprInfo.CollectUsedVariables());
match.PassResultProcessor(orderByProc);
}
else
{
// Check if the query is aggregation and not a simple query.
if ((this.exprInfo.Aggregates.Count == 0 && this.qEhelper.IsSetSingleGroupGroupBy) || (!this.qEhelper.IsSetSingleGroupGroupBy && !parsedClauses.ContainsKey(Parser.Clause.GroupBy)))
{
throw new ArgumentException($"{this.GetType()}, no grouping or sorting was specified. The Streamed version allows to compute only aggregations. If you want to use only patter matching, please change to mode Normal.");
}
var groupByProc = GroupByResultProcessor.Factory(exprInfo, qEhelper, variableMap.GetCount(), this.exprInfo.CollectUsedVariables(), isStreamed);
match.PassResultProcessor(groupByProc);
}
query.AddToEnd(match);
}
/// <summary>
/// Builds a query from an input.
/// Cannot contain order by and group by simultaneously.
/// Called by static Create method.
/// </summary>
/// <param name="tokens"> An input query. </param>
/// <param name="graph"> A graph to compute the query on. </param>
/// <param name="threadCount"> Maximum number of threads available to use. </param>
/// <param name="printer"> A printer to use. </param>
/// <param name="formater"> A formater to use by printer. </param>
/// <param name="verticesPerThread"> A number of vertices distributed to threads during parallel computation of the query.</param>
/// <param name="fileName"> A file to store results into if set, otherwise null. </param>
/// <param name="allowPrint"> Whether to allow printing. Mainly for benchmarking purposes.</param>
/// <param name="fixedArraySize"> The size of the arrays used for storing results of the matcher.</param>
/// <param name="grouperAlias"> A grouper to use when specifying group by.</param>
/// <param name="sorterAlias"> A sorter to use when specifying order by.</param>
private Query(List <Token> tokens, Graph graph, bool allowPrint, int threadCount, PrinterType printer, FormaterType formater, int verticesPerThread, string fileName, GrouperAlias grouperAlias, SorterAlias sorterAlias, int fixedArraySize)
{
this.graph = graph;
this.variableMap = new VariableMap();
this.qEhelper = new QueryExecutionHelper(threadCount, printer, formater, verticesPerThread, fixedArraySize, fileName, "DFSParallel", "DFSSingleThread", "SIMPLE", grouperAlias, sorterAlias);
// Parse input query.
var parsedClauses = Parser.Parse(tokens);
// Create execution chain.
if (parsedClauses.ContainsKey(Parser.Clause.GroupBy) && parsedClauses.ContainsKey(Parser.Clause.OrderBy))
{
throw new ArgumentException($"{this.GetType()}, the query cannot contain both order by and group by");
}
QueryObject groupBy = null;
QueryObject orderBy = null;
// MATCH is always leaf.
QueryObject match = QueryObject.Factory
(typeof(MatchObject), graph, qEhelper, variableMap, parsedClauses[Parser.Clause.Match], null);
// Second must be group by because it defines what can be in other clauses.
// GROUP BY
if (parsedClauses.ContainsKey(Parser.Clause.GroupBy))
{
this.exprInfo = new QueryExpressionInfo(true);
groupBy = QueryObject.Factory(typeof(GroupByObject), graph, qEhelper, variableMap, parsedClauses[Parser.Clause.GroupBy], exprInfo);
}
else
{
this.exprInfo = new QueryExpressionInfo(false);
}
// SELECT is the last one to process the resuls.
this.query = QueryObject.Factory
(typeof(SelectObject), graph, qEhelper, variableMap, parsedClauses[Parser.Clause.Select], exprInfo);
((SelectObject)this.query).allowPrint = allowPrint;
// Check if the results are in a single group.
this.SetSingleGroupFlags();
// ORDER BY
if (parsedClauses.ContainsKey(Parser.Clause.OrderBy))
{
orderBy = QueryObject.Factory(typeof(OrderByObject), graph, qEhelper, variableMap, parsedClauses[Parser.Clause.OrderBy], exprInfo);
query.AddToEnd(orderBy);
}
// If the single group by is set, add GroupBy object to the execution chain.
if (this.qEhelper.IsSetSingleGroupGroupBy && !this.qEhelper.IsSetGroupBy)
{
groupBy = QueryObject.Factory(typeof(GroupByObject), null, qEhelper, null, null, exprInfo);
}
if (groupBy != null)
{
query.AddToEnd(groupBy);
}
query.AddToEnd(match);
query.PassStoringVariables(this.exprInfo.CollectUsedVariables());
}
/// <summary>
/// Main algorith.
/// Infinite loop when user inputs queries which are subsequently computed and results are printed into the defined output.
/// </summary>
/// <param name="args"> Program arguments.</param>
/// <param name="reader"> Reader from which to read input.</param>
private static void Run(string[] args, TextReader reader)
{
if (args.Length < 7)
{
throw new ArgumentException("Wrong number of program parameters.");
}
QueryMode mode = GetMode(args[0]);
GrouperAlias gAlias = GetGrouperAlias(args[1]);
SorterAlias sAlias = GetSorterAlias(args[2]);
int fixedArraySize = GetFixedArraySize(args[3]);
Query.CheckAliases(gAlias, sAlias, mode);
int ThreadCount = GetThreadCount(args[4]);
PrinterType Printer = GetPrinter(args[5]);
FormaterType Formater = GetFormater(args[6]);
int VerticesPerThread = GetVerticesPerhread(ThreadCount, args);
string FileName = GetFileName(ThreadCount, Printer, args);
// Set only if on a desktop machine
using (Process p = Process.GetCurrentProcess())
p.PriorityClass = ProcessPriorityClass.RealTime; //High;
// Set a number of threads in the thread pool that will be immediatelly spawned on demand,
// without the need to wait.
if (ThreadCount != 1)
{
ThreadPool.SetMinThreads(ThreadCount, 0);
}
// Load the graph.
Graph graph = new Graph();
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
Console.Clear();
// Main loop of a program.
// Program awaits users input query and computes it.
// After computation, the user is promted to choose whether he wants to compute another query or close the app.
while (true)
{
Console.WriteLine("Enter Query:");
try
{
Console.WriteLine();
Query query = Query.Create(mode, reader, graph, ThreadCount, Printer, Formater, VerticesPerThread, FileName, gAlias, sAlias, fixedArraySize, true);
Console.WriteLine();
stopwatch.Start();
query.Compute();
stopwatch.Stop();
PrintElapsedTime("Completed with");
stopwatch.Reset();
Console.WriteLine("Finished the computation of the query.");
if (!ContinueWithAnotherQuery())
{
return;
}
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
}
catch (Exception e)
{
Console.WriteLine(e);
if (!ContinueWithAnotherQuery())
{
return;
}
stopwatch.Reset();
}
}
}
