public static bool GroupByReduce_ResultSelector_ComplexNewExpression()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
var aggregates = data.GroupBy(x => 0, (key, seq) => new KeyValuePair <int, KeyValuePair <double, double> >(key, new KeyValuePair <double, double>(seq.Average(), seq.Average()))).ToArray();
var expected = new KeyValuePair <int, KeyValuePair <double, double> >[] { new KeyValuePair <int, KeyValuePair <double, double> >(0, new KeyValuePair <double, double>(100.5, 100.5)) };
passed &= aggregates.SequenceEqual(expected);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_BuiltIn_First()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
int[] aggregates = data
.GroupBy(x => 0, (key, seq) => seq.First())
.ToArray();
// the output of First can be the first item of either partition.
passed &= aggregates.SequenceEqual(new[] { 1 }) || aggregates.SequenceEqual(new[] { 101 });
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduceWithCustomDecomposableFunction_NonDistributableCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
double[] aggregates = data
.Select(x => (double)x)
.GroupBy(x => 0, (key, seq) => DecomposableFunc4(seq)).ToArray();
double[] expected = new[] { Enumerable.Range(1, 200).Sum() / 100.0 };
//note the order of the result elements is not guaranteed, so order them before testing
double[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
double[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_BitwiseNegationOperator()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => new KeyValuePair <int, int>(k, ~g.Sum())).ToArray();
//local sort.. so that keys are in order.
var resultsSorted = results.OrderBy(list => list.Key).ToArray();
//key0: count = 6, sum = 42
//key1: count = 6, sum = 36
passed &= (resultsSorted[0].Key == 0); // "incorrect results.1"
passed &= (resultsSorted[0].Value == ~42); // "incorrect results.2"
passed &= (resultsSorted[1].Key == 1); // "incorrect results.3"
passed &= (resultsSorted[1].Value == ~36); // "incorrect results.4"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_SameDecomposableUsedTwice()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => MyFunc(k, DecomposableFunc5(g), DecomposableFunc5(g), g.Average())).ToArray();
//key0: count = 6, av = av(2,4,6,8,10,12) = 7
//key1: count = 6, av = av(1,3,5,7,9,11) = 6
//local sort.. so that keys are in order.
var results_sorted = results.OrderBy(x => x.Key).ToArray();
passed &= (results_sorted.Length == 2); // "wrong results"
passed &= (results_sorted[0].Key == 0); // "wrong results"
passed &= (results_sorted[0].A == 6); // "wrong results"
passed &= (results_sorted[0].B == 6); // "wrong results"
passed &= (results_sorted[0].Av == 7.0); // "wrong results"
passed &= (results_sorted[1].Key == 1); // "wrong results"
passed &= (results_sorted[1].A == 6); // "wrong results"
passed &= (results_sorted[1].B == 6); // "wrong results"
passed &= (results_sorted[1].Av == 6.0); // "wrong results"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_CustomListInitializerReducer()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => new MultiParamInitializerClass()
{
{ k, g.Count(), g.Sum() }, // one item, comprising three components
}).ToArray();
//local sort.. so that keys are in order.
var resultsSorted = results.OrderBy(list => list.Key).ToArray();
//key0: count = 6, sum = 42
//key1: count = 6, sum = 36
passed &= (resultsSorted[0].Key == 0); // "incorrect results.1"
passed &= (resultsSorted[0].Count() == 6); // "incorrect results.2"
passed &= (resultsSorted[0].Sum() == 42); // "incorrect results.3"
passed &= (resultsSorted[1].Key == 1); // "incorrect results.4"
passed &= (resultsSorted[1].Count() == 6); // "incorrect results.5"
passed &= (resultsSorted[1].Sum() == 36); // "incorrect results.6"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool template()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
//passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
internal static bool GroupByReduce_APIMisuse()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
if (context.LocalDebug)
{
// "decomposition logic doesn't run in LocalDebug.. skipping";
return(true);
}
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
// internal-visibility decomposable type should fail.
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable1(g)).ToArray();
passed &= false; // "exception should be thrown"
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeMustBePublic); // "error code is wrong"
}
// decomposable type doesn't implement IDecomposable or IDecomposableRecursive
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable2(g)).ToArray();
passed &= false; //"exception should be thrown");
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeDoesNotImplementInterface);
}
// decomposable type implements more than one IDecomposable or IDecomposableRecursive
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable3(g)).ToArray();
passed &= false;
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeImplementsTooManyInterfaces);
}
// decomposable type doesn't have public default ctor
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable4(g)).ToArray();
passed &= false;
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeDoesNotHavePublicDefaultCtor);
}
// decomposable type input type doesn't match
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable5(g)).ToArray();
passed &= false;
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypesDoNotMatch);
}
// decomposable type output type doesn't match
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable6(g)).ToArray();
passed &= false;
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypesDoNotMatch);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}