public static bool Bug13637_LocalDebugProducingZeroRecords(DryadLinqContext context)
{
string testName = "Bug13637_LocalDebugProducingZeroRecords";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug13637_LocalDebugProducingZeroRecords";
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable <int> pt2 = pt1.Where(x => false).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var queryInfo2 = pt2.Submit();
queryInfo2.Wait();
int[] data2 = pt2.ToArray();
passed &= (data2.Length == 0);
// query producing no records -> anonymous output
IQueryable <int> pt3 = pt1.Where(x => false);
var queryInfo3 = pt3.Submit();
queryInfo3.Wait();
int[] data3 = pt3.ToArray();
passed &= (data3.Length == 0);
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug13534_HashPartitionNegIndexIsError(DryadLinqContext context)
{
string testName = "Bug13534_HashPartitionNegIndexIsError";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
pt1.HashPartition(x => x, 0); // exception should be thrown
passed &= false;
}
}
catch (Exception)
{
}
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
pt1.HashPartition(x => x, -1); // exception should be thrown
passed &= false;
}
}
catch (Exception)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool MaterializeMentionsSameQueryTwice() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/MaterializeMentionsSameQueryTwice.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
try
{
info1 = DryadLinqQueryable.Submit(q, q); //materialize // throws
passed &= false; // for Config.cluster execution, second materialize should throw;
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_SameDecomposableUsedTwice()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => MyFunc(k, DecomposableFunc5(g), DecomposableFunc5(g), g.Average())).ToArray();
//key0: count = 6, av = av(2,4,6,8,10,12) = 7
//key1: count = 6, av = av(1,3,5,7,9,11) = 6
//local sort.. so that keys are in order.
var results_sorted = results.OrderBy(x => x.Key).ToArray();
passed &= (results_sorted.Length == 2); // "wrong results"
passed &= (results_sorted[0].Key == 0); // "wrong results"
passed &= (results_sorted[0].A == 6); // "wrong results"
passed &= (results_sorted[0].B == 6); // "wrong results"
passed &= (results_sorted[0].Av == 7.0); // "wrong results"
passed &= (results_sorted[1].Key == 1); // "wrong results"
passed &= (results_sorted[1].A == 6); // "wrong results"
passed &= (results_sorted[1].B == 6); // "wrong results"
passed &= (results_sorted[1].Av == 6.0); // "wrong results"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByWithAnonymousTypes_MultipleAnonymousTypes(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_MultipleAnonymousTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = i % 10 })
.Where(x => true)
.Select(i => new { Num2 = i.Num })
.GroupBy(x => x.Num2, x => x.Num2)
.ToArray();
// local
context.LocalDebug = true;
IQueryable <int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = i % 10 })
.Where(x => true)
.Select(i => new { Num2 = i.Num })
.GroupBy(x => x.Num2, x => x.Num2)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool GroupByReduce_ListInitializerReducer()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => new List <int>()
{
k, g.Count(), g.Sum()
}).ToArray();
//local sort.. so that keys are in order.
var resultsSorted = results.OrderBy(list => list[0]).ToArray();
//key0: count = 6, sum = 42
//key1: count = 6, sum = 36
passed &= (resultsSorted[0][0] == 0); // "incorrect results.1"
passed &= (resultsSorted[0][1] == 6); // "incorrect results.2"
passed &= (resultsSorted[0][2] == 42); // "incorrect results.3"
passed &= (resultsSorted[1][0] == 1); // "incorrect results.4"
passed &= (resultsSorted[1][1] == 6); // "incorrect results.5"
passed &= (resultsSorted[1][2] == 36); // "incorrect results.6"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug14192_MultiApplySubExpressionReuse(DryadLinqContext context)
{
string testName = "Bug14192_MultiApplySubExpressionReuse";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug14192_MultiApplySubExpressionReuse";
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var data = pt1.Select(x => x);
var info = pt1.Apply(new[] { pt1, pt1 }, (sources) => new int[] { 1, 2, 3 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
info.Wait();
// ToDo
//string queryPlan = TestUtils.GetRecentQueryXmlAsText();
//int nVerticesInPlan = TestUtils.GetVertexStageCount(queryPlan);
//passed &= (nVerticesInPlan == 7); // "Only seven vertices should appear (before bug, there were 10 of which the last three were extraneous.");
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool template()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
//passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool QueryOnDataBackedDLQ()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/QueryOnDataBackedDLQ.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.Select(x => 100 + x);
var outPT = q.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
outPT.Submit();
var outPT2_dummy_notUsed = outPT.Select(x => x); //BLOCKS HERE until the input is concrete
// source.Expression returns an expression for the backingDataDLQ
// CheckAndInitialize() on the backingData will block.
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
foreach (int x in outPT)
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11781_CountandFirstOrDefault()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11781.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
//Test Count()
var c = pt1.Count();
//Test CountAsQuery()
var q = pt1.CountAsQuery().ToStore(outFile);
DryadLinqJobInfo info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
// Also test FirstOrDefault
// the affected code for dlq.Execute() also has a branch for FirstOrDefault() and friends.
int y = pt1.FirstOrDefault();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByWithAnonymousTypes_NestedAnonTypes(DryadLinqContext context)
{
string testName = "GroupByWithAnonymousTypes_NestedAnonTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var result1 = pt1.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
.ToArray();
// local
context.LocalDebug = true;
IQueryable <int> pt2 = DataGenerator.GetSimpleFileSets(context);
var result2 = pt2.Select(i => new { Num = new { NumInner = new int?(i % 10) } }) // nullable-fields present particular challenge.
.GroupBy(x => x.Num.NumInner, x => x.Num.NumInner)
.ToArray();
passed &= (result1.Count() == result2.Count());
passed &= (result1.Where(g => g.Key == 1).SelectMany(g => g).Count() == result2.Where(g => g.Key == 1).SelectMany(g => g).Count());
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug11638_LongMethods()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11638_LongMethods.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q =
pt1
.LongSelect((x, i) => x)
.LongWhere((x, i) => true)
.LongSelectMany((x, i) => new[] { x })
.LongSelectMany((x, i) => new[] { x }, (i, seq) => seq) //overload#2
.LongTakeWhile((x, i) => true)
.LongSkipWhile((x, i) => false)
.ToStore(outFile);
var info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug13736_IndexedTakeWhile(DryadLinqContext context)
{
string testName = "Bug13736_IndexedTakeWhile";
TestLog.TestStart(testName);
bool passed = true;
// dataset #1
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 6).ToArray();
result[1] = pt1.AsEnumerable().TakeWhile((x, i) => i < 6).ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
// dataset #2
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
var pt2 = pt1.TakeWhile((x, i) => i < 125).ToArray();
result[0] = pt1.AsEnumerable().TakeWhile((x, i) => i < 125).ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception)
{
passed &= false;
}
}
catch (Exception)
{
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug13529_and_Bug13593_IndexedOperatorCompilation(DryadLinqContext context)
{
string testName = "Bug13529_and_Bug13593_IndexedOperatorCompilation";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[0] = pt1.Select((x, i) => x)
.LongSelect((x, i) => x)
.LongSelectMany((x, i) => new[] { x })
.SelectMany((x, i) => new[] { x })
.Where((x, i) => true)
.LongWhere((x, i) => true)
.TakeWhile((x, i) => true)
.LongTakeWhile((x, i) => true)
.SkipWhile((x, i) => false)
.LongSkipWhile((x, i) => false)
.ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
result[1] = pt1.Select((x, i) => x)
.LongSelect((x, i) => x)
.LongSelectMany((x, i) => new[] { x })
.SelectMany((x, i) => new[] { x })
.Where((x, i) => true)
.LongWhere((x, i) => true)
.TakeWhile((x, i) => true)
.LongTakeWhile((x, i) => true)
.SkipWhile((x, i) => false)
.LongSkipWhile((x, i) => false)
.ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" SerializationTests ");
TestLog.Message(" **********************");
context.LocalDebug = false;
IQueryable <int> source = DataGenerator.GetSimpleFileSets(context);
var tests = new Dictionary <string, Action>()
{
{ "UDT_Undecorated", () =>
SerializationTests.TestUDT <UDT_Undecorated>(context, source, "Basic UDT with no attribute. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
{ "UDT_MarkedSerializable", () =>
SerializationTests.TestUDT <UDT_MarkedSerializable>(context, source, "Basic UDT which is marked as serializable.", null, 0) },
{ "UDT_StaticSerializer", () =>
SerializationTests.TestUDT <UDT_StaticSerializer>(context, source, "Basic UDT with old stype static serializers. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
{ "UDT_SelfCustomSerializer", () =>
SerializationTests.TestUDT <UDT_SelfCustomSerializer>(context, source, "UDT implementing custom serializer for itself.", null, 0) },
{ "UDT_ExternalCustomSerializer", () =>
SerializationTests.TestUDT <UDT_ExternalCustomSerializer>(context, source, "UDT with a custom serializer", null, 0) },
{ "UDT_BadCustomSerializerType1", () =>
SerializationTests.TestUDT <UDT_BadCustomSerializerType1>(context, source, "UDT decorated with an invalid custom serializer type.", typeof(DryadLinqException), 0) },
{ "UDT_BadCustomSerializerType2", () =>
SerializationTests.TestUDT <UDT_BadCustomSerializerType2>(context, source, "UDT decorated with another UDT's custom serializer.", typeof(DryadLinqException), 0) },
{ "UDT_BaseType", () =>
SerializationTests.TestUDT <UDT_BaseType>(context, source, "UDT with subtypes. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleSubtypes")) },
{ "UDT_DerivedType", () =>
SerializationTests.TestUDT <UDT_DerivedType>(context, source, "UDT with a base type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleDerivedtypes")) },
{ "UDT_FieldOfNonPublicType", () =>
SerializationTests.TestUDT <UDT_FieldOfNonPublicType>(context, source, "UDT with a field of a non-public type. Should be rejected", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("UDTHasFieldOfNonPublicType")) },
{ "UDT_Nested_OuterSerializableInnerNotSerializable", () =>
SerializationTests.TestUDT <UDT_Nested_OuterSerializableInnerNotSerializable>(context, source, "Nested UDT, outer type Serializable, inner type not Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
{ "UDT_Nested_OuterNotSerializableInnerSerializable", () =>
SerializationTests.TestUDT <UDT_Nested_OuterNotSerializableInnerSerializable>(context, source, "Nested UDT, outer type not Serializable, inner type Serializable. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeNotSerializable")) },
{ "UDT_Nested_InnerAndOuterSerializable", () =>
SerializationTests.TestUDT <UDT_Nested_InnerAndOuterSerializable>(context, source, "Nested UDT, both outer and innter types marked Serializable.", null, 0) },
{ "UDT_Nested_InnerEnum_InnerAndOuterSerializable", () =>
SerializationTests.TestUDT <UDT_Nested_InnerEnum_InnerAndOuterSerializable>(context, source, "Nested UDT, inner enum, both outer and innter types marked Serializable.", null, 0) },
{ "int?", () =>
SerializationTests.TestUDT <int?>(context, source, "Nullable type as the UDT. Should be autoserialized.", null, 0) },
{ "UDT_FirstLevelCircular", () =>
SerializationTests.TestUDT <UDT_FirstLevelCircular>(context, source, "Circular UDT. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes")) },
{ "UDT_FirstLevelCircularArrayRef", () =>
SerializationTests.TestUDT <UDT_FirstLevelCircularArrayRef>(context, source, "Circular UDT with array reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes")) },
{ "UDT_SecondLevelCircular", () =>
SerializationTests.TestUDT <UDT_SecondLevelCircular>(context, source, "Circular UDT with indirect reference to self. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleCircularTypes")) },
{ "UDT_CircularTypeWithCustomSerializer", () =>
SerializationTests.TestUDT <UDT_CircularTypeWithCustomSerializer>(context, source, "Circular UDT with a custom serializer. Should not be rejected.", null, 0) },
{ "UDT_TypeContainingCustomSerializedCircularType", () =>
SerializationTests.TestUDT <UDT_TypeContainingCustomSerializedCircularType>(context, source, "An UDT containing a field of a circular UDT with a custom serializer. Should be autoserialized.", null, 0) },
{ "UDT_ObjectField", () =>
SerializationTests.TestUDT <UDT_ObjectField>(context, source, "UDT with a field of type System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields")) },
{ "UDT_ObjectArrayField", () =>
SerializationTests.TestUDT <UDT_ObjectArrayField>(context, source, "UDT with a field of type System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields")) },
{ "UDT_ObjectListField", () =>
SerializationTests.TestUDT <UDT_ObjectListField>(context, source, "UDT with a field of type List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields")) },
{ "object", () =>
SerializationTests.TestUDT <object>(context, source, "UDT = System.Object. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields")) },
{ "object[]", () =>
SerializationTests.TestUDT <object[]>(context, source, "UDT = System.Object[]. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields")) },
{ "List<object>", () =>
SerializationTests.TestUDT <List <object> >(context, source, "UDT = List<object>. Should not be autoserialized.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("CannotHandleObjectFields")) },
{ "UDT_EmptyType", () =>
SerializationTests.TestUDT <UDT_EmptyType>(context, source, "Empty UDT. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")) },
{ "UDT_ObjectFieldAndCustomSerializer", () =>
SerializationTests.TestUDT <UDT_ObjectFieldAndCustomSerializer>(context, source, "UDT with an Object field, and a custom serializer. Should not be rejected.", null, 0) },
{ "UDT_EmptyTypeWithCustomSerializer", () =>
SerializationTests.TestUDT <UDT_EmptyTypeWithCustomSerializer>(context, source, "Empty UDT with a custom serializer. Should be rejected.", typeof(DryadLinqException), ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")) },
{ "UDT_PrivateFieldOfPublicType", () =>
SerializationTests.TestUDT <UDT_PrivateFieldOfPublicType>(context, source, "UDT with a private field of a public type. Should be autoserialized.", null, 0) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static bool RP_rangeSeparators_customComparer(DryadLinqContext context)
{
string testName = "RP_rangeSeparators_customComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//these keys should be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should also be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should work
IQueryable <int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 5, new WeirdIntComparer(), false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
string outFile = "unittest/output/RP_rangeSeparators_customComparer";
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//these keys should be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should also be considered not-sorted
try
{
var results = pt1.RangePartition(x => x, new int[] { 4, 3, 2, 1 }, new WeirdIntComparer(), false).ToArray();
passed &= false; // "an exception should have been thrown (non-sorted separators)."
}
catch (ArgumentException)
{
//expected
}
//these keys should work
IQueryable <int> pt2 = pt1.RangePartition(x => x, new int[] { 6, 6, 3, 1 }, new WeirdIntComparer(), false)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool RP_keySelector_rangeKeys(DryadLinqContext context)
{
string testName = "RP_keySelector_rangeKeys";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
IEnumerable <int>[] result2 = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_rangeKeys_1";
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//increasing ranges keys.
IQueryable <int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 4, false);
result[0] = pt2.ToList();
string outFile2 = "unittest/output/RP_keySelector_rangeKeys_2";
IQueryable <int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
//decreasing ranges keys.
IQueryable <int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile2), true);
var jobInfo2 = pt2_2.Submit();
jobInfo2.Wait();
passed &= TestRangePartitioned(pt2_2, 4, true);
result2[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
//increasing ranges keys.
IQueryable <int> pt2 = pt1.RangePartition(x => x, new[] { 2, 5, 8 });
result[1] = pt2.ToList();
IQueryable <int> pt1_2 = DataGenerator.GetRangePartitionDataSet(context);
//decreasing ranges keys.
IQueryable <int> pt2_2 = pt1_2.RangePartition(x => x, new[] { 8, 5, 2 });
result2[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
Validate.Check(result2);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
internal static bool GroupByReduce_APIMisuse()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
if (context.LocalDebug)
{
// "decomposition logic doesn't run in LocalDebug.. skipping";
return(true);
}
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
// internal-visibility decomposable type should fail.
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable1(g)).ToArray();
passed &= false; // "exception should be thrown"
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeMustBePublic); // "error code is wrong"
}
// decomposable type doesn't implement IDecomposable or IDecomposableRecursive
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable2(g)).ToArray();
passed &= false; //"exception should be thrown");
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeDoesNotImplementInterface);
}
// decomposable type implements more than one IDecomposable or IDecomposableRecursive
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable3(g)).ToArray();
passed &= false;
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeImplementsTooManyInterfaces);
}
// decomposable type doesn't have public default ctor
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable4(g)).ToArray();
passed &= false;
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeDoesNotHavePublicDefaultCtor);
}
// decomposable type input type doesn't match
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable5(g)).ToArray();
passed &= false;
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypesDoNotMatch);
}
// decomposable type output type doesn't match
try
{
pt1.GroupBy(x => x, (k, g) => BadDecomposable6(g)).ToArray();
passed &= false;
}
catch (DryadLinqException)
{
//??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypesDoNotMatch);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug_16341_VariousTestsForSubmit()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
int[] data = new[] { 1, 2, 3 };
var badQ1 = data.AsQueryable().Select(x => 100 + x);
var badQ2 = data.AsQueryable().Select(x => 100 + x);
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> goodQ1 = simple.Select(x => x.First());
IQueryable <LineRecord> input_copy = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple_copy = input_copy.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> goodQ2 = simple_copy.Select(x => x.First());
try
{
badQ1.Submit();
passed &= false; // "should throw as input isn't a L2H query"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.Submit((IQueryable)null); //this-Query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentException)
{
//although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
//hence we throw ArgumentException rather than ArgumentNullException.
}
try
{
DryadLinqQueryable.Submit((IQueryable[])null); //multi-query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentNullException)
{
}
try
{
DryadLinqQueryable.Submit(goodQ1, null); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is null"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.Submit(goodQ1, badQ1); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is not a L2H"
}
catch (ArgumentException)
{
}
//----------
// same tests again for SubmitAndWait
try
{
badQ1.SubmitAndWait();
passed &= false; // "should throw as input isn't a L2H query"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait((IQueryable)null); //this-Query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentException)
{
//although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
//hence we throw ArgumentException rather than ArgumentNullException.
}
try
{
DryadLinqQueryable.SubmitAndWait((IQueryable[])null); //multi-query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentNullException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait(goodQ1, null); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is null"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait(goodQ1, badQ1); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is not a L2H"
}
catch (ArgumentException)
{
}
}
catch (DryadLinqException)
{
}
return(passed);
}