public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
{
string testName = "Bug15371_NoDataMembersForSerialization";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable <NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var result = context.FromStore <NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
passed &= false;
}
}
catch (DryadLinqException Ex)
{
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Aggregate_WithCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool CopyPlainDataViaToStoreMaterialize()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
foreach (int x in q)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool EnumeratePlainData()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
foreach (int x in pt1) // throws
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> query = pt1.Select(x => 100 + x);
DryadLinqQueryable.Submit(query); //materialize // throws
foreach (int x in query)
{
//Console.WriteLine(x);
}
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool SubmitNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x);
var q2 = q1.Where(x => true);
q2.SubmitAndWait(); // throws here
var outPT = q2.ToList();
foreach (int x in outPT)
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool ToStoreGetEnumeratorThrows() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> q1 = pt1.Select(x => 100 + x);
var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
output.GetEnumerator();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11782_LowLevelQueryableManipulation()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
Expression lambda = Expression.Lambda <Func <int, int> >(
Expression.Constant(1),
new[] { Expression.Parameter(typeof(int), "x") });
var z = pt1.Provider.CreateQuery(
Expression.Call(
typeof(Queryable), "Select",
new Type[] { pt1.ElementType, pt1.ElementType },
pt1.Expression, Expression.Quote(lambda)));
passed &= false; // the use of non-generic Provider.CreateQuery() should have thrown
}
catch (DryadLinqException)
{
passed &= true;
}
return(passed);
}
public static bool GroupByReduce_ResultSelector_ComplexNewExpression()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
var aggregates = data.GroupBy(x => 0, (key, seq) => new KeyValuePair <int, KeyValuePair <double, double> >(key, new KeyValuePair <double, double>(seq.Average(), seq.Average()))).ToArray();
var expected = new KeyValuePair <int, KeyValuePair <double, double> >[] { new KeyValuePair <int, KeyValuePair <double, double> >(0, new KeyValuePair <double, double>(100.5, 100.5)) };
passed &= aggregates.SequenceEqual(expected);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduceWithCustomDecomposableFunction_NonDistributableCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
double[] aggregates = data
.Select(x => (double)x)
.GroupBy(x => 0, (key, seq) => DecomposableFunc4(seq)).ToArray();
double[] expected = new[] { Enumerable.Range(1, 200).Sum() / 100.0 };
//note the order of the result elements is not guaranteed, so order them before testing
double[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
double[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_BuiltIn_First()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
int[] aggregates = data
.GroupBy(x => 0, (key, seq) => seq.First())
.ToArray();
// the output of First can be the first item of either partition.
passed &= aggregates.SequenceEqual(new[] { 1 }) || aggregates.SequenceEqual(new[] { 101 });
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool BuiltInCountIsDistributable()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
// Built in Count is Distributable as built-in logic knows to use Sum() as the combiner function.
// Count(a,b,c,d) = Sum(Count(a,b), Count(c,d))
int[] aggregates = pt1.GroupBy(x => x % 2, (key, seq) => seq.Count()).ToArray();
int[] expected = new[] { 6, 6 }; // six elements in each full group.
//note the order of the result elements is not guaranteed, so order them before testing
int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
int[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_BitwiseNegationOperator()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => new KeyValuePair <int, int>(k, ~g.Sum())).ToArray();
//local sort.. so that keys are in order.
var resultsSorted = results.OrderBy(list => list.Key).ToArray();
//key0: count = 6, sum = 42
//key1: count = 6, sum = 36
passed &= (resultsSorted[0].Key == 0); // "incorrect results.1"
passed &= (resultsSorted[0].Value == ~42); // "incorrect results.2"
passed &= (resultsSorted[1].Key == 1); // "incorrect results.3"
passed &= (resultsSorted[1].Value == ~36); // "incorrect results.4"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool template()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/x.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool ToStoreSubmitGetEnumerator() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreSubmitGetEnumerator.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x).HashPartition(x => x);
var q2 = q1.Where(x => true);
IQueryable <int> output = q2.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = output.SubmitAndWait();
foreach (int x in output) // should not run a new dryad job.
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool HomomorphicUnaryApply()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/HomomorphicUnaryApply.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
var jobInfo = q1.ToStore <int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool AssumeRangePartition()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/BasicAPITests_AssumeRangePartition.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q =
pt1
.AssumeRangePartition(x => x, false)
.Select(x => 100 + x).ToStore(outFile);
var info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool DistributiveSelect_1()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
// this result selector satisfies "DistributiveOverConcat"
int[] aggregates = pt1.GroupBy(x => x % 2).Select(group => group.Sum()).ToArray();
int[] expected = new[] { 1 + 3 + 5 + 7 + 9 + 11, 2 + 4 + 6 + 8 + 10 + 12 };
//note the order of the result elements is not guaranteed, so order them before testing
int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
int[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11782_Aggregate()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11782_Aggregate.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
//test Aggregate()
var c = pt1.Select(x => x).Aggregate((x, y) => x + y);
//test AggregateAsQuery()
var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GetEnumeratorNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> q1 = pt1.Select(x => 100 + x);
IQueryable <int> q2 = q1.Where(x => true);
foreach (int x in q2) // throws here
{
//Console.WriteLine(x);
}
//@TODO: perform a sequence-equals test.
//IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
//DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
// "unittest/output/test2.txt")).SubmitAndWait();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool FullHomomorphicBinaryApply_DifferentDataSets(DryadLinqContext context)
{
string testName = "FullHomomorphicBinaryApply_DifferentDataSets";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets";
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable <int> pt2 = DataGenerator.GetSimpleFileSets(context);
IQueryable <int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
var jobInfo = q1.ToStore <int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
passed &= Validate.outFileExists(outFile);
result[0] = q1;
}
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable <int> pt2 = DataGenerator.GetSimpleFileSets(context);
IQueryable <int> q1 = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
result[1] = q1;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug12584_HashPartitionOutputCount(DryadLinqContext context) // ToDo
{
string testName = "Bug12584_HashPartitionOutputCount";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug12584_HashPartitionOutputCount";
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.HashPartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
info.Wait();
//// partition verification are only valid for cluster execution..
//// check that nOutputPartitions == nInputPartitions.
//// Note: this is today's behavior, but we don't strictly guarantee this and the rules may change.
//var inFS = context.DscService.GetFileSet(DataGenerators.SIMPLE_FILESET_NAME);
//var inPCount = inFS.GetFiles().Count();
//var outFS = context.DscService.GetFileSet("DevUnitTest/Bug12584_out");
//var outPCount = outFS.GetFiles().Count();
//passed &= TestUtils.Assert(outPCount == inPCount, "Output nPartitions should be equal to input nPartitions.");
}
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool RP_singlePartition_autoSeparators(DryadLinqContext context)
{
string testName = "RP_singlePartition_autoSeparators";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_singlePartition_autoSeparators";
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable <int> pt2 = pt1.Apply(x => x) // force a merge
.RangePartition(x => x)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 1, false);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable <int> pt2 = pt1.Apply(x => x) // force a merge
.RangePartition(x => x);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug_16341_SubmitThrowsForDifferentContexts()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
var context2 = new DryadLinqContext(Config.cluster);
context2.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.Submit(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.SubmitAndWait(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
return(passed);
}
public static void Test5(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1);
Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "test", "testwrite");
IEnumerable <LineRecord> lines = DataProvider.ReadData <LineRecord>(context, dataUri);
foreach (var x in lines)
{
Console.WriteLine(x);
}
}
/*
* public static bool PlainEnumerableAsDryadQueryToStoreSubmit()
* {
* var context = new DryadLinqContext(Config.cluster);
* context.LocalExecution = false;
* bool passed = true;
* try
* {
* string outFile = "unittest/output/PlainEnumerableAsDryadQueryToStoreSubmit.txt";
*
* int[] plainData = { 5, 6, 7 };
*
* var q = context.AsDryadQuery(plainData, CompressionScheme.None).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile);
* DryadLinqJobInfo info = q.Submit();
* info.Wait();
*
* foreach (int x in q)
* {
* //Console.WriteLine(x);
* }
*
* passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
* }
* catch (DryadLinqException e)
* {
* passed &= false;
* }
* return passed;
* }
*/
public static bool RepeatSubmit()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/RepeatSubmit.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
DryadLinqJobInfo info2 = null;
try
{
info1 = q.Submit();
info2 = q.Submit(); // does not throw
if (!context.LocalDebug)
{
passed &= false;
}
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
if (info2 != null)
{
info2.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool RP_keySelector_rangeKeys_keyComparer_isDescending(DryadLinqContext context)
{
string testName = "RP_keySelector_rangeKeys_keyComparer_isDescending";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer_isDescending";
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable <int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer <int> .Default, true)
.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
var jobInfo = pt2.Submit();
jobInfo.Wait();
passed &= TestRangePartitioned(pt2, 4, true);
result[0] = pt2.ToList();
}
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
IQueryable <int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer <int> .Default, true);
result[1] = pt2.ToList();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug14010_AlreadyDisposedContext(DryadLinqContext context)
{
string testName = "Bug14010_AlreadyDisposedContext";
TestLog.TestStart(testName);
bool passed = true;
context.LocalDebug = false;
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
ctx.Dispose();
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
int output = pt1.Select(x => x).First();
passed &= false;
}
catch (Exception)
{
passed &= true;
}
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
ctx.Dispose();
int output = pt1.Select(x => x).First();
passed &= false;
}
catch (Exception)
{
passed &= true;
}
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
ctx.Dispose();
IQueryable <int> query = pt1.Select(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "abc"), true);
var info = DryadLinqQueryable.Submit(query);
passed &= false;
}
catch (Exception)
{
passed &= true;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool ToStoreThrowsForNonQuery()
{
bool passed = true;
try
{
int[] data = new[] { 1, 2, 3 };
var q1 = data.AsQueryable().Select(x => 100 + x).ToStore <int>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"dummy")).ToArray();
//Should throw as we got into DryadLinq via AsQueryable() rather than via context.
passed &= false;
}
catch (ArgumentException)
{
//expected
}
return(passed);
}
public static void WordCountExample()
{
#if local
// This overload runs the computation on your local computer using a single worker
var config = new DryadLinqContext(1);
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
// You can create inputs from any IEnumerable source using this method
var input = config.FromEnumerable(lines);
#else
string clusterName = "Replace with your HDInsight 3.0 cluster name";
// to use the davinci.txt example input below, select your cluster's default
// storage account and container, which automatically includes the sample text
string accountName = "Replace with a storage account name";
string containerName = "Replace with a storage container name";
// This overload creates an Azure-based computation
var config = new DryadLinqContext(clusterName);
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
// plain text files should be read as type LineRecord
var input = config.FromStore <LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
"example/data/gutenberg/davinci.txt"));
#endif
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair <string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
#if local
// any collection computed by the query can be materialized back at the client,
// not just the 'output' collection. For large collections this is expensive!
foreach (LineRecord line in toOutput)
{
Console.WriteLine(line.Line);
}
#else
// the 'true' parameter to ToStore means the output will be over-written if you run
// the job more than once
var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
"wc-out.txt"), true).SubmitAndWait();
#endif
}
