public static bool Bug_16341_SubmitThrowsForDifferentContexts()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
var context2 = new DryadLinqContext(Config.cluster);
context2.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.Submit(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.SubmitAndWait(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
return(passed);
}
public static bool CopyPlainDataViaToStoreMaterialize()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
foreach (int x in q)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool EnumeratePlainData()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
foreach (int x in pt1) // throws
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> query = pt1.Select(x => 100 + x);
DryadLinqQueryable.Submit(query); //materialize // throws
foreach (int x in query)
{
//Console.WriteLine(x);
}
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool SubmitNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x);
var q2 = q1.Where(x => true);
q2.SubmitAndWait(); // throws here
var outPT = q2.ToList();
foreach (int x in outPT)
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool template()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/x.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool ToStoreGetEnumeratorThrows() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> q1 = pt1.Select(x => 100 + x);
var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
output.GetEnumerator();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
{
string testName = "Bug15371_NoDataMembersForSerialization";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable <NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var result = context.FromStore <NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
passed &= false;
}
}
catch (DryadLinqException Ex)
{
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Aggregate_WithCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool AssumeRangePartition()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/BasicAPITests_AssumeRangePartition.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q =
pt1
.AssumeRangePartition(x => x, false)
.Select(x => 100 + x).ToStore(outFile);
var info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11782_LowLevelQueryableManipulation()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
Expression lambda = Expression.Lambda <Func <int, int> >(
Expression.Constant(1),
new[] { Expression.Parameter(typeof(int), "x") });
var z = pt1.Provider.CreateQuery(
Expression.Call(
typeof(Queryable), "Select",
new Type[] { pt1.ElementType, pt1.ElementType },
pt1.Expression, Expression.Quote(lambda)));
passed &= false; // the use of non-generic Provider.CreateQuery() should have thrown
}
catch (DryadLinqException)
{
passed &= true;
}
return(passed);
}
public static bool ToStoreSubmitGetEnumerator() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreSubmitGetEnumerator.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x).HashPartition(x => x);
var q2 = q1.Where(x => true);
IQueryable <int> output = q2.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = output.SubmitAndWait();
foreach (int x in output) // should not run a new dryad job.
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11782_Aggregate()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11782_Aggregate.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
//test Aggregate()
var c = pt1.Select(x => x).Aggregate((x, y) => x + y);
//test AggregateAsQuery()
var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool BuiltInCountIsDistributable()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
// Built in Count is Distributable as built-in logic knows to use Sum() as the combiner function.
// Count(a,b,c,d) = Sum(Count(a,b), Count(c,d))
int[] aggregates = pt1.GroupBy(x => x % 2, (key, seq) => seq.Count()).ToArray();
int[] expected = new[] { 6, 6 }; // six elements in each full group.
//note the order of the result elements is not guaranteed, so order them before testing
int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
int[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GetEnumeratorNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> q1 = pt1.Select(x => 100 + x);
IQueryable <int> q2 = q1.Where(x => true);
foreach (int x in q2) // throws here
{
//Console.WriteLine(x);
}
//@TODO: perform a sequence-equals test.
//IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
//DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
// "unittest/output/test2.txt")).SubmitAndWait();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GetEnumeratorNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> q1 = pt1.Select(x => 100 + x);
IQueryable<int> q2 = q1.Where(x => true);
foreach (int x in q2) // throws here
{
//Console.WriteLine(x);
}
//@TODO: perform a sequence-equals test.
//IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
//DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
// "unittest/output/test2.txt")).SubmitAndWait();
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool ToStoreGetEnumeratorThrows() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> q1 = pt1.Select(x => 100 + x);
var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
output.GetEnumerator();
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool GroupByReduce_ResultSelector_ComplexNewExpression()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
var aggregates = data.GroupBy(x => 0, (key, seq) => new KeyValuePair <int, KeyValuePair <double, double> >(key, new KeyValuePair <double, double>(seq.Average(), seq.Average()))).ToArray();
var expected = new KeyValuePair <int, KeyValuePair <double, double> >[] { new KeyValuePair <int, KeyValuePair <double, double> >(0, new KeyValuePair <double, double>(100.5, 100.5)) };
passed &= aggregates.SequenceEqual(expected);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool DistributiveSelect_1()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
// this result selector satisfies "DistributiveOverConcat"
int[] aggregates = pt1.GroupBy(x => x % 2).Select(group => group.Sum()).ToArray();
int[] expected = new[] { 1 + 3 + 5 + 7 + 9 + 11, 2 + 4 + 6 + 8 + 10 + 12 };
//note the order of the result elements is not guaranteed, so order them before testing
int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
int[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduceWithCustomDecomposableFunction_NonDistributableCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
double[] aggregates = data
.Select(x => (double)x)
.GroupBy(x => 0, (key, seq) => DecomposableFunc4(seq)).ToArray();
double[] expected = new[] { Enumerable.Range(1, 200).Sum() / 100.0 };
//note the order of the result elements is not guaranteed, so order them before testing
double[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
double[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_BuiltIn_First()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
int[] aggregates = data
.GroupBy(x => 0, (key, seq) => seq.First())
.ToArray();
// the output of First can be the first item of either partition.
passed &= aggregates.SequenceEqual(new[] { 1 }) || aggregates.SequenceEqual(new[] { 101 });
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_BitwiseNegationOperator()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => new KeyValuePair <int, int>(k, ~g.Sum())).ToArray();
//local sort.. so that keys are in order.
var resultsSorted = results.OrderBy(list => list.Key).ToArray();
//key0: count = 6, sum = 42
//key1: count = 6, sum = 36
passed &= (resultsSorted[0].Key == 0); // "incorrect results.1"
passed &= (resultsSorted[0].Value == ~42); // "incorrect results.2"
passed &= (resultsSorted[1].Key == 1); // "incorrect results.3"
passed &= (resultsSorted[1].Value == ~36); // "incorrect results.4"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool HomomorphicUnaryApply()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/HomomorphicUnaryApply.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
var jobInfo = q1.ToStore <int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static void Test4(string[] args)
{
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
var input = context.FromStore <LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
var lines = input.Where(x => x.Line.Contains("white"));
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
result.SubmitAndWait();
}
public static void Test1(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var lines = input.Where(x => x.Line.Contains("white"));
foreach (var x in lines) Console.WriteLine(x);
//var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
//result.SubmitAndWait();
}
public static void Test2(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var q1 = input.Where(x => x.Line.Contains("white"));
var q2 = input.Where(x => x.Line.Contains("the"));
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
DryadLinqQueryable.SubmitAndWait(res1, res2);
}
public static void Test3(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
result.SubmitAndWait();
}
public static void Test2(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore <LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var q1 = input.Where(x => x.Line.Contains("white"));
var q2 = input.Where(x => x.Line.Contains("the"));
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
DryadLinqQueryable.SubmitAndWait(res1, res2);
}
/*
* public static bool PlainEnumerableAsDryadQueryToStoreSubmit()
* {
* var context = new DryadLinqContext(Config.cluster);
* context.LocalExecution = false;
* bool passed = true;
* try
* {
* string outFile = "unittest/output/PlainEnumerableAsDryadQueryToStoreSubmit.txt";
*
* int[] plainData = { 5, 6, 7 };
*
* var q = context.AsDryadQuery(plainData, CompressionScheme.None).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile);
* DryadLinqJobInfo info = q.Submit();
* info.Wait();
*
* foreach (int x in q)
* {
* //Console.WriteLine(x);
* }
*
* passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
* }
* catch (DryadLinqException e)
* {
* passed &= false;
* }
* return passed;
* }
*/
public static bool RepeatSubmit()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/RepeatSubmit.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
DryadLinqJobInfo info2 = null;
try
{
info1 = q.Submit();
info2 = q.Submit(); // does not throw
if (!context.LocalDebug)
{
passed &= false;
}
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
if (info2 != null)
{
info2.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static void Test3(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore <LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair <string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
result.SubmitAndWait();
}
public static void Test1(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
var input = context.FromStore <LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var lines = input.Where(x => x.Line.Contains("white"));
foreach (var x in lines)
{
Console.WriteLine(x);
}
//var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
//result.SubmitAndWait();
}
public static void WordCountExample()
{
#if local
// This overload runs the computation on your local computer using a single worker
var config = new DryadLinqContext(1);
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
// You can create inputs from any IEnumerable source using this method
var input = config.FromEnumerable(lines);
#else
string clusterName = "Replace with your HDInsight 3.0 cluster name";
// to use the davinci.txt example input below, select your cluster's default
// storage account and container, which automatically includes the sample text
string accountName = "Replace with a storage account name";
string containerName = "Replace with a storage container name";
// This overload creates an Azure-based computation
var config = new DryadLinqContext(clusterName);
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
// plain text files should be read as type LineRecord
var input = config.FromStore<LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
"example/data/gutenberg/davinci.txt"));
#endif
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
#if local
// any collection computed by the query can be materialized back at the client,
// not just the 'output' collection. For large collections this is expensive!
foreach (LineRecord line in toOutput)
{
Console.WriteLine(line.Line);
}
#else
// the 'true' parameter to ToStore means the output will be over-written if you run
// the job more than once
var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
"wc-out.txt"), true).SubmitAndWait();
#endif
}
public static void WordCountExample()
{
#if local
// This overload runs the computation on your local computer using a single worker
var config = new DryadLinqContext(1);
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
// You can create inputs from any IEnumerable source using this method
var input = config.FromEnumerable(lines);
#else
string clusterName = "Replace with your HDInsight 3.0 cluster name";
// to use the davinci.txt example input below, select your cluster's default
// storage account and container, which automatically includes the sample text
string accountName = "Replace with a storage account name";
string containerName = "Replace with a storage container name";
// This overload creates an Azure-based computation
var config = new DryadLinqContext(clusterName);
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
// plain text files should be read as type LineRecord
var input = config.FromStore <LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
"example/data/gutenberg/davinci.txt"));
#endif
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair <string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
#if local
// any collection computed by the query can be materialized back at the client,
// not just the 'output' collection. For large collections this is expensive!
foreach (LineRecord line in toOutput)
{
Console.WriteLine(line.Line);
}
#else
// the 'true' parameter to ToStore means the output will be over-written if you run
// the job more than once
var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
"wc-out.txt"), true).SubmitAndWait();
#endif
}
public static bool GroupByReduce_ProgrammingManualExample()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string filesetName = "DevUnitTest/0to999integers";
Utils.DeleteFile(Config.accountName, Config.storageKey, Config.containerName, filesetName, true);
IEnumerable <IEnumerable <int> > rawdata = new[] { Enumerable.Range(0, 334), Enumerable.Range(334, 333), Enumerable.Range(667, 333) };
// ??? DscIngressHelpers.AsDryadQueryPartitions(context, rawdata, filesetName, DscCompressionScheme.None);
var data = context.FromStore <int>(filesetName);
var count = data.AsEnumerable().Count();
var sum = data.AsEnumerable().Sum();
var min = data.AsEnumerable().Min();
var max = data.AsEnumerable().Max();
var uniques = data.AsEnumerable().Distinct().Count();
//Console.WriteLine("DATA:: count:{0} uniques:{1} sum:{2}, min:{3}, max:{4}", count, uniques, sum, min, max);
// ???
//var results = data
// .GroupBy(x => x % 10, (key, seq) => new KeyValuePair<int, double>(key, seq.MyAverage()))
// .OrderBy(y => y.Key)
// .ToArray();
////foreach (var result in results)
//// Console.WriteLine("For group {0} the average is {1}", result.Key, result.Value);
//passed &= (results.Count() == 10);
//passed &= (results[0].Key == 0); // "first element should be key=0");
//passed &= (results[0].Value == 495); // "first element should be value=495 ie avg(0,10,20,..,990)");
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile_a = "unittest/output/MaterializeToStoreTerminated_a.txt";
string outFile_b = "unittest/output/MaterializeToStoreTerminated_b.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> query = pt1.Select(x => 100 + x);
var q1 = query.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_a), true); //stream name w/o prefixed slash
var q2 = query.Where(x => true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_b), true); //stream name w/ prefixed slash
DryadLinqQueryable.Submit(q1, q2); //materialize // throws
var __unused2 = q1.Select(x => x); // Legal call, but BLOCKS
foreach (int x in q2)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_a);
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_b);
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduce_SameDecomposableUsedTwice()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => MyFunc(k, DecomposableFunc5(g), DecomposableFunc5(g), g.Average())).ToArray();
//key0: count = 6, av = av(2,4,6,8,10,12) = 7
//key1: count = 6, av = av(1,3,5,7,9,11) = 6
//local sort.. so that keys are in order.
var results_sorted = results.OrderBy(x => x.Key).ToArray();
passed &= (results_sorted.Length == 2); // "wrong results"
passed &= (results_sorted[0].Key == 0); // "wrong results"
passed &= (results_sorted[0].A == 6); // "wrong results"
passed &= (results_sorted[0].B == 6); // "wrong results"
passed &= (results_sorted[0].Av == 7.0); // "wrong results"
passed &= (results_sorted[1].Key == 1); // "wrong results"
passed &= (results_sorted[1].A == 6); // "wrong results"
passed &= (results_sorted[1].B == 6); // "wrong results"
passed &= (results_sorted[1].Av == 6.0); // "wrong results"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool SubmitNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x);
var q2 = q1.Where(x => true);
q2.SubmitAndWait(); // throws here
var outPT = q2.ToList();
foreach (int x in outPT)
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static void Test4(string[] args)
{
DryadLinqContext context = new DryadLinqContext("svc-d2-01");
var input = context.FromStore<LineRecord>("hdfs://svc-d2-01:8033/user/misard/foo.txt");
var lines = input.Where(x => x.Line.Contains("white"));
var result = lines.ToStore("hdfs://svc-d2-01:8033/user/yuanbyu/foo.txt", true);
result.SubmitAndWait();
}
public static void WordCountExample()
{
#if local
// This overload runs the computation on your local computer using a single worker
var config = new DryadLinqContext(1);
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
// You can create inputs from any IEnumerable source using this method
var input = config.FromEnumerable(lines);
#else
#if azure
string clusterName = "Replace with your HDInsight 3.1 cluster name";
// to use the davinci.txt example input below, select your cluster's default
// storage account and container, which automatically includes the sample text
string accountName = "Replace with a storage account name";
string containerName = "Replace with a storage container name";
// This overload creates an Azure-based computation
var config = new DryadLinqContext(clusterName);
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
// plain text files should be read as type LineRecord
var input = config.FromStore<LineRecord>(Utils.ToAzureUri(accountName, containerName,
"example/data/gutenberg/davinci.txt"));
#else
// to use a yarn cluster, fill in the username, resource node machine name and port, and name node and hdfs port below (use -1 for the default hdfs port).
string user = "******";
string resourceNode = "Replace with the name of the computer your resource node is running on";
int rmPort = 8088;
string nameNode = "Replace with the name of the computer your name node is running on";
int hdfsPort = -1;
// set the YARN queue to submit your job on below. Leave null to use the default queue
string queue = null;
// set the number of worker containers to start for the DryadLINQ job below
int numberOfWorkers = 2;
// set the amount of memory requested for the DryadLINQ job manager container below: 8GB should be enough for even the largest jobs, and 2GB will normally suffice
int amMemoryMB = 2000;
// set the amount of memory requested for the DryadLINQ worker containers below. The amount needed will depend on the code you are running
int workerMemoryMB = 8000;
// This overload runs the computation on your local computer using a single worker
var cluster = new DryadLinqYarnCluster(user, numberOfWorkers, amMemoryMB, workerMemoryMB, queue, resourceNode, rmPort, nameNode, hdfsPort);
var config = new DryadLinqContext(cluster);
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
// You can create inputs from any IEnumerable source using this method
var input = config.FromEnumerable(lines);
#endif
#endif
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
#if azure
// the 'true' parameter to ToStore means the output will be over-written if you run
// the job more than once
var info = toOutput.ToStore(Utils.ToAzureUri(accountName, containerName,
"wc-out.txt"), true).SubmitAndWait();
#else
// any collection computed by the query can be materialized back at the client,
// not just the 'output' collection. For large collections this is expensive!
foreach (LineRecord line in toOutput)
{
Console.WriteLine(line.Line);
}
#endif
}
public static bool RepeatMaterialize()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/RepeatMaterialize.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
DryadLinqJobInfo info2 = null;
try
{
info1 = DryadLinqQueryable.Submit(new[] { q }); //materialize
info2 = DryadLinqQueryable.Submit(new[] { q }); //materialize // does not throw
if (!context.LocalDebug)
{
passed &= false;
}
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
if (info2 != null)
{
info2.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool Bug_16341_VariousTestsForSubmit()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
int[] data = new[] { 1, 2, 3 };
var badQ1 = data.AsQueryable().Select(x => 100 + x);
var badQ2 = data.AsQueryable().Select(x => 100 + x);
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> goodQ1 = simple.Select(x => x.First());
IQueryable<LineRecord> input_copy = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple_copy = input_copy.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> goodQ2 = simple_copy.Select(x => x.First());
try
{
badQ1.Submit();
passed &= false; // "should throw as input isn't a L2H query"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.Submit((IQueryable)null); //this-Query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentException)
{
//although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
//hence we throw ArgumentException rather than ArgumentNullException.
}
try
{
DryadLinqQueryable.Submit((IQueryable[])null); //multi-query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentNullException)
{
}
try
{
DryadLinqQueryable.Submit(goodQ1, null); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is null"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.Submit(goodQ1, badQ1); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is not a L2H"
}
catch (ArgumentException)
{
}
//----------
// same tests again for SubmitAndWait
try
{
badQ1.SubmitAndWait();
passed &= false; // "should throw as input isn't a L2H query"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait((IQueryable)null); //this-Query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentException)
{
//although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
//hence we throw ArgumentException rather than ArgumentNullException.
}
try
{
DryadLinqQueryable.SubmitAndWait((IQueryable[])null); //multi-query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentNullException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait(goodQ1, null); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is null"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait(goodQ1, badQ1); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is not a L2H"
}
catch (ArgumentException)
{
}
}
catch (DryadLinqException)
{
}
return passed;
}
public static bool EnumeratePlainData()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
foreach (int x in pt1) // throws
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool Bug11781_CountandFirstOrDefault()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11781.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
//Test Count()
var c = pt1.Count();
//Test CountAsQuery()
var q = pt1.CountAsQuery().ToStore(outFile);
DryadLinqJobInfo info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
// Also test FirstOrDefault
// the affected code for dlq.Execute() also has a branch for FirstOrDefault() and friends.
int y = pt1.FirstOrDefault();
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool MaterializeMentionsSameQueryTwice() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/MaterializeMentionsSameQueryTwice.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
try
{
info1 = DryadLinqQueryable.Submit(q, q); //materialize // throws
passed &= false; // for Config.cluster execution, second materialize should throw;
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
{
string testName = "Bug15371_NoDataMembersForSerialization";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var result = context.FromStore<NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
passed &= false;
}
}
catch (DryadLinqException Ex)
{
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug11782_LowLevelQueryableManipulation()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
Expression lambda = Expression.Lambda<Func<int, int>>(
Expression.Constant(1),
new[] { Expression.Parameter(typeof(int), "x") });
var z = pt1.Provider.CreateQuery(
Expression.Call(
typeof(Queryable), "Select",
new Type[] { pt1.ElementType, pt1.ElementType },
pt1.Expression, Expression.Quote(lambda)));
passed &= false; // the use of non-generic Provider.CreateQuery() should have thrown
}
catch (DryadLinqException)
{
passed &= true;
}
return passed;
}
public static bool Bug11782_Aggregate()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11782_Aggregate.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
//test Aggregate()
var c = pt1.Select(x => x).Aggregate((x, y) => x + y);
//test AggregateAsQuery()
var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool template()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/x.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool CopyPlainDataViaToStoreMaterialize()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
foreach (int x in q)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool Bug_16341_SubmitThrowsForDifferentContexts()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
var context2 = new DryadLinqContext(Config.cluster);
context2.LocalExecution = false;
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<LineRecord> input2 = context2.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.Submit(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<LineRecord> input2 = context2.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.SubmitAndWait(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
return passed;
}
public static bool AssumeRangePartition()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/BasicAPITests_AssumeRangePartition.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q =
pt1
.AssumeRangePartition(x => x, false)
.Select(x => 100 + x).ToStore(outFile);
var info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool QueryOnDataBackedDLQ()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/QueryOnDataBackedDLQ.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q = pt1.Select(x => 100 + x);
var outPT = q.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
outPT.Submit();
var outPT2_dummy_notUsed = outPT.Select(x => x); //BLOCKS HERE until the input is concrete
// source.Expression returns an expression for the backingDataDLQ
// CheckAndInitialize() on the backingData will block.
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
foreach (int x in outPT)
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool Bug11638_LongMethods()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11638_LongMethods.out";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q =
pt1
.LongSelect((x, i) => x)
.LongWhere((x, i) => true)
.LongSelectMany((x, i) => new[] { x })
.LongSelectMany((x, i) => new[] { x }, (i, seq) => seq) //overload#2
.LongTakeWhile((x, i) => true)
.LongSkipWhile((x, i) => false)
.ToStore(outFile);
var info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool ToStoreSubmitGetEnumerator() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreSubmitGetEnumerator.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x).HashPartition(x => x);
var q2 = q1.Where(x => true);
IQueryable<int> output = q2.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = output.SubmitAndWait();
foreach (int x in output) // should not run a new dryad job.
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool MaterializeToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile_a = "unittest/output/MaterializeToStoreTerminated_a.txt";
string outFile_b = "unittest/output/MaterializeToStoreTerminated_b.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> query = pt1.Select(x => 100 + x);
var q1 = query.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_a), true); //stream name w/o prefixed slash
var q2 = query.Where(x => true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_b), true); //stream name w/ prefixed slash
DryadLinqQueryable.Submit(q1, q2); //materialize // throws
var __unused2 = q1.Select(x => x); // Legal call, but BLOCKS
foreach (int x in q2)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_a);
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_b);
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool MaterializeNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> query = pt1.Select(x => 100 + x);
DryadLinqQueryable.Submit(query); //materialize // throws
foreach (int x in query)
{
//Console.WriteLine(x);
}
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
