public static bool SubmitNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x);
var q2 = q1.Where(x => true);
q2.SubmitAndWait(); // throws here
var outPT = q2.ToList();
foreach (int x in outPT)
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool template()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/x.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool DistributiveSelect_1()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
// this result selector satisfies "DistributiveOverConcat"
int[] aggregates = pt1.GroupBy(x => x % 2).Select(group => group.Sum()).ToArray();
int[] expected = new[] { 1 + 3 + 5 + 7 + 9 + 11, 2 + 4 + 6 + 8 + 10 + 12 };
//note the order of the result elements is not guaranteed, so order them before testing
int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
int[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Aggregate_WithCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11782_LowLevelQueryableManipulation()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
Expression lambda = Expression.Lambda <Func <int, int> >(
Expression.Constant(1),
new[] { Expression.Parameter(typeof(int), "x") });
var z = pt1.Provider.CreateQuery(
Expression.Call(
typeof(Queryable), "Select",
new Type[] { pt1.ElementType, pt1.ElementType },
pt1.Expression, Expression.Quote(lambda)));
passed &= false; // the use of non-generic Provider.CreateQuery() should have thrown
}
catch (DryadLinqException)
{
passed &= true;
}
return(passed);
}
public static bool AssumeRangePartition()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/BasicAPITests_AssumeRangePartition.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q =
pt1
.AssumeRangePartition(x => x, false)
.Select(x => 100 + x).ToStore(outFile);
var info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> query = pt1.Select(x => 100 + x);
DryadLinqQueryable.Submit(query); //materialize // throws
foreach (int x in query)
{
//Console.WriteLine(x);
}
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11782_Aggregate()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11782_Aggregate.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
//test Aggregate()
var c = pt1.Select(x => x).Aggregate((x, y) => x + y);
//test AggregateAsQuery()
var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool HomomorphicUnaryApply()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/HomomorphicUnaryApply.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
var jobInfo = q1.ToStore <int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool ToStoreGetEnumeratorThrows() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> q1 = pt1.Select(x => 100 + x);
var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
output.GetEnumerator();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool BuiltInCountIsDistributable()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
// Built in Count is Distributable as built-in logic knows to use Sum() as the combiner function.
// Count(a,b,c,d) = Sum(Count(a,b), Count(c,d))
int[] aggregates = pt1.GroupBy(x => x % 2, (key, seq) => seq.Count()).ToArray();
int[] expected = new[] { 6, 6 }; // six elements in each full group.
//note the order of the result elements is not guaranteed, so order them before testing
int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
int[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool CopyPlainDataViaToStoreMaterialize()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
foreach (int x in q)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool EnumeratePlainData()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
foreach (int x in pt1) // throws
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool ToStoreSubmitGetEnumerator() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreSubmitGetEnumerator.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x).HashPartition(x => x);
var q2 = q1.Where(x => true);
IQueryable <int> output = q2.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = output.SubmitAndWait();
foreach (int x in output) // should not run a new dryad job.
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GetEnumeratorNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> q1 = pt1.Select(x => 100 + x);
IQueryable <int> q2 = q1.Where(x => true);
foreach (int x in q2) // throws here
{
//Console.WriteLine(x);
}
//@TODO: perform a sequence-equals test.
//IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
//DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
// "unittest/output/test2.txt")).SubmitAndWait();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug_16341_SubmitThrowsForDifferentContexts()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
var context2 = new DryadLinqContext(Config.cluster);
context2.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.Submit(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.SubmitAndWait(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
return(passed);
}
/*
* public static bool PlainEnumerableAsDryadQueryToStoreSubmit()
* {
* var context = new DryadLinqContext(Config.cluster);
* context.LocalExecution = false;
* bool passed = true;
* try
* {
* string outFile = "unittest/output/PlainEnumerableAsDryadQueryToStoreSubmit.txt";
*
* int[] plainData = { 5, 6, 7 };
*
* var q = context.AsDryadQuery(plainData, CompressionScheme.None).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile);
* DryadLinqJobInfo info = q.Submit();
* info.Wait();
*
* foreach (int x in q)
* {
* //Console.WriteLine(x);
* }
*
* passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
* }
* catch (DryadLinqException e)
* {
* passed &= false;
* }
* return passed;
* }
*/
public static bool RepeatSubmit()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/RepeatSubmit.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
DryadLinqJobInfo info2 = null;
try
{
info1 = q.Submit();
info2 = q.Submit(); // does not throw
if (!context.LocalDebug)
{
passed &= false;
}
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
if (info2 != null)
{
info2.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile_a = "unittest/output/MaterializeToStoreTerminated_a.txt";
string outFile_b = "unittest/output/MaterializeToStoreTerminated_b.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> query = pt1.Select(x => 100 + x);
var q1 = query.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_a), true); //stream name w/o prefixed slash
var q2 = query.Where(x => true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_b), true); //stream name w/ prefixed slash
DryadLinqQueryable.Submit(q1, q2); //materialize // throws
var __unused2 = q1.Select(x => x); // Legal call, but BLOCKS
foreach (int x in q2)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_a);
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_b);
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeMentionsSameQueryTwice() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/MaterializeMentionsSameQueryTwice.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
try
{
info1 = DryadLinqQueryable.Submit(q, q); //materialize // throws
passed &= false; // for Config.cluster execution, second materialize should throw;
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool QueryOnDataBackedDLQ()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/QueryOnDataBackedDLQ.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.Select(x => 100 + x);
var outPT = q.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
outPT.Submit();
var outPT2_dummy_notUsed = outPT.Select(x => x); //BLOCKS HERE until the input is concrete
// source.Expression returns an expression for the backingDataDLQ
// CheckAndInitialize() on the backingData will block.
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
foreach (int x in outPT)
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11781_CountandFirstOrDefault()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11781.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
//Test Count()
var c = pt1.Count();
//Test CountAsQuery()
var q = pt1.CountAsQuery().ToStore(outFile);
DryadLinqJobInfo info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
// Also test FirstOrDefault
// the affected code for dlq.Execute() also has a branch for FirstOrDefault() and friends.
int y = pt1.FirstOrDefault();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11638_LongMethods()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11638_LongMethods.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q =
pt1
.LongSelect((x, i) => x)
.LongWhere((x, i) => true)
.LongSelectMany((x, i) => new[] { x })
.LongSelectMany((x, i) => new[] { x }, (i, seq) => seq) //overload#2
.LongTakeWhile((x, i) => true)
.LongSkipWhile((x, i) => false)
.ToStore(outFile);
var info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug_16341_VariousTestsForSubmit()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
int[] data = new[] { 1, 2, 3 };
var badQ1 = data.AsQueryable().Select(x => 100 + x);
var badQ2 = data.AsQueryable().Select(x => 100 + x);
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> goodQ1 = simple.Select(x => x.First());
IQueryable <LineRecord> input_copy = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple_copy = input_copy.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> goodQ2 = simple_copy.Select(x => x.First());
try
{
badQ1.Submit();
passed &= false; // "should throw as input isn't a L2H query"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.Submit((IQueryable)null); //this-Query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentException)
{
//although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
//hence we throw ArgumentException rather than ArgumentNullException.
}
try
{
DryadLinqQueryable.Submit((IQueryable[])null); //multi-query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentNullException)
{
}
try
{
DryadLinqQueryable.Submit(goodQ1, null); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is null"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.Submit(goodQ1, badQ1); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is not a L2H"
}
catch (ArgumentException)
{
}
//----------
// same tests again for SubmitAndWait
try
{
badQ1.SubmitAndWait();
passed &= false; // "should throw as input isn't a L2H query"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait((IQueryable)null); //this-Query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentException)
{
//although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
//hence we throw ArgumentException rather than ArgumentNullException.
}
try
{
DryadLinqQueryable.SubmitAndWait((IQueryable[])null); //multi-query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentNullException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait(goodQ1, null); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is null"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait(goodQ1, badQ1); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is not a L2H"
}
catch (ArgumentException)
{
}
}
catch (DryadLinqException)
{
}
return(passed);
}
