public static bool Bug11782_Aggregate()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11782_Aggregate.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
//test Aggregate()
var c = pt1.Select(x => x).Aggregate((x, y) => x + y);
//test AggregateAsQuery()
var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
DoWhile <T>(this IQueryable <T> source,
Func <IQueryable <T>, IQueryable <T> > body,
Func <IQueryable <T>, IQueryable <T>, IQueryable <bool> > cond,
Int32 count)
{
if (count < 0)
{
throw new ArgumentOutOfRangeException("count");
}
if (count == 0)
{
return(source);
}
IQueryable <T> before = source;
while (true)
{
IQueryable <T> after = before;
for (int i = 0; i < count; i++)
{
after = body(after);
}
var more = cond(before, after);
DryadLinqQueryable.SubmitAndWait(after, more);
if (!more.Single())
{
return(after);
}
before = after;
}
}
public static bool CopyPlainDataViaToStoreMaterialize()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
foreach (int x in q)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> query = pt1.Select(x => 100 + x);
DryadLinqQueryable.Submit(query); //materialize // throws
foreach (int x in query)
{
//Console.WriteLine(x);
}
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static void Test2(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore <LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var q1 = input.Where(x => x.Line.Contains("white"));
var q2 = input.Where(x => x.Line.Contains("the"));
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
DryadLinqQueryable.SubmitAndWait(res1, res2);
}
public static bool Bug_16341_SubmitThrowsForDifferentContexts()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
var context2 = new DryadLinqContext(Config.cluster);
context2.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.Submit(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt2 = simple2.Select(x => x.First());
DryadLinqQueryable.SubmitAndWait(pt1, pt2);
passed &= false;
}
catch (DryadLinqException)
{
}
return(passed);
}
public static bool Bug14010_AlreadyDisposedContext(DryadLinqContext context)
{
string testName = "Bug14010_AlreadyDisposedContext";
TestLog.TestStart(testName);
bool passed = true;
context.LocalDebug = false;
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
ctx.Dispose();
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
int output = pt1.Select(x => x).First();
passed &= false;
}
catch (Exception)
{
passed &= true;
}
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
ctx.Dispose();
int output = pt1.Select(x => x).First();
passed &= false;
}
catch (Exception)
{
passed &= true;
}
try
{
DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
ctx.Dispose();
IQueryable <int> query = pt1.Select(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "abc"), true);
var info = DryadLinqQueryable.Submit(query);
passed &= false;
}
catch (Exception)
{
passed &= true;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool RepeatMaterialize()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/RepeatMaterialize.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
DryadLinqJobInfo info2 = null;
try
{
info1 = DryadLinqQueryable.Submit(new[] { q }); //materialize
info2 = DryadLinqQueryable.Submit(new[] { q }); //materialize // does not throw
if (!context.LocalDebug)
{
passed &= false;
}
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
if (info2 != null)
{
info2.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
DoWhile <T>(this IQueryable <T> source,
Func <IQueryable <T>, IQueryable <T> > body,
Func <IQueryable <T>, IQueryable <T>, IQueryable <bool> > cond)
{
IQueryable <T> before = source;
while (true)
{
IQueryable <T> after = body(before);
var more = cond(before, after);
DryadLinqQueryable.SubmitAndWait(after, more);
if (!more.Single())
{
return(after);
}
before = after;
}
}
public static bool MaterializeToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile_a = "unittest/output/MaterializeToStoreTerminated_a.txt";
string outFile_b = "unittest/output/MaterializeToStoreTerminated_b.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> query = pt1.Select(x => 100 + x);
var q1 = query.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_a), true); //stream name w/o prefixed slash
var q2 = query.Where(x => true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_b), true); //stream name w/ prefixed slash
DryadLinqQueryable.Submit(q1, q2); //materialize // throws
var __unused2 = q1.Select(x => x); // Legal call, but BLOCKS
foreach (int x in q2)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_a);
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_b);
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeMentionsSameQueryTwice() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/MaterializeMentionsSameQueryTwice.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info1 = null;
try
{
info1 = DryadLinqQueryable.Submit(q, q); //materialize // throws
passed &= false; // for Config.cluster execution, second materialize should throw;
}
catch (ArgumentException)
{
passed &= true;
}
//wait for any jobs to complete.
if (info1 != null)
{
info1.Wait();
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug_16341_VariousTestsForSubmit()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
int[] data = new[] { 1, 2, 3 };
var badQ1 = data.AsQueryable().Select(x => 100 + x);
var badQ2 = data.AsQueryable().Select(x => 100 + x);
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> goodQ1 = simple.Select(x => x.First());
IQueryable <LineRecord> input_copy = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple_copy = input_copy.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> goodQ2 = simple_copy.Select(x => x.First());
try
{
badQ1.Submit();
passed &= false; // "should throw as input isn't a L2H query"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.Submit((IQueryable)null); //this-Query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentException)
{
//although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
//hence we throw ArgumentException rather than ArgumentNullException.
}
try
{
DryadLinqQueryable.Submit((IQueryable[])null); //multi-query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentNullException)
{
}
try
{
DryadLinqQueryable.Submit(goodQ1, null); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is null"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.Submit(goodQ1, badQ1); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is not a L2H"
}
catch (ArgumentException)
{
}
//----------
// same tests again for SubmitAndWait
try
{
badQ1.SubmitAndWait();
passed &= false; // "should throw as input isn't a L2H query"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait((IQueryable)null); //this-Query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentException)
{
//although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
//hence we throw ArgumentException rather than ArgumentNullException.
}
try
{
DryadLinqQueryable.SubmitAndWait((IQueryable[])null); //multi-query overload
passed &= false; // "should throw ArgNull as input is null"
}
catch (ArgumentNullException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait(goodQ1, null); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is null"
}
catch (ArgumentException)
{
}
try
{
DryadLinqQueryable.SubmitAndWait(goodQ1, badQ1); //multi-query overload
passed &= false; // "should throw ArgEx as one of the inputs is not a L2H"
}
catch (ArgumentException)
{
}
}
catch (DryadLinqException)
{
}
return(passed);
}
