public static bool Bug11782_Aggregate() { var context = new DryadLinqContext(Config.cluster); context.LocalExecution = false; bool passed = true; try { string outFile = "unittest/output/Bug11782_Aggregate.out"; IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "unittest/inputdata/SimpleFile.txt")); IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable <int> pt1 = simple.Select(x => x.First()); //test Aggregate() var c = pt1.Select(x => x).Aggregate((x, y) => x + y); //test AggregateAsQuery() var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile); DryadLinqJobInfo info = DryadLinqQueryable.Submit(q); info.Wait(); passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); } catch (DryadLinqException) { passed &= false; } return(passed); }
public static bool CopyPlainDataViaToStoreMaterialize() { var context = new DryadLinqContext(Config.cluster); context.LocalExecution = false; bool passed = true; try { string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt"; IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, "unittest/inputdata/SimpleFile.txt")); IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable <int> pt1 = simple.Select(x => x.First()); var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true); DryadLinqJobInfo info = DryadLinqQueryable.Submit(q); info.Wait(); foreach (int x in q) { //Console.WriteLine(x); } passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); } catch (DryadLinqException) { passed &= false; } return(passed); }
/* * public static bool PlainEnumerableAsDryadQueryToStoreSubmit() * { * var context = new DryadLinqContext(Config.cluster); * context.LocalExecution = false; * bool passed = true; * try * { * string outFile = "unittest/output/PlainEnumerableAsDryadQueryToStoreSubmit.txt"; * * int[] plainData = { 5, 6, 7 }; * * var q = context.AsDryadQuery(plainData, CompressionScheme.None).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile); * DryadLinqJobInfo info = q.Submit(); * info.Wait(); * * foreach (int x in q) * { * //Console.WriteLine(x); * } * * passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); * } * catch (DryadLinqException e) * { * passed &= false; * } * return passed; * } */ public static bool RepeatSubmit() { var context = new DryadLinqContext(Config.cluster); context.LocalExecution = false; bool passed = true; try { string outFile = "unittest/output/RepeatSubmit.txt"; IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, "unittest/inputdata/SimpleFile.txt")); IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable <int> pt1 = simple.Select(x => x.First()); var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true); DryadLinqJobInfo info1 = null; DryadLinqJobInfo info2 = null; try { info1 = q.Submit(); info2 = q.Submit(); // does not throw if (!context.LocalDebug) { passed &= false; } } catch (ArgumentException) { passed &= true; } //wait for any jobs to complete. if (info1 != null) { info1.Wait(); } if (info2 != null) { info2.Wait(); } } catch (DryadLinqException) { passed &= false; } return(passed); }
public static bool MaterializeMentionsSameQueryTwice() // pass { var context = new DryadLinqContext(Config.cluster); context.LocalExecution = false; bool passed = true; try { string outFile = "unittest/output/MaterializeMentionsSameQueryTwice.txt"; IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, "unittest/inputdata/SimpleFile.txt")); IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable <int> pt1 = simple.Select(x => x.First()); var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true); DryadLinqJobInfo info1 = null; try { info1 = DryadLinqQueryable.Submit(q, q); //materialize // throws passed &= false; // for Config.cluster execution, second materialize should throw; } catch (ArgumentException) { passed &= true; } //wait for any jobs to complete. if (info1 != null) { info1.Wait(); } } catch (DryadLinqException) { passed &= false; } return(passed); }
public static bool Bug11781_CountandFirstOrDefault() { var context = new DryadLinqContext(Config.cluster); context.LocalExecution = false; bool passed = true; try { string outFile = "unittest/output/Bug11781.out"; IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "unittest/inputdata/SimpleFile.txt")); IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets()); IQueryable <int> pt1 = simple.Select(x => x.First()); //Test Count() var c = pt1.Count(); //Test CountAsQuery() var q = pt1.CountAsQuery().ToStore(outFile); DryadLinqJobInfo info = q.Submit(); info.Wait(); passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile); // Also test FirstOrDefault // the affected code for dlq.Execute() also has a branch for FirstOrDefault() and friends. int y = pt1.FirstOrDefault(); } catch (DryadLinqException) { passed &= false; } return(passed); }