コード例 #1
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool SubmitNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q1 = pt1.Select(x => 100 + x);
                var q2 = q1.Where(x => true);
                q2.SubmitAndWait(); // throws here
                var outPT = q2.ToList();
                foreach (int x in outPT)
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #2
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool template()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/x.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #3
0
        public static bool DistributiveSelect_1()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                // this result selector satisfies "DistributiveOverConcat"
                int[] aggregates = pt1.GroupBy(x => x % 2).Select(group => group.Sum()).ToArray();
                int[] expected   = new[] { 1 + 3 + 5 + 7 + 9 + 11, 2 + 4 + 6 + 8 + 10 + 12 };

                //note the order of the result elements is not guaranteed, so order them before testing
                int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                int[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #4
0
        public static bool Aggregate_WithCombiner()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));

                passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #5
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool Bug11782_LowLevelQueryableManipulation()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                Expression lambda = Expression.Lambda <Func <int, int> >(
                    Expression.Constant(1),
                    new[] { Expression.Parameter(typeof(int), "x") });
                var z = pt1.Provider.CreateQuery(
                    Expression.Call(
                        typeof(Queryable), "Select",
                        new Type[] { pt1.ElementType, pt1.ElementType },
                        pt1.Expression, Expression.Quote(lambda)));

                passed &= false; // the use of non-generic Provider.CreateQuery() should have thrown
            }
            catch (DryadLinqException)
            {
                passed &= true;
            }
            return(passed);
        }
コード例 #6
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool AssumeRangePartition()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/BasicAPITests_AssumeRangePartition.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q =
                    pt1
                    .AssumeRangePartition(x => x, false)
                    .Select(x => 100 + x).ToStore(outFile);
                var info = q.Submit();
                info.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #7
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool MaterializeNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1   = simple.Select(x => x.First());
                IQueryable <int> query = pt1.Select(x => 100 + x);

                DryadLinqQueryable.Submit(query); //materialize // throws

                foreach (int x in query)
                {
                    //Console.WriteLine(x);
                }

                //@TODO: assert that only one query execution occurred.
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #8
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool Bug11782_Aggregate()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/Bug11782_Aggregate.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                //test Aggregate()
                var c = pt1.Select(x => x).Aggregate((x, y) => x + y);

                //test AggregateAsQuery()
                var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
                DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
                info.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #9
0
        public static bool HomomorphicUnaryApply()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/HomomorphicUnaryApply.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q1      = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
                var jobInfo = q1.ToStore <int>(outFile).Submit();
                jobInfo.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #10
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool ToStoreGetEnumeratorThrows() // pass
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));
                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());

                IQueryable <int> pt1 = simple.Select(x => x.First());
                IQueryable <int> q1  = pt1.Select(x => 100 + x);

                var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
                output.GetEnumerator();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #11
0
        public static bool BuiltInCountIsDistributable()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                // Built in Count is Distributable as built-in logic knows to use Sum() as the combiner function.
                // Count(a,b,c,d) = Sum(Count(a,b), Count(c,d))
                int[] aggregates = pt1.GroupBy(x => x % 2, (key, seq) => seq.Count()).ToArray();
                int[] expected   = new[] { 6, 6 }; // six elements in each full group.

                //note the order of the result elements is not guaranteed, so order them before testing
                int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                int[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #12
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool CopyPlainDataViaToStoreMaterialize()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
                info.Wait();

                foreach (int x in q)
                {
                    //Console.WriteLine(x);
                }

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #13
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool EnumeratePlainData()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());
                foreach (int x in pt1) // throws
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #14
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool ToStoreSubmitGetEnumerator() // pass
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/ToStoreSubmitGetEnumerator.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());
                var q1 = pt1.Select(x => 100 + x).HashPartition(x => x);
                var q2 = q1.Where(x => true);
                IQueryable <int> output = q2.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info   = output.SubmitAndWait();

                foreach (int x in output) // should not run a new dryad job.
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #15
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool GetEnumeratorNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                IQueryable <int> q1 = pt1.Select(x => 100 + x);
                IQueryable <int> q2 = q1.Where(x => true);
                foreach (int x in q2) // throws here
                {
                    //Console.WriteLine(x);
                }
                //@TODO: perform a sequence-equals test.

                //IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
                //DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                //           "unittest/output/test2.txt")).SubmitAndWait();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #16
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool Bug_16341_SubmitThrowsForDifferentContexts()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            var context2 = new DryadLinqContext(Config.cluster);

            context2.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                       "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt2 = simple2.Select(x => x.First());

                DryadLinqQueryable.Submit(pt1, pt2);
                passed &= false;
            }
            catch (DryadLinqException)
            {
            }

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                       "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt2 = simple2.Select(x => x.First());

                DryadLinqQueryable.SubmitAndWait(pt1, pt2);
                passed &= false;
            }
            catch (DryadLinqException)
            {
            }

            return(passed);
        }
コード例 #17
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        /*
         * public static bool PlainEnumerableAsDryadQueryToStoreSubmit()
         * {
         *  var context = new DryadLinqContext(Config.cluster);
         *  context.LocalExecution = false;
         *  bool passed = true;
         *  try
         *  {
         *      string outFile = "unittest/output/PlainEnumerableAsDryadQueryToStoreSubmit.txt";
         *
         *      int[] plainData = { 5, 6, 7 };
         *
         *      var q = context.AsDryadQuery(plainData, CompressionScheme.None).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile);
         *      DryadLinqJobInfo info = q.Submit();
         *      info.Wait();
         *
         *      foreach (int x in q)
         *      {
         *          //Console.WriteLine(x);
         *      }
         *
         *      passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
         *  }
         *  catch (DryadLinqException e)
         *  {
         *      passed &= false;
         *  }
         *  return passed;
         * }
         */
        public static bool RepeatSubmit()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/RepeatSubmit.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());


                var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info1 = null;
                DryadLinqJobInfo info2 = null;
                try
                {
                    info1 = q.Submit();
                    info2 = q.Submit(); // does not throw

                    if (!context.LocalDebug)
                    {
                        passed &= false;
                    }
                }
                catch (ArgumentException)
                {
                    passed &= true;
                }

                //wait for any jobs to complete.
                if (info1 != null)
                {
                    info1.Wait();
                }

                if (info2 != null)
                {
                    info2.Wait();
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #18
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool MaterializeToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile_a = "unittest/output/MaterializeToStoreTerminated_a.txt";
                string outFile_b = "unittest/output/MaterializeToStoreTerminated_b.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1   = simple.Select(x => x.First());
                IQueryable <int> query = pt1.Select(x => 100 + x);

                var q1 = query.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_a), true);                  //stream name w/o prefixed slash

                var q2 = query.Where(x => true).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile_b), true); //stream name w/ prefixed slash

                DryadLinqQueryable.Submit(q1, q2);                                                                                                            //materialize  // throws

                var __unused2 = q1.Select(x => x);                                                                                                            // Legal call, but BLOCKS
                foreach (int x in q2)
                {
                    //Console.WriteLine(x);
                }

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_a);
                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile_b);

                //@TODO: assert that only one query execution occurred.
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #19
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool MaterializeMentionsSameQueryTwice() // pass
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/MaterializeMentionsSameQueryTwice.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info1 = null;
                try
                {
                    info1   = DryadLinqQueryable.Submit(q, q); //materialize // throws
                    passed &= false;                           // for Config.cluster execution, second materialize should throw;
                }
                catch (ArgumentException)
                {
                    passed &= true;
                }

                //wait for any jobs to complete.
                if (info1 != null)
                {
                    info1.Wait();
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #20
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool QueryOnDataBackedDLQ()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/QueryOnDataBackedDLQ.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());
                var q     = pt1.Select(x => 100 + x);
                var outPT = q.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
                outPT.Submit();

                var outPT2_dummy_notUsed = outPT.Select(x => x);  //BLOCKS HERE until the input is concrete
                // source.Expression returns an expression for the backingDataDLQ
                // CheckAndInitialize() on the backingData will block.

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);

                foreach (int x in outPT)
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #21
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool Bug11781_CountandFirstOrDefault()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/Bug11781.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                //Test Count()
                var c = pt1.Count();

                //Test CountAsQuery()
                var q = pt1.CountAsQuery().ToStore(outFile);
                DryadLinqJobInfo info = q.Submit();
                info.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);

                // Also test FirstOrDefault
                // the affected code for dlq.Execute() also has a branch for FirstOrDefault() and friends.
                int y = pt1.FirstOrDefault();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #22
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool Bug11638_LongMethods()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/Bug11638_LongMethods.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q =
                    pt1
                    .LongSelect((x, i) => x)
                    .LongWhere((x, i) => true)
                    .LongSelectMany((x, i) => new[] { x })
                    .LongSelectMany((x, i) => new[] { x }, (i, seq) => seq)  //overload#2
                    .LongTakeWhile((x, i) => true)
                    .LongSkipWhile((x, i) => false)
                    .ToStore(outFile);
                var info = q.Submit();
                info.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #23
0
ファイル: BasicAPITests.cs プロジェクト: animeshinvinci/Dryad
        public static bool Bug_16341_VariousTestsForSubmit()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                int[] data  = new[] { 1, 2, 3 };
                var   badQ1 = data.AsQueryable().Select(x => 100 + x);
                var   badQ2 = data.AsQueryable().Select(x => 100 + x);

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> goodQ1 = simple.Select(x => x.First());

                IQueryable <LineRecord> input_copy = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                          "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple_copy = input_copy.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> goodQ2 = simple_copy.Select(x => x.First());


                try
                {
                    badQ1.Submit();
                    passed &= false; // "should throw as input isn't a L2H query"
                }
                catch (ArgumentException)
                {
                }

                try
                {
                    DryadLinqQueryable.Submit((IQueryable)null); //this-Query overload
                    passed &= false;                             // "should throw ArgNull as input is null"
                }
                catch (ArgumentException)
                {
                    //although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
                    //hence we throw ArgumentException rather than ArgumentNullException.
                }

                try
                {
                    DryadLinqQueryable.Submit((IQueryable[])null); //multi-query overload
                    passed &= false;                               // "should throw ArgNull as input is null"
                }
                catch (ArgumentNullException)
                {
                }

                try
                {
                    DryadLinqQueryable.Submit(goodQ1, null); //multi-query overload
                    passed &= false;                         // "should throw ArgEx as one of the inputs is null"
                }
                catch (ArgumentException)
                {
                }

                try
                {
                    DryadLinqQueryable.Submit(goodQ1, badQ1); //multi-query overload
                    passed &= false;                          // "should throw ArgEx as one of the inputs is not a L2H"
                }
                catch (ArgumentException)
                {
                }

                //----------
                // same tests again for SubmitAndWait

                try
                {
                    badQ1.SubmitAndWait();
                    passed &= false; // "should throw as input isn't a L2H query"
                }
                catch (ArgumentException)
                {
                }

                try
                {
                    DryadLinqQueryable.SubmitAndWait((IQueryable)null); //this-Query overload
                    passed &= false;                                    // "should throw ArgNull as input is null"
                }
                catch (ArgumentException)
                {
                    //although we pass null, it goes to params[] overload which creates an actual array[1] containing one null
                    //hence we throw ArgumentException rather than ArgumentNullException.
                }

                try
                {
                    DryadLinqQueryable.SubmitAndWait((IQueryable[])null); //multi-query overload
                    passed &= false;                                      // "should throw ArgNull as input is null"
                }
                catch (ArgumentNullException)
                {
                }

                try
                {
                    DryadLinqQueryable.SubmitAndWait(goodQ1, null); //multi-query overload
                    passed &= false;                                // "should throw ArgEx as one of the inputs is null"
                }
                catch (ArgumentException)
                {
                }

                try
                {
                    DryadLinqQueryable.SubmitAndWait(goodQ1, badQ1); //multi-query overload
                    passed &= false;                                 // "should throw ArgEx as one of the inputs is not a L2H"
                }
                catch (ArgumentException)
                {
                }
            }
            catch (DryadLinqException)
            {
            }
            return(passed);
        }