Example #1
0
        public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
        {
            string testName = "Bug15371_NoDataMembersForSerialization";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";

                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    IQueryable <NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
                    var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
                    jobInfo.Wait();
                    var result = context.FromStore <NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
                    passed &= false;
                }
            }
            catch (DryadLinqException Ex)
            {
                passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
                           Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Example #2
0
        public static bool Aggregate_WithCombiner()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));

                passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #3
0
        public static bool CopyPlainDataViaToStoreMaterialize()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
                info.Wait();

                foreach (int x in q)
                {
                    //Console.WriteLine(x);
                }

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #4
0
        public static bool EnumeratePlainData()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());
                foreach (int x in pt1) // throws
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #5
0
        public static bool MaterializeNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1   = simple.Select(x => x.First());
                IQueryable <int> query = pt1.Select(x => 100 + x);

                DryadLinqQueryable.Submit(query); //materialize // throws

                foreach (int x in query)
                {
                    //Console.WriteLine(x);
                }

                //@TODO: assert that only one query execution occurred.
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #6
0
        public static bool SubmitNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q1 = pt1.Select(x => 100 + x);
                var q2 = q1.Where(x => true);
                q2.SubmitAndWait(); // throws here
                var outPT = q2.ToList();
                foreach (int x in outPT)
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #7
0
        public static bool ToStoreGetEnumeratorThrows() // pass
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));
                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());

                IQueryable <int> pt1 = simple.Select(x => x.First());
                IQueryable <int> q1  = pt1.Select(x => 100 + x);

                var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
                output.GetEnumerator();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #8
0
        public static bool Bug11782_LowLevelQueryableManipulation()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                Expression lambda = Expression.Lambda <Func <int, int> >(
                    Expression.Constant(1),
                    new[] { Expression.Parameter(typeof(int), "x") });
                var z = pt1.Provider.CreateQuery(
                    Expression.Call(
                        typeof(Queryable), "Select",
                        new Type[] { pt1.ElementType, pt1.ElementType },
                        pt1.Expression, Expression.Quote(lambda)));

                passed &= false; // the use of non-generic Provider.CreateQuery() should have thrown
            }
            catch (DryadLinqException)
            {
                passed &= true;
            }
            return(passed);
        }
Example #9
0
        public static bool GroupByReduce_ResultSelector_ComplexNewExpression()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                var aggregates = data.GroupBy(x => 0, (key, seq) => new KeyValuePair <int, KeyValuePair <double, double> >(key, new KeyValuePair <double, double>(seq.Average(), seq.Average()))).ToArray();

                var expected = new KeyValuePair <int, KeyValuePair <double, double> >[] { new KeyValuePair <int, KeyValuePair <double, double> >(0, new KeyValuePair <double, double>(100.5, 100.5)) };

                passed &= aggregates.SequenceEqual(expected);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #10
0
        public static bool GroupByReduceWithCustomDecomposableFunction_NonDistributableCombiner()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                double[] aggregates = data
                                      .Select(x => (double)x)
                                      .GroupBy(x => 0, (key, seq) => DecomposableFunc4(seq)).ToArray();
                double[] expected = new[] { Enumerable.Range(1, 200).Sum() / 100.0 };

                //note the order of the result elements is not guaranteed, so order them before testing
                double[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                double[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #11
0
        public static bool GroupByReduce_BuiltIn_First()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                int[] aggregates = data
                                   .GroupBy(x => 0, (key, seq) => seq.First())
                                   .ToArray();

                // the output of First can be the first item of either partition.
                passed &= aggregates.SequenceEqual(new[] { 1 }) || aggregates.SequenceEqual(new[] { 101 });
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #12
0
        public static bool BuiltInCountIsDistributable()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                // Built in Count is Distributable as built-in logic knows to use Sum() as the combiner function.
                // Count(a,b,c,d) = Sum(Count(a,b), Count(c,d))
                int[] aggregates = pt1.GroupBy(x => x % 2, (key, seq) => seq.Count()).ToArray();
                int[] expected   = new[] { 6, 6 }; // six elements in each full group.

                //note the order of the result elements is not guaranteed, so order them before testing
                int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                int[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #13
0
        public static bool GroupByReduce_BitwiseNegationOperator()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var results = pt1.GroupBy(x => x % 2, (k, g) => new KeyValuePair <int, int>(k, ~g.Sum())).ToArray();

                //local sort.. so that keys are in order.
                var resultsSorted = results.OrderBy(list => list.Key).ToArray();

                //key0: count = 6, sum = 42
                //key1: count = 6, sum = 36

                passed &= (resultsSorted[0].Key == 0);     // "incorrect results.1"
                passed &= (resultsSorted[0].Value == ~42); // "incorrect results.2"

                passed &= (resultsSorted[1].Key == 1);     // "incorrect results.3"
                passed &= (resultsSorted[1].Value == ~36); // "incorrect results.4"
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #14
0
        public static bool template()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/x.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #15
0
        public static bool ToStoreSubmitGetEnumerator() // pass
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/ToStoreSubmitGetEnumerator.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());
                var q1 = pt1.Select(x => 100 + x).HashPartition(x => x);
                var q2 = q1.Where(x => true);
                IQueryable <int> output = q2.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info   = output.SubmitAndWait();

                foreach (int x in output) // should not run a new dryad job.
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #16
0
        public static bool HomomorphicUnaryApply()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/HomomorphicUnaryApply.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q1      = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
                var jobInfo = q1.ToStore <int>(outFile).Submit();
                jobInfo.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #17
0
        public static bool AssumeRangePartition()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/BasicAPITests_AssumeRangePartition.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q =
                    pt1
                    .AssumeRangePartition(x => x, false)
                    .Select(x => 100 + x).ToStore(outFile);
                var info = q.Submit();
                info.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #18
0
        public static bool DistributiveSelect_1()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                // this result selector satisfies "DistributiveOverConcat"
                int[] aggregates = pt1.GroupBy(x => x % 2).Select(group => group.Sum()).ToArray();
                int[] expected   = new[] { 1 + 3 + 5 + 7 + 9 + 11, 2 + 4 + 6 + 8 + 10 + 12 };

                //note the order of the result elements is not guaranteed, so order them before testing
                int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                int[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #19
0
        public static bool Bug11782_Aggregate()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/Bug11782_Aggregate.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                //test Aggregate()
                var c = pt1.Select(x => x).Aggregate((x, y) => x + y);

                //test AggregateAsQuery()
                var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
                DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
                info.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #20
0
        public static bool GetEnumeratorNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                IQueryable <int> q1 = pt1.Select(x => 100 + x);
                IQueryable <int> q2 = q1.Where(x => true);
                foreach (int x in q2) // throws here
                {
                    //Console.WriteLine(x);
                }
                //@TODO: perform a sequence-equals test.

                //IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
                //DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                //           "unittest/output/test2.txt")).SubmitAndWait();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Example #21
0
        public static bool FullHomomorphicBinaryApply_DifferentDataSets(DryadLinqContext context)
        {
            string testName = "FullHomomorphicBinaryApply_DifferentDataSets";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                IEnumerable <int>[] result = new IEnumerable <int> [2];

                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/FullHomomorphicBinaryApply_DifferentDataSets";

                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    IQueryable <int> pt2 = DataGenerator.GetSimpleFileSets(context);
                    IQueryable <int> q1  = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
                    var jobInfo          = q1.ToStore <int>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
                    jobInfo.Wait();

                    passed   &= Validate.outFileExists(outFile);
                    result[0] = q1;
                }

                // local
                {
                    context.LocalDebug = true;
                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    IQueryable <int> pt2 = DataGenerator.GetSimpleFileSets(context);
                    IQueryable <int> q1  = pt1.ApplyPerPartition(pt2, (x, y) => FullHomomorphic_Binary_Func(x, y), false);
                    result[1] = q1;
                }

                // compare result
                try
                {
                    Validate.Check(result);
                }
                catch (Exception ex)
                {
                    TestLog.Message("Error: " + ex.Message);
                    passed &= false;
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Example #22
0
        public static bool Bug12584_HashPartitionOutputCount(DryadLinqContext context) // ToDo
        {
            string testName = "Bug12584_HashPartitionOutputCount";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                IEnumerable <int>[] result = new IEnumerable <int> [2];
                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/Bug12584_HashPartitionOutputCount";

                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var info             = pt1.HashPartition(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
                    info.Wait();

                    //// partition verification are only valid for cluster execution..

                    //// check that nOutputPartitions == nInputPartitions.
                    //// Note: this is today's behavior, but we don't strictly guarantee this and the rules may change.
                    //var inFS = context.DscService.GetFileSet(DataGenerators.SIMPLE_FILESET_NAME);
                    //var inPCount = inFS.GetFiles().Count();
                    //var outFS = context.DscService.GetFileSet("DevUnitTest/Bug12584_out");
                    //var outPCount = outFS.GetFiles().Count();
                    //passed &= TestUtils.Assert(outPCount == inPCount, "Output nPartitions should be equal to input nPartitions.");
                }
                // local
                {
                    context.LocalDebug = true;
                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                }

                // compare result
                try
                {
                    Validate.Check(result);
                }
                catch (Exception ex)
                {
                    TestLog.Message("Error: " + ex.Message);
                    passed &= false;
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
        public static bool RP_singlePartition_autoSeparators(DryadLinqContext context)
        {
            string testName = "RP_singlePartition_autoSeparators";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                IEnumerable <int>[] result = new IEnumerable <int> [2];
                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/RP_singlePartition_autoSeparators";

                    IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
                    IQueryable <int> pt2 = pt1.Apply(x => x) // force a merge
                                           .RangePartition(x => x)
                                           .ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
                    var jobInfo = pt2.Submit();
                    jobInfo.Wait();

                    passed   &= TestRangePartitioned(pt2, 1, false);
                    result[0] = pt2.ToList();
                }
                // local
                {
                    context.LocalDebug = true;
                    IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
                    IQueryable <int> pt2 = pt1.Apply(x => x) // force a merge
                                           .RangePartition(x => x);
                    result[1] = pt2.ToList();
                }

                // compare result
                try
                {
                    Validate.Check(result);
                }
                catch (Exception ex)
                {
                    TestLog.Message("Error: " + ex.Message);
                    passed &= false;
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Example #24
0
        public static bool Bug_16341_SubmitThrowsForDifferentContexts()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            var context2 = new DryadLinqContext(Config.cluster);

            context2.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                       "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt2 = simple2.Select(x => x.First());

                DryadLinqQueryable.Submit(pt1, pt2);
                passed &= false;
            }
            catch (DryadLinqException)
            {
            }

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                IQueryable <LineRecord> input2 = context2.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                       "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple2 = input2.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt2 = simple2.Select(x => x.First());

                DryadLinqQueryable.SubmitAndWait(pt1, pt2);
                passed &= false;
            }
            catch (DryadLinqException)
            {
            }

            return(passed);
        }
Example #25
0
        public static void Test5(string[] args)
        {
            DryadLinqContext context = new DryadLinqContext(1);
            Uri dataUri = AzureUtils.ToAzureUri("msrsvc", "test", "testwrite");
            IEnumerable <LineRecord> lines = DataProvider.ReadData <LineRecord>(context, dataUri);

            foreach (var x in lines)
            {
                Console.WriteLine(x);
            }
        }
Example #26
0
        /*
         * public static bool PlainEnumerableAsDryadQueryToStoreSubmit()
         * {
         *  var context = new DryadLinqContext(Config.cluster);
         *  context.LocalExecution = false;
         *  bool passed = true;
         *  try
         *  {
         *      string outFile = "unittest/output/PlainEnumerableAsDryadQueryToStoreSubmit.txt";
         *
         *      int[] plainData = { 5, 6, 7 };
         *
         *      var q = context.AsDryadQuery(plainData, CompressionScheme.None).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile);
         *      DryadLinqJobInfo info = q.Submit();
         *      info.Wait();
         *
         *      foreach (int x in q)
         *      {
         *          //Console.WriteLine(x);
         *      }
         *
         *      passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
         *  }
         *  catch (DryadLinqException e)
         *  {
         *      passed &= false;
         *  }
         *  return passed;
         * }
         */
        public static bool RepeatSubmit()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/RepeatSubmit.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());


                var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info1 = null;
                DryadLinqJobInfo info2 = null;
                try
                {
                    info1 = q.Submit();
                    info2 = q.Submit(); // does not throw

                    if (!context.LocalDebug)
                    {
                        passed &= false;
                    }
                }
                catch (ArgumentException)
                {
                    passed &= true;
                }

                //wait for any jobs to complete.
                if (info1 != null)
                {
                    info1.Wait();
                }

                if (info2 != null)
                {
                    info2.Wait();
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
        public static bool RP_keySelector_rangeKeys_keyComparer_isDescending(DryadLinqContext context)
        {
            string testName = "RP_keySelector_rangeKeys_keyComparer_isDescending";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                IEnumerable <int>[] result = new IEnumerable <int> [2];
                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/RP_keySelector_rangeKeys_keyComparer_isDescending";

                    IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
                    IQueryable <int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer <int> .Default, true)
                                           .ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
                    var jobInfo = pt2.Submit();
                    jobInfo.Wait();

                    passed   &= TestRangePartitioned(pt2, 4, true);
                    result[0] = pt2.ToList();
                }
                // local
                {
                    context.LocalDebug = true;
                    IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
                    IQueryable <int> pt2 = pt1.RangePartition(x => x, new[] { 8, 5, 2 }, Comparer <int> .Default, true);
                    result[1] = pt2.ToList();
                }

                // compare result
                try
                {
                    Validate.Check(result);
                }
                catch (Exception ex)
                {
                    TestLog.Message("Error: " + ex.Message);
                    passed &= false;
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Example #28
0
        public static bool Bug14010_AlreadyDisposedContext(DryadLinqContext context)
        {
            string testName = "Bug14010_AlreadyDisposedContext";

            TestLog.TestStart(testName);

            bool passed = true;

            context.LocalDebug = false;
            try
            {
                DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
                ctx.Dispose();
                IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
                int output           = pt1.Select(x => x).First();
                passed &= false;
            }
            catch (Exception)
            {
                passed &= true;
            }

            try
            {
                DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
                IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
                ctx.Dispose();
                int output = pt1.Select(x => x).First();
                passed &= false;
            }
            catch (Exception)
            {
                passed &= true;
            }

            try
            {
                DryadLinqContext ctx = new DryadLinqContext(Config.cluster);
                IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(ctx);
                ctx.Dispose();
                IQueryable <int> query = pt1.Select(x => x).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, "abc"), true);
                var info = DryadLinqQueryable.Submit(query);
                passed &= false;
            }
            catch (Exception)
            {
                passed &= true;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Example #29
0
        public static bool ToStoreThrowsForNonQuery()
        {
            bool passed = true;

            try
            {
                int[] data = new[] { 1, 2, 3 };
                var   q1   = data.AsQueryable().Select(x => 100 + x).ToStore <int>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                         "dummy")).ToArray();
                //Should throw as we got into DryadLinq via AsQueryable() rather than via context.
                passed &= false;
            }
            catch (ArgumentException)
            {
                //expected
            }
            return(passed);
        }
Example #30
0
        public static void WordCountExample()
        {
#if local
            // This overload runs the computation on your local computer using a single worker
            var config = new DryadLinqContext(1);

            var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
            // You can create inputs from any IEnumerable source using this method
            var input = config.FromEnumerable(lines);
#else
            string clusterName = "Replace with your HDInsight 3.0 cluster name";
            // to use the davinci.txt example input below, select your cluster's default
            // storage account and container, which automatically includes the sample text
            string accountName   = "Replace with a storage account name";
            string containerName = "Replace with a storage container name";

            // This overload creates an Azure-based computation
            var config = new DryadLinqContext(clusterName);
            config.JobFriendlyName = "DryadLINQ Sample Wordcount";

            // plain text files should be read as type LineRecord
            var input = config.FromStore <LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
                                                                            "example/data/gutenberg/davinci.txt"));
#endif

            var words    = input.SelectMany(x => x.Line.Split(' '));
            var groups   = words.GroupBy(x => x);
            var counts   = groups.Select(x => new KeyValuePair <string, int>(x.Key, x.Count()));
            var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));

#if local
            // any collection computed by the query can be materialized back at the client,
            // not just the 'output' collection. For large collections this is expensive!
            foreach (LineRecord line in toOutput)
            {
                Console.WriteLine(line.Line);
            }
#else
            // the 'true' parameter to ToStore means the output will be over-written if you run
            // the job more than once
            var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
                                                              "wc-out.txt"), true).SubmitAndWait();
#endif
        }