コード例 #1
0
        public static bool GroupByReduce_ResultSelector_ComplexNewExpression()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                var aggregates = data.GroupBy(x => 0, (key, seq) => new KeyValuePair <int, KeyValuePair <double, double> >(key, new KeyValuePair <double, double>(seq.Average(), seq.Average()))).ToArray();

                var expected = new KeyValuePair <int, KeyValuePair <double, double> >[] { new KeyValuePair <int, KeyValuePair <double, double> >(0, new KeyValuePair <double, double>(100.5, 100.5)) };

                passed &= aggregates.SequenceEqual(expected);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #2
0
        public static bool GroupByReduce_BuiltIn_First()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                int[] aggregates = data
                                   .GroupBy(x => 0, (key, seq) => seq.First())
                                   .ToArray();

                // the output of First can be the first item of either partition.
                passed &= aggregates.SequenceEqual(new[] { 1 }) || aggregates.SequenceEqual(new[] { 101 });
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #3
0
        public static bool GroupByReduceWithCustomDecomposableFunction_NonDistributableCombiner()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                double[] aggregates = data
                                      .Select(x => (double)x)
                                      .GroupBy(x => 0, (key, seq) => DecomposableFunc4(seq)).ToArray();
                double[] expected = new[] { Enumerable.Range(1, 200).Sum() / 100.0 };

                //note the order of the result elements is not guaranteed, so order them before testing
                double[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                double[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #4
0
        public static bool GroupByReduce_BitwiseNegationOperator()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var results = pt1.GroupBy(x => x % 2, (k, g) => new KeyValuePair <int, int>(k, ~g.Sum())).ToArray();

                //local sort.. so that keys are in order.
                var resultsSorted = results.OrderBy(list => list.Key).ToArray();

                //key0: count = 6, sum = 42
                //key1: count = 6, sum = 36

                passed &= (resultsSorted[0].Key == 0);     // "incorrect results.1"
                passed &= (resultsSorted[0].Value == ~42); // "incorrect results.2"

                passed &= (resultsSorted[1].Key == 1);     // "incorrect results.3"
                passed &= (resultsSorted[1].Value == ~36); // "incorrect results.4"
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #5
0
        public static bool GroupByReduce_SameDecomposableUsedTwice()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var results = pt1.GroupBy(x => x % 2, (k, g) => MyFunc(k, DecomposableFunc5(g), DecomposableFunc5(g), g.Average())).ToArray();

                //key0: count = 6, av = av(2,4,6,8,10,12) = 7
                //key1: count = 6, av = av(1,3,5,7,9,11) = 6

                //local sort.. so that keys are in order.
                var results_sorted = results.OrderBy(x => x.Key).ToArray();

                passed &= (results_sorted.Length == 2);  // "wrong results"

                passed &= (results_sorted[0].Key == 0);  // "wrong results"
                passed &= (results_sorted[0].A == 6);    // "wrong results"
                passed &= (results_sorted[0].B == 6);    // "wrong results"
                passed &= (results_sorted[0].Av == 7.0); // "wrong results"

                passed &= (results_sorted[1].Key == 1);  // "wrong results"
                passed &= (results_sorted[1].A == 6);    // "wrong results"
                passed &= (results_sorted[1].B == 6);    // "wrong results"
                passed &= (results_sorted[1].Av == 6.0); // "wrong results"
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #6
0
        public static bool GroupByReduce_CustomListInitializerReducer()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var results = pt1.GroupBy(x => x % 2, (k, g) => new MultiParamInitializerClass()
                {
                    { k, g.Count(), g.Sum() },                                                                 // one item, comprising three components
                }).ToArray();
                //local sort.. so that keys are in order.
                var resultsSorted = results.OrderBy(list => list.Key).ToArray();

                //key0: count = 6, sum = 42
                //key1: count = 6, sum = 36

                passed &= (resultsSorted[0].Key == 0);     // "incorrect results.1"
                passed &= (resultsSorted[0].Count() == 6); // "incorrect results.2"
                passed &= (resultsSorted[0].Sum() == 42);  // "incorrect results.3"

                passed &= (resultsSorted[1].Key == 1);     // "incorrect results.4"
                passed &= (resultsSorted[1].Count() == 6); // "incorrect results.5"
                passed &= (resultsSorted[1].Sum() == 36);  // "incorrect results.6"
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #7
0
        public static bool template()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                //passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
コード例 #8
0
        internal static bool GroupByReduce_APIMisuse()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                if (context.LocalDebug)
                {
                    // "decomposition logic doesn't run in LocalDebug.. skipping";
                    return(true);
                }

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                // internal-visibility decomposable type should fail.
                try
                {
                    pt1.GroupBy(x => x, (k, g) => BadDecomposable1(g)).ToArray();
                    passed &= false; // "exception should be thrown"
                }
                catch (DryadLinqException)
                {
                    //??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeMustBePublic); // "error code is wrong"
                }

                // decomposable type doesn't implement IDecomposable or IDecomposableRecursive
                try
                {
                    pt1.GroupBy(x => x, (k, g) => BadDecomposable2(g)).ToArray();
                    passed &= false; //"exception should be thrown");
                }
                catch (DryadLinqException)
                {
                    //??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeDoesNotImplementInterface);
                }

                // decomposable type implements more than one IDecomposable or IDecomposableRecursive
                try
                {
                    pt1.GroupBy(x => x, (k, g) => BadDecomposable3(g)).ToArray();
                    passed &= false;
                }
                catch (DryadLinqException)
                {
                    //??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeImplementsTooManyInterfaces);
                }

                // decomposable type doesn't have public default ctor
                try
                {
                    pt1.GroupBy(x => x, (k, g) => BadDecomposable4(g)).ToArray();
                    passed &= false;
                }
                catch (DryadLinqException)
                {
                    //??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypeDoesNotHavePublicDefaultCtor);
                }

                // decomposable type input type doesn't match
                try
                {
                    pt1.GroupBy(x => x, (k, g) => BadDecomposable5(g)).ToArray();
                    passed &= false;
                }
                catch (DryadLinqException)
                {
                    //??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypesDoNotMatch);
                }

                // decomposable type output type doesn't match
                try
                {
                    pt1.GroupBy(x => x, (k, g) => BadDecomposable6(g)).ToArray();
                    passed &= false;
                }
                catch (DryadLinqException)
                {
                    //??? passed &= (Ex.ErrorCode == DryadLinqErrorCode.DecomposerTypesDoNotMatch);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }