Beispiel #1
0
        public static void Run(DryadLinqContext context, string matchPattern)
        {
            TestLog.Message(" **********************");
            TestLog.Message(" ApplyAndForkTests ");
            TestLog.Message(" **********************");

            var tests = new Dictionary <string, Action>()
            {
                { "NonHomomorphicUnaryApply", () => NonHomomorphicUnaryApply(context) },
                { "HomomorphicUnaryApply", () => HomomorphicUnaryApply(context) },
                { "NonHomomorphicBinaryApply", () => NonHomomorphicBinaryApply(context) },
                { "LeftHomomorphicBinaryApply", () => LeftHomomorphicBinaryApply(context) },
                { "FullHomomorphicBinaryApply_DifferentDataSets", () => FullHomomorphicBinaryApply_DifferentDataSets(context) },
                { "FullHomomorphicBinaryApply_IdenticalDataSets", () => FullHomomorphicBinaryApply_IdenticalDataSets(context) },
                { "Aggregate_WithCombiner", () => Aggregate_WithCombiner(context) },
            };

            foreach (var test in tests)
            {
                if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
                {
                    test.Value.Invoke();
                }
            }
        }
        public static bool RP_rangeSeparators_nullCustomComparer(DryadLinqContext context)
        {
            string testName = "RP_rangeSeparators_nullCustomComparer";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                // cluster
                {
                    context.LocalDebug = false;
                    IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
                    var results1         = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null, false).ToArray();
                    var results2         = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null).ToArray();
                    //passing is not throwing.
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #3
0
        public static bool GetEnumeratorNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);
            context.LocalExecution = false;
            bool passed = true;
            try
            {
                IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                             "unittest/inputdata/SimpleFile.txt"));

                IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable<int> pt1 = simple.Select(x => x.First());

                IQueryable<int> q1 = pt1.Select(x => 100 + x);
                IQueryable<int> q2 = q1.Where(x => true);
                foreach (int x in q2) // throws here
                {
                    //Console.WriteLine(x);
                }
                //@TODO: perform a sequence-equals test.

                //IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
                //DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                //           "unittest/output/test2.txt")).SubmitAndWait();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return passed;
        }
Beispiel #4
0
        public static bool GroupByReduce_BuiltIn_First()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                int[] aggregates = data
                                   .GroupBy(x => 0, (key, seq) => seq.First())
                                   .ToArray();

                // the output of First can be the first item of either partition.
                passed &= aggregates.SequenceEqual(new[] { 1 }) || aggregates.SequenceEqual(new[] { 101 });
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #5
0
        public static bool DistributiveSelect_1()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                // this result selector satisfies "DistributiveOverConcat"
                int[] aggregates = pt1.GroupBy(x => x % 2).Select(group => group.Sum()).ToArray();
                int[] expected   = new[] { 1 + 3 + 5 + 7 + 9 + 11, 2 + 4 + 6 + 8 + 10 + 12 };

                //note the order of the result elements is not guaranteed, so order them before testing
                int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                int[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #6
0
        public static bool Bug13970_MismatchedDataTypes(DryadLinqContext context)
        {
            string testName = "Bug13970_MismatchedDataTypes";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                // local
                {
                    context.LocalDebug = true;
                    IQueryable <int> pt1    = DataGenerator.GetGroupByReduceDataSet(context);
                    double?          output = pt1.AverageAsQuery().Single();
                    passed &= false;
                }
            }
            catch (Exception)
            {
                passed &= true; // expected
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #7
0
        public static bool GroupByReduce_BitwiseNegationOperator()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var results = pt1.GroupBy(x => x % 2, (k, g) => new KeyValuePair <int, int>(k, ~g.Sum())).ToArray();

                //local sort.. so that keys are in order.
                var resultsSorted = results.OrderBy(list => list.Key).ToArray();

                //key0: count = 6, sum = 42
                //key1: count = 6, sum = 36

                passed &= (resultsSorted[0].Key == 0);     // "incorrect results.1"
                passed &= (resultsSorted[0].Value == ~42); // "incorrect results.2"

                passed &= (resultsSorted[1].Key == 1);     // "incorrect results.3"
                passed &= (resultsSorted[1].Value == ~36); // "incorrect results.4"
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #8
0
        public static bool EnumeratePlainData()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());
                foreach (int x in pt1) // throws
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #9
0
        public static bool CopyPlainDataViaToStoreMaterialize()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
                info.Wait();

                foreach (int x in q)
                {
                    //Console.WriteLine(x);
                }

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #10
0
        public static bool SubmitNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q1 = pt1.Select(x => 100 + x);
                var q2 = q1.Where(x => true);
                q2.SubmitAndWait(); // throws here
                var outPT = q2.ToList();
                foreach (int x in outPT)
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #11
0
        public static bool MaterializeNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1   = simple.Select(x => x.First());
                IQueryable <int> query = pt1.Select(x => 100 + x);

                DryadLinqQueryable.Submit(query); //materialize // throws

                foreach (int x in query)
                {
                    //Console.WriteLine(x);
                }

                //@TODO: assert that only one query execution occurred.
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #12
0
        public static bool template()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/x.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #13
0
        public static bool HomomorphicUnaryApply()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/HomomorphicUnaryApply.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q1      = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
                var jobInfo = q1.ToStore <int>(outFile).Submit();
                jobInfo.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #14
0
        public static bool Aggregate_WithCombiner()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));

                passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #15
0
        public static bool Aggregate_WithCombiner(DryadLinqContext context)
        {
            string testName = "Aggregate_WithCombiner";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                // cluster
                {
                    context.LocalDebug = false;
                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    string           q1  = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
                    passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #16
0
        public static bool ToStoreGetEnumeratorThrows() // pass
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));
                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());

                IQueryable <int> pt1 = simple.Select(x => x.First());
                IQueryable <int> q1  = pt1.Select(x => 100 + x);

                var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
                output.GetEnumerator();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
        public PeloponneseJobSubmission(DryadLinqContext context)
        {
            m_context = context;
            m_localResources = new Dictionary<string, HashSet<string>>();

            m_peloponneseGMFiles = Peloponnese.ClusterUtils.ConfigHelpers
                .ListPeloponneseResources(context.PeloponneseHomeDirectory)
                .Select(r => r.ToLower());
            IEnumerable<string> graphManagerFiles = Peloponnese.Shared.DependencyLister.Lister
                .ListDependencies(Path.Combine(context.DryadHomeDirectory, "Microsoft.Research.Dryad.GraphManager.exe"))
                .Select(r => r.ToLower());
            m_dryadGMFiles = graphManagerFiles.Except(m_peloponneseGMFiles);

            string[] additionalWorkerFiles =
            {
                "Microsoft.Research.Dryadlinq.dll",
                "Microsoft.Research.Dryad.DryadLinq.NativeWrapper.dll"
            };

            IEnumerable<string> processServiceFiles = Peloponnese.Shared.DependencyLister.Lister
                .ListDependencies(Path.Combine(context.DryadHomeDirectory, "Microsoft.Research.Dryad.ProcessService.exe"))
                .Select(r => r.ToLower());
            IEnumerable<string> vertexHostFiles = Peloponnese.Shared.DependencyLister.Lister
                .ListDependencies(Path.Combine(context.DryadHomeDirectory, "Microsoft.Research.Dryad.VertexHost.exe"))
                .Concat(additionalWorkerFiles.Select(f => Path.Combine(context.DryadHomeDirectory, f)))
                .Select(r => r.ToLower());
            IEnumerable<string> workerFiles = processServiceFiles.Union(vertexHostFiles);

            m_peloponneseWorkerFiles = workerFiles.Intersect(m_peloponneseGMFiles);
            m_dryadWorkerFiles = workerFiles.Except(m_peloponneseGMFiles);
        }
Beispiel #18
0
        public static bool GetEnumeratorNonToStoreTerminated()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                IQueryable <int> q1 = pt1.Select(x => 100 + x);
                IQueryable <int> q2 = q1.Where(x => true);
                foreach (int x in q2) // throws here
                {
                    //Console.WriteLine(x);
                }
                //@TODO: perform a sequence-equals test.

                //IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
                //DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                //           "unittest/output/test2.txt")).SubmitAndWait();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #19
0
        public static bool BuiltInCountIsDistributable()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                // Built in Count is Distributable as built-in logic knows to use Sum() as the combiner function.
                // Count(a,b,c,d) = Sum(Count(a,b), Count(c,d))
                int[] aggregates = pt1.GroupBy(x => x % 2, (key, seq) => seq.Count()).ToArray();
                int[] expected   = new[] { 6, 6 }; // six elements in each full group.

                //note the order of the result elements is not guaranteed, so order them before testing
                int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                int[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #20
0
        public static bool Bug11782_Aggregate()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/Bug11782_Aggregate.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                //test Aggregate()
                var c = pt1.Select(x => x).Aggregate((x, y) => x + y);

                //test AggregateAsQuery()
                var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
                DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
                info.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #21
0
        public static bool GroupByReduceWithCustomDecomposableFunction_NonDistributableCombiner()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                double[] aggregates = data
                                      .Select(x => (double)x)
                                      .GroupBy(x => 0, (key, seq) => DecomposableFunc4(seq)).ToArray();
                double[] expected = new[] { Enumerable.Range(1, 200).Sum() / 100.0 };

                //note the order of the result elements is not guaranteed, so order them before testing
                double[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
                double[] expectedOrdered   = expected.OrderBy(x => x).ToArray();

                passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #22
0
        public static bool Bug11782_LowLevelQueryableManipulation()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                Expression lambda = Expression.Lambda <Func <int, int> >(
                    Expression.Constant(1),
                    new[] { Expression.Parameter(typeof(int), "x") });
                var z = pt1.Provider.CreateQuery(
                    Expression.Call(
                        typeof(Queryable), "Select",
                        new Type[] { pt1.ElementType, pt1.ElementType },
                        pt1.Expression, Expression.Quote(lambda)));

                passed &= false; // the use of non-generic Provider.CreateQuery() should have thrown
            }
            catch (DryadLinqException)
            {
                passed &= true;
            }
            return(passed);
        }
Beispiel #23
0
        public static bool GroupByReduce_ResultSelector_ComplexNewExpression()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
                IQueryable <int> data = simple.Select(x => x.First());

                var aggregates = data.GroupBy(x => 0, (key, seq) => new KeyValuePair <int, KeyValuePair <double, double> >(key, new KeyValuePair <double, double>(seq.Average(), seq.Average()))).ToArray();

                var expected = new KeyValuePair <int, KeyValuePair <double, double> >[] { new KeyValuePair <int, KeyValuePair <double, double> >(0, new KeyValuePair <double, double>(100.5, 100.5)) };

                passed &= aggregates.SequenceEqual(expected);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #24
0
        public static bool AssumeRangePartition()
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/BasicAPITests_AssumeRangePartition.out";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());

                var q =
                    pt1
                    .AssumeRangePartition(x => x, false)
                    .Select(x => 100 + x).ToStore(outFile);
                var info = q.Submit();
                info.Wait();

                passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
        public static void Run(DryadLinqContext context, string matchPattern)
        {
            TestLog.Message(" **********************");
            TestLog.Message(" RangePartitionAPICoverageTests ");
            TestLog.Message(" **********************");

            var tests = new Dictionary <string, Action>()
            {
                { "RP_keySelector", () => RP_keySelector(context) },
                { "RP_keySelector_pcount", () => RP_keySelector_pcount(context) },
                { "RP_keySelector_isDescending", () => RP_keySelector_isDescending(context) },
                { "RP_keySelector_rangeKeys", () => RP_keySelector_rangeKeys(context) },
                { "RP_keySelector_isDescending_pcount", () => RP_keySelector_isDescending_pcount(context) },
                { "RP_keySelector_keyComparer_isDescending", () => RP_keySelector_keyComparer_isDescending(context) },
                { "RP_keySelector_rangeKeys_keyComparer", () => RP_keySelector_rangeKeys_keyComparer(context) },
                { "RP_keySelector_keyComparer_isDescending_pcount", () => RP_keySelector_keyComparer_isDescending_pcount(context) },
                { "RP_keySelector_rangeKeys_keyComparer_isDescending", () => RP_keySelector_rangeKeys_keyComparer_isDescending(context) },
                { "RP_singlePartition_autoSeparators", () => RP_singlePartition_autoSeparators(context) },
                { "RP_rangeSeparators_customComparer", () => RP_rangeSeparators_customComparer(context) },
                { "RP_rangeSeparators_nullCustomComparer", () => RP_rangeSeparators_nullCustomComparer(context) },
            };

            foreach (var test in tests)
            {
                if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
                {
                    test.Value.Invoke();
                }
            }
        }
Beispiel #26
0
        public static bool ToStoreSubmitGetEnumerator() // pass
        {
            var context = new DryadLinqContext(Config.cluster);

            context.LocalExecution = false;
            bool passed = true;

            try
            {
                string outFile = "unittest/output/ToStoreSubmitGetEnumerator.txt";

                IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                                                                                                     "unittest/inputdata/SimpleFile.txt"));

                IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
                IQueryable <int> pt1 = simple.Select(x => x.First());
                var q1 = pt1.Select(x => 100 + x).HashPartition(x => x);
                var q2 = q1.Where(x => true);
                IQueryable <int> output = q2.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
                DryadLinqJobInfo info   = output.SubmitAndWait();

                foreach (int x in output) // should not run a new dryad job.
                {
                    //Console.WriteLine(x);
                }
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return(passed);
        }
Beispiel #27
0
        public static bool ToStoreGetEnumeratorThrows() // pass
        {
            var context = new DryadLinqContext(Config.cluster);
            context.LocalExecution = false;
            bool passed = true;
            try
            {
                string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";

                IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
                                                         "unittest/inputdata/SimpleFile.txt"));
                IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());

                IQueryable<int> pt1 = simple.Select(x => x.First());
                IQueryable<int> q1 = pt1.Select(x => 100 + x);

                var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
                output.GetEnumerator();
            }
            catch (DryadLinqException)
            {
                passed &= false;
            }
            return passed;
        }
Beispiel #28
0
        public static bool Bug14870_LongIndexTakeWhile(DryadLinqContext context)
        {
            string testName = "Bug14870_LongIndexTakeWhile";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                IEnumerable <int>[] result = new IEnumerable <int> [2];
                // cluster
                {
                    context.LocalDebug = false;

                    // ToDo - move to data generator
                    int[][] data = new[] {
                        new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { }, new int[] { }
                    };

                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var output           = pt1.LongTakeWhile((x, i) => i < 8).ToArray();
                    passed &= (output.Length == 8); // "Only eight items should be returned."
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #29
0
        public static bool Bug13108_DisableSequenceEquals(DryadLinqContext context)
        {
            string testName = "Bug13108_DisableSequenceEquals";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                // cluster
                {
                    context.LocalDebug = false;

                    IQueryable <int> pt1    = DataGenerator.GetSimpleFileSets(context);
                    IQueryable <int> pt2    = DataGenerator.GetSimpleFileSets(context);
                    bool             result = pt1.Select(x => x).SequenceEqual(pt2.Select(x => x));
                    passed &= false; // NotSupportedException should have been thrown
                }
            }
            catch (Exception Ex)
            {
            }
            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #30
0
        public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
        {
            string testName = "Bug15371_NoDataMembersForSerialization";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";

                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    IQueryable <NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
                    var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
                    jobInfo.Wait();
                    var result = context.FromStore <NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
                    passed &= false;
                }
            }
            catch (DryadLinqException Ex)
            {
                passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
                           Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #31
0
        public static bool Bug13245_FromDsc_Submit_throws(DryadLinqContext context)
        {
            string testName = "Bug13245_FromDsc_Submit_throws";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                IEnumerable <int>[] result = new IEnumerable <int> [2];
                // cluster
                {
                    context.LocalDebug = false;
                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var info             = pt1.Submit();
                    passed &= false; // should throw ArgEx as the input isn't well formed
                }
            }
            catch (ArgumentException)
            {
            }
            catch (Exception)
            {
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #32
0
        public static bool Bug13302_ConfigEnvironmentCleanup(DryadLinqContext context)
        {
            string testName = "Bug13302_ConfigEnvironmentCleanup";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                // cluster
                {
                    context.LocalDebug = false;
                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var info             = pt1.Apply((en) => CheckEnvVar(en)).Submit();
                    info.Wait();
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #33
0
        public static bool Bug13245_QueryUsingNonHpcLinqOperator(DryadLinqContext context)
        {
            string testName = "Bug13245_QueryUsingNonHpcLinqOperator";

            TestLog.TestStart(testName);

            bool passed = true;

            try
            {
                IEnumerable <int>[] result = new IEnumerable <int> [2];
                // cluster
                {
                    context.LocalDebug = false;

                    IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var info             = pt1.Blah().Submit();
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return(passed);
        }
Beispiel #34
0
 public static void Test1(string[] args)
 {
     DryadLinqContext context = new DryadLinqContext(1, "partfile");
     //context.PartitionUncPath = "DryadLinqTemp/PartFiles";
     var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
     var lines = input.Where(x => x.Line.Contains("white"));
     foreach (var x in lines) Console.WriteLine(x);
     //var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
     //result.SubmitAndWait();
 }
Beispiel #35
0
 public static void Test2(string[] args)
 {
     DryadLinqContext context = new DryadLinqContext(1, "partfile");
     var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
     var q1 = input.Where(x => x.Line.Contains("white"));
     var q2 = input.Where(x => x.Line.Contains("the"));
     var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
     var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
     DryadLinqQueryable.SubmitAndWait(res1, res2);
 }
Beispiel #36
0
 public static void Test3(string[] args)
 {
     DryadLinqContext context = new DryadLinqContext(1, "partfile");
     var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
     var words = input.SelectMany(x => x.Line.Split(' '));
     var groups = words.GroupBy(x => x);
     var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
     var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
     var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
     result.SubmitAndWait();
 }
Beispiel #37
0
 // This constructor is specifically to support enumeration of a query.
 // It assumes that the Expressions all terminate with a ToStore node.        
 internal DryadLinqQueryGen(DryadLinqContext context,
                            VertexCodeGen vertexCodeGen,
                            Expression queryExpr,
                            Uri tableUri,
                            bool isTempOutput)
 {
     this.m_queryExprs = new Expression[] { queryExpr };
     this.m_outputTableUris = new Uri[] { tableUri };
     this.m_isTempOutput = new bool[] { isTempOutput };
     this.m_context = context;
     this.Initialize(vertexCodeGen);
 }
Beispiel #38
0
 /// <summary>
 /// Create a new job executor object.
 /// </summary>
 public DryadLinqJobExecutor(DryadLinqContext context)
 {
     // use a new job submission object for each query
     this.m_context = context;
     this.m_currentStatus = JobStatus.NotSubmitted;
     if (context.LocalExecution)
     {
         this.m_jobSubmission = new LocalJobSubmission(context);
     }
     else
     {
         this.m_jobSubmission = new YarnJobSubmission(context);
     }
 }
        internal DryadLinqMetaData(DryadLinqContext context,
                                   Type recordType,
                                   Uri dataSetUri,
                                   CompressionScheme compressionScheme)
        {
            this.m_context = context;
            this.m_dataSetUri = dataSetUri;
            this.m_elemType = recordType;
            this.m_compressionScheme = compressionScheme;
            //this.m_version = context.ClientVersion();
            //this.InitializeFlags();

            //this.m_fp = 0UL;
            //this.m_dataSetInfo = node.OutputDataSetInfo;
        }
 /// <summary>
 /// Create a new job executor object.
 /// </summary>
 public DryadLinqJobExecutor(DryadLinqContext context)
 {
     // use a new job submission object for each query
     this.m_context = context;
     this.m_currentStatus = JobStatus.NotSubmitted;
     if (context.PlatformKind == PlatformKind.LOCAL)
     {
         this.m_jobSubmission = new LocalJobSubmission(context);
     }
     else
     {
         this.m_jobSubmission = new YarnJobSubmission(
             context, context.Cluster.MakeInternalClusterUri("tmp", "staging"));
     }
 }
Beispiel #41
0
        public static void Run(DryadLinqContext context, string matchPattern)
        {
            TestLog.Message(" **********************");
            TestLog.Message(" BasicAPITests ");
            TestLog.Message(" **********************");

            var tests = new Dictionary<string, Action>()
              {
                  {"ToStoreThrowsForNonQuery", () => ToStoreThrowsForNonQuery(context) },
                  {"ToStoreGetEnumerator", () => ToStoreGetEnumerator(context) },
                  {"GetEnumeratorNonToStoreTerminated", () => GetEnumeratorNonToStoreTerminated(context) },
                  {"ToStoreSubmitGetEnumerator", () => ToStoreSubmitGetEnumerator(context) },
                  {"SubmitNonToStoreTerminated", () => SubmitNonToStoreTerminated(context) },
                  {"MaterializeToStoreTerminated", () => MaterializeToStoreTerminated(context) },
                  {"MaterializeNonToStoreTerminated", () => MaterializeNonToStoreTerminated(context) },
                  {"EnumeratePlainData", () => EnumeratePlainData(context) }, 
                  {"CopyPlainDataViaToStoreSubmit", () => CopyPlainDataViaToStoreSubmit(context) },
                  {"CopyPlainDataViaToStoreMaterialize", () => CopyPlainDataViaToStoreMaterialize(context) },
                  {"PlainEnumerableAsDryadQueryToStoreSubmit", () => PlainEnumerableAsDryadQueryToStoreSubmit(context) },
                  {"RepeatSubmit", () => RepeatSubmit(context) }, 
                  // ToDo {"RepeatMaterialize", () => RepeatMaterialize(context) }, hangs
                  {"MaterializeMentionsSameQueryTwice", () => MaterializeMentionsSameQueryTwice(context) },
                  {"QueryOnDataBackedDLQ", () => QueryOnDataBackedDLQ(context) },
                  {"Bug11781_CountandFirstOrDefault", () => Bug11781_CountandFirstOrDefault(context) },
                  {"Bug11782_Aggregate", () => Bug11782_Aggregate(context) },
                  {"Bug11782_LowLevelQueryableManipulation", () => Bug11782_LowLevelQueryableManipulation(context) },
                  {"Bug11638_LongWhere", () => Bug11638_LongWhere(context) },
                  {"AssumeRangePartition", () => AssumeRangePartition(context) },
                  {"Bug11638_LongMethods", () => Bug11638_LongMethods(context) },
                  {"ContextConfigIsReadOnly", () => ContextConfigIsReadOnly(context) },
                  // ToDo{"ToggleSpeculativeDuplication", () => ToggleSpeculativeDuplication(context) },
                  {"Bug15068_ConfigResourcesAPI", () => Bug15068_ConfigResourcesAPI(context) },
                  {"Bug14449_ContextShouldExposeVersionIDs", () => Bug14449_ContextShouldExposeVersionIDs(context) }, // not valid anymore
                  {"Bug_16341_SubmitThrowsForDifferentContexts", () => Bug_16341_SubmitThrowsForDifferentContexts(context) },
                  {"Bug_16341_VariousTestsForSubmit", () => Bug_16341_VariousTestsForSubmit(context) },
              };

            foreach (var test in tests)
            {
                if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
                {
                    test.Value.Invoke();
                }
            }
        }
        public static bool Bug11447_GroupByWithComparer(DryadLinqContext context)
        {
            string testName = "Bug11447_GroupByWithComparer";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                IEnumerable<int>[] result = new IEnumerable<int>[2];
                // cluster
                {
                    context.LocalDebug = false;
                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
                    result[0] = a;
                }
                // local
                {
                    context.LocalDebug = true;
                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
                    result[1] = a;
                }

                // compare result
                try
                {
                    Validate.Check(result);
                }
                catch (Exception ex)
                {
                    TestLog.Message("Error: " + ex.Message);
                    passed &= false;
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }
Beispiel #43
0
        public static void WordCountExample()
        {
#if local
			// This overload runs the computation on your local computer using a single worker
            var config = new DryadLinqContext(1);

            var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
            // You can create inputs from any IEnumerable source using this method
            var input = config.FromEnumerable(lines);
#else
			string clusterName = "Replace with your HDInsight 3.0 cluster name";
            // to use the davinci.txt example input below, select your cluster's default
            // storage account and container, which automatically includes the sample text
			string accountName = "Replace with a storage account name";
			string containerName = "Replace with a storage container name";

			// This overload creates an Azure-based computation
            var config = new DryadLinqContext(clusterName);
            config.JobFriendlyName = "DryadLINQ Sample Wordcount";

            // plain text files should be read as type LineRecord
			var input = config.FromStore<LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
					                                 "example/data/gutenberg/davinci.txt"));
#endif

            var words = input.SelectMany(x => x.Line.Split(' '));
            var groups = words.GroupBy(x => x);
            var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
            var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));

#if local
            // any collection computed by the query can be materialized back at the client,
            // not just the 'output' collection. For large collections this is expensive!
            foreach (LineRecord line in toOutput)
            {
                Console.WriteLine(line.Line);
            }
#else
            // the 'true' parameter to ToStore means the output will be over-written if you run
            // the job more than once
            var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
			           "wc-out.txt"), true).SubmitAndWait();
#endif
        }
        public static void Run(DryadLinqContext context, string matchPattern)
        {
            TestLog.Message(" **********************");
            TestLog.Message(" MiscBugFixTests ");
            TestLog.Message(" **********************");

            var tests = new Dictionary<string, Action>()
              {
                  {"Bug11447_GroupByWithComparer", () => Bug11447_GroupByWithComparer(context) },
                  {"Bug12584_HashPartitionOutputCount", () => Bug12584_HashPartitionOutputCount(context) },
                  {"Bug13108_DisableSequenceEquals", () => Bug13108_DisableSequenceEquals(context) },
                  {"Bug13529_and_Bug13593_IndexedOperatorCompilation", () => Bug13529_and_Bug13593_IndexedOperatorCompilation(context) },
                  {"Bug13130_ReverseOperator", () => Bug13130_ReverseOperator(context) },
                  {"Bug13736_IndexedTakeWhile", () => Bug13736_IndexedTakeWhile(context) },
                  {"Bug13534_HashPartitionNegIndexIsError", () => Bug13534_HashPartitionNegIndexIsError(context) },
                  {"Bug13474_and_Bug13483_FromDscOnBadFileSet", () => Bug13474_and_Bug13483_FromDscOnBadFileSet(context) },
                  {"Bug13637_EmptyFilesInFilesets", () => Bug13637_EmptyFilesInFilesets(context) },
                  {"Bug13637_LocalDebugProducingZeroRecords", () => Bug13637_LocalDebugProducingZeroRecords(context) },
                  {"Bug13245_FromDsc_Submit_throws", () => Bug13245_FromDsc_Submit_throws(context) },
                  {"Bug13245_QueryUsingNonHpcLinqOperator", () => Bug13245_QueryUsingNonHpcLinqOperator(context) },
                  {"Bug13302_ConfigEnvironmentCleanup", () => Bug13302_ConfigEnvironmentCleanup(context) },
                  {"Bug13970_MismatchedDataTypes", () => Bug13970_MismatchedDataTypes(context) },
                  {"Bug14010_AlreadyDisposedContext", () => Bug14010_AlreadyDisposedContext(context) },
                  {"Bug14256_LeaseOnTempDscFileset", () => Bug14256_LeaseOnTempDscFileset(context) },
                  {"Bug14189_OrderPreservation", () => Bug14189_OrderPreservation(context) },
                  {"Bug14190_MergeJoin_DecreasingOrder", () => Bug14190_MergeJoin_DecreasingOrder(context) },
                  {"Bug14192_MultiApplySubExpressionReuse", () => Bug14192_MultiApplySubExpressionReuse(context) },
                  {"Bug14870_LongIndexTakeWhile", () => Bug14870_LongIndexTakeWhile(context) },
                  {"Bug15159_NotOperatorForNullableBool", () => Bug15159_NotOperatorForNullableBool(context) },
                  {"Bug15371_NoDataMembersForSerialization", () => Bug15371_NoDataMembersForSerialization(context) },
                  {"Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes", () => Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(context) },
              };

            foreach (var test in tests)
            {
                if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
                {
                    test.Value.Invoke();
                }
            }
        }
        // create DryadLinqMetaData from a query OutputNode
        internal static DryadLinqMetaData Get(DryadLinqContext context, DLinqOutputNode node)
        {
            DryadLinqMetaData metaData = new DryadLinqMetaData();

            if (!DataPath.IsValidDataPath(node.OutputUri))
            {
                throw new InvalidOperationException();
            }

            metaData.m_context = context;
            metaData.m_dataSetUri = node.OutputUri;
            metaData.m_elemType = node.OutputTypes[0];
            metaData.m_compressionScheme = node.OutputCompressionScheme;
            //metaData.m_version = context.ClientVersion();
            //metaData.InitializeFlags();
            
            //metaData.m_fp = 0UL;
            //metaData.m_dataSetInfo = node.OutputDataSetInfo;

            return metaData;
        }
Beispiel #46
0
        public static bool ToStoreThrowsForNonQuery(DryadLinqContext context) 
        {
            string testName = "ToStoreThrowsForNonQuery";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                int[] data = new[] { 1, 2, 3 };
                var q1 = data.AsQueryable().Select(x => 100 + x).ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
                           "dummy")).ToArray();
                //Should throw as we got into DryadLinq via AsQueryable() rather than via context.
                passed &= false;
            }
            catch (Exception Ex)
            {
                //expected
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }
        public static bool Bug14870_LongIndexTakeWhile(DryadLinqContext context)
        {
            string testName = "Bug14870_LongIndexTakeWhile";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                IEnumerable<int>[] result = new IEnumerable<int>[2];
                // cluster
                {
                    context.LocalDebug = false;

                    // ToDo - move to data generator
                    int[][] data = new[] { 
                        new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { }, new int[] { } }; 

                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var output = pt1.LongTakeWhile((x, i) => i < 8).ToArray();
                    passed &= (output.Length == 8); // "Only eight items should be returned."
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }
        public static bool Bug15159_NotOperatorForNullableBool(DryadLinqContext context)
        {
            string testName = "Bug15159_NotOperatorForNullableBool";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                IEnumerable<bool?>[] result = new IEnumerable<bool?>[2];
                // cluster
                {
                    context.LocalDebug = false;
                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
                    result[0] = output.ToArray();
                }
                // local
                {
                    context.LocalDebug = true;
                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
                    result[1] = output.ToArray();
                }

                // compare result
                try
                {
                    Validate.Check(result);
                }
                catch (Exception ex)
                {
                    TestLog.Message("Error: " + ex.Message);
                    passed &= false;
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }
Beispiel #49
0
 internal VertexCodeGen(DryadLinqContext context)
 {
     this.m_context = context;
 }
 internal static DryadLinqMetaData Get(DryadLinqContext context, Uri dataSetUri)
 {
     string scheme = DataPath.GetScheme(dataSetUri);
     DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
     return dataProvider.GetMetaData(context, dataSetUri);
 }
Beispiel #51
0
        /// <summary>
        /// Gets the <see cref="DryadLinqFactory{T}"/> for a specified type. If a factory doesn't exist,
        /// the method generates the serialization code and creates a new factory for the type.
        /// </summary>
        /// <param name="context">An instnance of <see cref="DryadLinqContext"/></param>
        /// <param name="type">A specified type</param>
        /// <returns>A <see cref="DryadLinqFactory{T}"/> for the type</returns>
        public static object GetFactory(DryadLinqContext context, Type type)
        {
            lock (s_codeGenLock)
            {
                if (s_TypeToFactory.ContainsKey(type))
                {
                    return s_TypeToFactory[type];
                }
                
                DryadLinqCodeGen codeGen = new DryadLinqCodeGen(context, new VertexCodeGen(context));
                codeGen.AddDryadCodeForType(type);
                
                // build assembly, and load into memory, because we'll next instantiate
                // the factory type out of the generated assembly.
                codeGen.BuildAssembly(true);

                string factoryTypeFullName = TargetNamespace + "." + DryadLinqFactoryClassName(type);
                object factory = codeGen.m_loadedVertexAssembly.CreateInstance(factoryTypeFullName);
                s_TypeToFactory.Add(type, factory);
                return factory;
            }
        }
Beispiel #52
0
        internal DryadLinqCodeGen(DryadLinqContext context, VertexCodeGen vertexCodeGen)
        {
            this.m_context = context;
            this.m_vertexCodeGen = vertexCodeGen;
            this.m_loadedVertexAssembly = null;
            this.m_dryadLinqUnit = new CodeCompileUnit();

            // Create a namespace
            this.m_dryadCodeSpace = new CodeNamespace(TargetNamespace);
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections.Generic"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Text"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq.Expressions"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Diagnostics"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Runtime.Serialization"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.SqlTypes"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq.Mapping"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq"));
            this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq.Internal"));

            this.m_dryadLinqUnit.Namespaces.Add(this.m_dryadCodeSpace);

            // Create the class for all the DryadLinq extension methods
            this.m_dryadExtensionClass = new CodeTypeDeclaration(ExtensionClassName);
            this.m_dryadExtensionClass.IsClass = true;
            this.m_dryadExtensionClass.IsPartial = true;
            this.m_dryadExtensionClass.TypeAttributes = TypeAttributes.Public;
            this.m_dryadCodeSpace.Types.Add(this.m_dryadExtensionClass);

            // Create the static constructor for the vertex extension class
            this.m_extensionStaticCtor = new CodeTypeConstructor();
            this.m_dryadExtensionClass.Members.Add(this.m_extensionStaticCtor);

            // Create the class for all the DryadLinq vertex methods
            this.m_dryadVertexClass = new CodeTypeDeclaration(VertexClassName);
            this.m_dryadVertexClass.IsClass = true;
            this.m_dryadVertexClass.TypeAttributes = TypeAttributes.Public | TypeAttributes.Sealed;
            this.m_dryadCodeSpace.Types.Add(this.m_dryadVertexClass);
            this.AddCopyResourcesMethod();

            // The set of input/output channel datatypes
            this.m_dryadDataTypes = new HashSet<Type>();
            this.m_dryadDataTypes.Add(typeof(byte));
            this.m_dryadDataTypes.Add(typeof(sbyte));
            this.m_dryadDataTypes.Add(typeof(bool));
            this.m_dryadDataTypes.Add(typeof(char));
            this.m_dryadDataTypes.Add(typeof(short));
            this.m_dryadDataTypes.Add(typeof(ushort));
            this.m_dryadDataTypes.Add(typeof(int));
            this.m_dryadDataTypes.Add(typeof(uint));
            this.m_dryadDataTypes.Add(typeof(long));
            this.m_dryadDataTypes.Add(typeof(ulong));
            this.m_dryadDataTypes.Add(typeof(float));
            this.m_dryadDataTypes.Add(typeof(decimal));
            this.m_dryadDataTypes.Add(typeof(double));
            this.m_dryadDataTypes.Add(typeof(DateTime));      
            this.m_dryadDataTypes.Add(typeof(string));
            this.m_dryadDataTypes.Add(typeof(LineRecord));
            this.m_dryadDataTypes.Add(typeof(SqlDateTime));
            this.m_dryadDataTypes.Add(typeof(Guid)); 
            
            // The set of datatypes we have added serialization methods
            this.m_serializationDatatypes = new HashSet<Type>();

            this.m_fieldToStaticName = new Dictionary<FieldInfo, string>();
            this.m_staticFieldDefined = new HashSet<string>();
            this.m_typeToSerializerName = new Dictionary<Type, string>();
            this.m_anonymousTypeToName = new Dictionary<Type, string>();
            this.m_nameToAlias = new Dictionary<string, string>();
        }
 public YarnJobSubmission(DryadLinqContext context, Uri baseStagingUri) : base(context)
 {
     this.baseStagingUri = baseStagingUri;
 }
        public static bool Bug14190_MergeJoin_DecreasingOrder(DryadLinqContext context) // TODO
        {
            string testName = "Bug14190_MergeJoin_DecreasingOrder";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                IEnumerable<int>[] result = new IEnumerable<int>[2];
                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/Bug14190_MergeJoin_DecreasingOrder";

                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                }
                // local
                {
                    context.LocalDebug = true;
                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                }

                // compare result
                try
                {
                    Validate.Check(result);
                }
                catch (Exception ex)
                {
                    TestLog.Message("Error: " + ex.Message);
                    passed &= false;
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }
Beispiel #55
0
 internal static bool DoAutoTypeInference(DryadLinqContext context, Type type)
 {
     if (!StaticConfig.AllowAutoTypeInference) return false;
     object[] a = type.GetCustomAttributes(typeof(AutoTypeInferenceAttribute), true);
     return (a.Length != 0);
 }
        public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
        {
            string testName = "Bug15371_NoDataMembersForSerialization";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";

                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    IQueryable<NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
                    var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
                    jobInfo.Wait();
                    var result = context.FromStore<NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
                    passed &= false;
                }
            }
            catch (DryadLinqException Ex)
            {
                passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
                           Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }
        public static bool Bug14192_MultiApplySubExpressionReuse(DryadLinqContext context)
        {
            string testName = "Bug14192_MultiApplySubExpressionReuse";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                IEnumerable<int>[] result = new IEnumerable<int>[2];
                // cluster
                {
                    context.LocalDebug = false;
                    string outFile = "unittest/output/Bug14192_MultiApplySubExpressionReuse";

                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    var data = pt1.Select(x => x);

                    var info = pt1.Apply(new[] { pt1, pt1 }, (sources) => new int[] { 1, 2, 3 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
                    info.Wait();

                    // ToDo
                    //string queryPlan = TestUtils.GetRecentQueryXmlAsText();
                    //int nVerticesInPlan = TestUtils.GetVertexStageCount(queryPlan);

                    //passed &= (nVerticesInPlan == 7); // "Only seven vertices should appear (before bug, there were 10 of which the last three were extraneous.");
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }
        public static bool Bug14189_OrderPreservation(DryadLinqContext context)
        {
            string testName = "Bug14189_OrderPreservation";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                IGrouping<int, int>[] clusterSorted, localSorted;
                // cluster
                {
                    context.LocalDebug = false;
                    IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
                    var output = pt1.OrderBy(x => x % 4)
                                .Select(x => x).Select(x => x).Where(x => true)  // this pipeline was not preserving order correctly. 
                                .GroupBy(x => x % 4)
                                .ToArray();

                    passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
                    // sort back on the key for deterministic output.
                    clusterSorted = output.OrderBy(x => x.Key).ToArray();
                }

                // local
                {
                    context.LocalDebug = true;
                    IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
                    var output = pt1.OrderBy(x => x % 4)
                                .Select(x => x).Select(x => x).Where(x => true)  // this pipeline was not preserving order correctly. 
                                .GroupBy(x => x % 4)
                                .ToArray();

                    passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
                    // sort back on the key for deterministic output.
                    localSorted = output.OrderBy(x => x.Key).ToArray();
                }

                // check that each group of output has the same elements as the LINQ groups.
                for (int i = 0; i < 4; i++)
                {
                    var a = clusterSorted[i].OrderBy(x => x);
                    var b = localSorted[i].OrderBy(x => x);

                    passed &= a.SequenceEqual(b); //the output should match linq. Error for group: + i);
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }
Beispiel #59
0
        internal static bool RecordCanBeNull(DryadLinqContext context, Type type)
        {
            if (type == null || type.IsValueType) return false;

            object[] attribs = type.GetCustomAttributes(typeof(NullableAttribute), true);
            if (attribs.Length == 0)
            {
                return StaticConfig.AllowNullRecords;
            }
            return ((NullableAttribute)attribs[0]).CanBeNull;
        }
        public static bool Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(DryadLinqContext context)
        {
            string testName = "Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes";
            TestLog.TestStart(testName);

            bool passed = true;
            try
            {
                IEnumerable<int>[] result = new IEnumerable<int>[2];
                // cluster
                {
                    context.LocalDebug = false;
                    IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
                    // use an anonymous type with nullable fields, and make sure we at least have some null values in the stream
                    var output = pt1.Select(x => new { FirstField = (x % 2 == 0) ? new int?(x) : default(int?), SecondField = x.ToString() })
                                    .GroupBy(x => x.FirstField, y => y.SecondField);     // use of GB ensures we exercise the emitted GetHashCode() overload
                    passed &= (output.Count() != 0); // "Query return 0 length output"
                }
            }
            catch (Exception Ex)
            {
                TestLog.Message("Error: " + Ex.Message);
                passed &= false;
            }

            TestLog.LogResult(new TestResult(testName, context, passed));
            return passed;
        }