public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" ApplyAndForkTests ");
TestLog.Message(" **********************");
var tests = new Dictionary <string, Action>()
{
{ "NonHomomorphicUnaryApply", () => NonHomomorphicUnaryApply(context) },
{ "HomomorphicUnaryApply", () => HomomorphicUnaryApply(context) },
{ "NonHomomorphicBinaryApply", () => NonHomomorphicBinaryApply(context) },
{ "LeftHomomorphicBinaryApply", () => LeftHomomorphicBinaryApply(context) },
{ "FullHomomorphicBinaryApply_DifferentDataSets", () => FullHomomorphicBinaryApply_DifferentDataSets(context) },
{ "FullHomomorphicBinaryApply_IdenticalDataSets", () => FullHomomorphicBinaryApply_IdenticalDataSets(context) },
{ "Aggregate_WithCombiner", () => Aggregate_WithCombiner(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static bool RP_rangeSeparators_nullCustomComparer(DryadLinqContext context)
{
string testName = "RP_rangeSeparators_nullCustomComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var results1 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null, false).ToArray();
var results2 = pt1.RangePartition(x => x, new int[] { 1, 2, 3, 4 }, null).ToArray();
//passing is not throwing.
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool GetEnumeratorNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> q1 = pt1.Select(x => 100 + x);
IQueryable<int> q2 = q1.Where(x => true);
foreach (int x in q2) // throws here
{
//Console.WriteLine(x);
}
//@TODO: perform a sequence-equals test.
//IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
//DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
// "unittest/output/test2.txt")).SubmitAndWait();
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool GroupByReduce_BuiltIn_First()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
int[] aggregates = data
.GroupBy(x => 0, (key, seq) => seq.First())
.ToArray();
// the output of First can be the first item of either partition.
passed &= aggregates.SequenceEqual(new[] { 1 }) || aggregates.SequenceEqual(new[] { 101 });
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool DistributiveSelect_1()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
// this result selector satisfies "DistributiveOverConcat"
int[] aggregates = pt1.GroupBy(x => x % 2).Select(group => group.Sum()).ToArray();
int[] expected = new[] { 1 + 3 + 5 + 7 + 9 + 11, 2 + 4 + 6 + 8 + 10 + 12 };
//note the order of the result elements is not guaranteed, so order them before testing
int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
int[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug13970_MismatchedDataTypes(DryadLinqContext context)
{
string testName = "Bug13970_MismatchedDataTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
// local
{
context.LocalDebug = true;
IQueryable <int> pt1 = DataGenerator.GetGroupByReduceDataSet(context);
double? output = pt1.AverageAsQuery().Single();
passed &= false;
}
}
catch (Exception)
{
passed &= true; // expected
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool GroupByReduce_BitwiseNegationOperator()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> pt1 = simple.Select(x => x.First());
var results = pt1.GroupBy(x => x % 2, (k, g) => new KeyValuePair <int, int>(k, ~g.Sum())).ToArray();
//local sort.. so that keys are in order.
var resultsSorted = results.OrderBy(list => list.Key).ToArray();
//key0: count = 6, sum = 42
//key1: count = 6, sum = 36
passed &= (resultsSorted[0].Key == 0); // "incorrect results.1"
passed &= (resultsSorted[0].Value == ~42); // "incorrect results.2"
passed &= (resultsSorted[1].Key == 1); // "incorrect results.3"
passed &= (resultsSorted[1].Value == ~36); // "incorrect results.4"
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool EnumeratePlainData()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
foreach (int x in pt1) // throws
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool CopyPlainDataViaToStoreMaterialize()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/CopyPlainDataViaToStoreMaterialize.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q = pt1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
foreach (int x in q)
{
//Console.WriteLine(x);
}
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool SubmitNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x);
var q2 = q1.Where(x => true);
q2.SubmitAndWait(); // throws here
var outPT = q2.ToList();
foreach (int x in outPT)
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool MaterializeNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> query = pt1.Select(x => 100 + x);
DryadLinqQueryable.Submit(query); //materialize // throws
foreach (int x in query)
{
//Console.WriteLine(x);
}
//@TODO: assert that only one query execution occurred.
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool template()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/x.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool HomomorphicUnaryApply()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/HomomorphicUnaryApply.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.ApplyPerPartition(x => Homomorphic_Unary_Func(x));
var jobInfo = q1.ToStore <int>(outFile).Submit();
jobInfo.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Aggregate_WithCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Aggregate_WithCombiner(DryadLinqContext context)
{
string testName = "Aggregate_WithCombiner";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
string q1 = pt1.Aggregate("", (str, x) => IntToStringCSVAggregator(str, x));
passed &= (q1.Length == 27); // string should have numbers 1..12 separated by commas
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool ToStoreGetEnumeratorThrows() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> q1 = pt1.Select(x => 100 + x);
var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
output.GetEnumerator();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public PeloponneseJobSubmission(DryadLinqContext context)
{
m_context = context;
m_localResources = new Dictionary<string, HashSet<string>>();
m_peloponneseGMFiles = Peloponnese.ClusterUtils.ConfigHelpers
.ListPeloponneseResources(context.PeloponneseHomeDirectory)
.Select(r => r.ToLower());
IEnumerable<string> graphManagerFiles = Peloponnese.Shared.DependencyLister.Lister
.ListDependencies(Path.Combine(context.DryadHomeDirectory, "Microsoft.Research.Dryad.GraphManager.exe"))
.Select(r => r.ToLower());
m_dryadGMFiles = graphManagerFiles.Except(m_peloponneseGMFiles);
string[] additionalWorkerFiles =
{
"Microsoft.Research.Dryadlinq.dll",
"Microsoft.Research.Dryad.DryadLinq.NativeWrapper.dll"
};
IEnumerable<string> processServiceFiles = Peloponnese.Shared.DependencyLister.Lister
.ListDependencies(Path.Combine(context.DryadHomeDirectory, "Microsoft.Research.Dryad.ProcessService.exe"))
.Select(r => r.ToLower());
IEnumerable<string> vertexHostFiles = Peloponnese.Shared.DependencyLister.Lister
.ListDependencies(Path.Combine(context.DryadHomeDirectory, "Microsoft.Research.Dryad.VertexHost.exe"))
.Concat(additionalWorkerFiles.Select(f => Path.Combine(context.DryadHomeDirectory, f)))
.Select(r => r.ToLower());
IEnumerable<string> workerFiles = processServiceFiles.Union(vertexHostFiles);
m_peloponneseWorkerFiles = workerFiles.Intersect(m_peloponneseGMFiles);
m_dryadWorkerFiles = workerFiles.Except(m_peloponneseGMFiles);
}
public static bool GetEnumeratorNonToStoreTerminated()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
IQueryable <int> q1 = pt1.Select(x => 100 + x);
IQueryable <int> q2 = q1.Where(x => true);
foreach (int x in q2) // throws here
{
//Console.WriteLine(x);
}
//@TODO: perform a sequence-equals test.
//IQueryable<LineRecord> format = q2.Select(x => new LineRecord(String.Format("{0}", x)));
//DryadLinqJobInfo output = format.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
// "unittest/output/test2.txt")).SubmitAndWait();
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool BuiltInCountIsDistributable()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
// Built in Count is Distributable as built-in logic knows to use Sum() as the combiner function.
// Count(a,b,c,d) = Sum(Count(a,b), Count(c,d))
int[] aggregates = pt1.GroupBy(x => x % 2, (key, seq) => seq.Count()).ToArray();
int[] expected = new[] { 6, 6 }; // six elements in each full group.
//note the order of the result elements is not guaranteed, so order them before testing
int[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
int[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11782_Aggregate()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/Bug11782_Aggregate.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
//test Aggregate()
var c = pt1.Select(x => x).Aggregate((x, y) => x + y);
//test AggregateAsQuery()
var q = pt1.Select(x => x).AggregateAsQuery((x, y) => x + y).ToStore(outFile);
DryadLinqJobInfo info = DryadLinqQueryable.Submit(q);
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool GroupByReduceWithCustomDecomposableFunction_NonDistributableCombiner()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
double[] aggregates = data
.Select(x => (double)x)
.GroupBy(x => 0, (key, seq) => DecomposableFunc4(seq)).ToArray();
double[] expected = new[] { Enumerable.Range(1, 200).Sum() / 100.0 };
//note the order of the result elements is not guaranteed, so order them before testing
double[] aggregatesOrdered = aggregates.OrderBy(x => x).ToArray();
double[] expectedOrdered = expected.OrderBy(x => x).ToArray();
passed &= aggregatesOrdered.SequenceEqual(expectedOrdered);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool Bug11782_LowLevelQueryableManipulation()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
Expression lambda = Expression.Lambda <Func <int, int> >(
Expression.Constant(1),
new[] { Expression.Parameter(typeof(int), "x") });
var z = pt1.Provider.CreateQuery(
Expression.Call(
typeof(Queryable), "Select",
new Type[] { pt1.ElementType, pt1.ElementType },
pt1.Expression, Expression.Quote(lambda)));
passed &= false; // the use of non-generic Provider.CreateQuery() should have thrown
}
catch (DryadLinqException)
{
passed &= true;
}
return(passed);
}
public static bool GroupByReduce_ResultSelector_ComplexNewExpression()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateGroupByReduceDataSet());
IQueryable <int> data = simple.Select(x => x.First());
var aggregates = data.GroupBy(x => 0, (key, seq) => new KeyValuePair <int, KeyValuePair <double, double> >(key, new KeyValuePair <double, double>(seq.Average(), seq.Average()))).ToArray();
var expected = new KeyValuePair <int, KeyValuePair <double, double> >[] { new KeyValuePair <int, KeyValuePair <double, double> >(0, new KeyValuePair <double, double>(100.5, 100.5)) };
passed &= aggregates.SequenceEqual(expected);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool AssumeRangePartition()
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/BasicAPITests_AssumeRangePartition.out";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q =
pt1
.AssumeRangePartition(x => x, false)
.Select(x => 100 + x).ToStore(outFile);
var info = q.Submit();
info.Wait();
passed &= Utils.FileExists(Config.accountName, Config.storageKey, Config.containerName, outFile);
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" RangePartitionAPICoverageTests ");
TestLog.Message(" **********************");
var tests = new Dictionary <string, Action>()
{
{ "RP_keySelector", () => RP_keySelector(context) },
{ "RP_keySelector_pcount", () => RP_keySelector_pcount(context) },
{ "RP_keySelector_isDescending", () => RP_keySelector_isDescending(context) },
{ "RP_keySelector_rangeKeys", () => RP_keySelector_rangeKeys(context) },
{ "RP_keySelector_isDescending_pcount", () => RP_keySelector_isDescending_pcount(context) },
{ "RP_keySelector_keyComparer_isDescending", () => RP_keySelector_keyComparer_isDescending(context) },
{ "RP_keySelector_rangeKeys_keyComparer", () => RP_keySelector_rangeKeys_keyComparer(context) },
{ "RP_keySelector_keyComparer_isDescending_pcount", () => RP_keySelector_keyComparer_isDescending_pcount(context) },
{ "RP_keySelector_rangeKeys_keyComparer_isDescending", () => RP_keySelector_rangeKeys_keyComparer_isDescending(context) },
{ "RP_singlePartition_autoSeparators", () => RP_singlePartition_autoSeparators(context) },
{ "RP_rangeSeparators_customComparer", () => RP_rangeSeparators_customComparer(context) },
{ "RP_rangeSeparators_nullCustomComparer", () => RP_rangeSeparators_nullCustomComparer(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static bool ToStoreSubmitGetEnumerator() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreSubmitGetEnumerator.txt";
IQueryable <LineRecord> input = context.FromStore <LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable <IEnumerable <int> > simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable <int> pt1 = simple.Select(x => x.First());
var q1 = pt1.Select(x => 100 + x).HashPartition(x => x);
var q2 = q1.Where(x => true);
IQueryable <int> output = q2.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile), true);
DryadLinqJobInfo info = output.SubmitAndWait();
foreach (int x in output) // should not run a new dryad job.
{
//Console.WriteLine(x);
}
}
catch (DryadLinqException)
{
passed &= false;
}
return(passed);
}
public static bool ToStoreGetEnumeratorThrows() // pass
{
var context = new DryadLinqContext(Config.cluster);
context.LocalExecution = false;
bool passed = true;
try
{
string outFile = "unittest/output/ToStoreGetEnumeratorThrows.txt";
IQueryable<LineRecord> input = context.FromStore<LineRecord>(AzureUtils.ToAzureUri(Config.accountName, Config.containerName,
"unittest/inputdata/SimpleFile.txt"));
IQueryable<IEnumerable<int>> simple = input.Apply(x => DataGenerator.CreateSimpleFileSets());
IQueryable<int> pt1 = simple.Select(x => x.First());
IQueryable<int> q1 = pt1.Select(x => 100 + x);
var output = q1.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true);
output.GetEnumerator();
}
catch (DryadLinqException)
{
passed &= false;
}
return passed;
}
public static bool Bug14870_LongIndexTakeWhile(DryadLinqContext context)
{
string testName = "Bug14870_LongIndexTakeWhile";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
// ToDo - move to data generator
int[][] data = new[] {
new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { }, new int[] { }
};
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.LongTakeWhile((x, i) => i < 8).ToArray();
passed &= (output.Length == 8); // "Only eight items should be returned."
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug13108_DisableSequenceEquals(DryadLinqContext context)
{
string testName = "Bug13108_DisableSequenceEquals";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable <int> pt2 = DataGenerator.GetSimpleFileSets(context);
bool result = pt1.Select(x => x).SequenceEqual(pt2.Select(x => x));
passed &= false; // NotSupportedException should have been thrown
}
}
catch (Exception Ex)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
{
string testName = "Bug15371_NoDataMembersForSerialization";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable <NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var result = context.FromStore <NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
passed &= false;
}
}
catch (DryadLinqException Ex)
{
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug13245_FromDsc_Submit_throws(DryadLinqContext context)
{
string testName = "Bug13245_FromDsc_Submit_throws";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Submit();
passed &= false; // should throw ArgEx as the input isn't well formed
}
}
catch (ArgumentException)
{
}
catch (Exception)
{
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug13302_ConfigEnvironmentCleanup(DryadLinqContext context)
{
string testName = "Bug13302_ConfigEnvironmentCleanup";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Apply((en) => CheckEnvVar(en)).Submit();
info.Wait();
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static bool Bug13245_QueryUsingNonHpcLinqOperator(DryadLinqContext context)
{
string testName = "Bug13245_QueryUsingNonHpcLinqOperator";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable <int>[] result = new IEnumerable <int> [2];
// cluster
{
context.LocalDebug = false;
IQueryable <int> pt1 = DataGenerator.GetSimpleFileSets(context);
var info = pt1.Blah().Submit();
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return(passed);
}
public static void Test1(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
//context.PartitionUncPath = "DryadLinqTemp/PartFiles";
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var lines = input.Where(x => x.Line.Contains("white"));
foreach (var x in lines) Console.WriteLine(x);
//var result = lines.ToStore("partfile://svc-yuanbyu-3/DryadLinqTemp/PartFiles/res1.pt", true);
//result.SubmitAndWait();
}
public static void Test2(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var q1 = input.Where(x => x.Line.Contains("white"));
var q2 = input.Where(x => x.Line.Contains("the"));
var res1 = q1.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res1.pt", true);
var res2 = q2.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
DryadLinqQueryable.SubmitAndWait(res1, res2);
}
public static void Test3(string[] args)
{
DryadLinqContext context = new DryadLinqContext(1, "partfile");
var input = context.FromStore<LineRecord>("partfile:///d:/DryadLinqTemp/PartFiles/foo.pt");
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
var result = toOutput.ToStore("partfile:///d:/DryadLinqTemp/PartFiles/res2.pt", true);
result.SubmitAndWait();
}
// This constructor is specifically to support enumeration of a query.
// It assumes that the Expressions all terminate with a ToStore node.
internal DryadLinqQueryGen(DryadLinqContext context,
VertexCodeGen vertexCodeGen,
Expression queryExpr,
Uri tableUri,
bool isTempOutput)
{
this.m_queryExprs = new Expression[] { queryExpr };
this.m_outputTableUris = new Uri[] { tableUri };
this.m_isTempOutput = new bool[] { isTempOutput };
this.m_context = context;
this.Initialize(vertexCodeGen);
}
/// <summary>
/// Create a new job executor object.
/// </summary>
public DryadLinqJobExecutor(DryadLinqContext context)
{
// use a new job submission object for each query
this.m_context = context;
this.m_currentStatus = JobStatus.NotSubmitted;
if (context.LocalExecution)
{
this.m_jobSubmission = new LocalJobSubmission(context);
}
else
{
this.m_jobSubmission = new YarnJobSubmission(context);
}
}
internal DryadLinqMetaData(DryadLinqContext context,
Type recordType,
Uri dataSetUri,
CompressionScheme compressionScheme)
{
this.m_context = context;
this.m_dataSetUri = dataSetUri;
this.m_elemType = recordType;
this.m_compressionScheme = compressionScheme;
//this.m_version = context.ClientVersion();
//this.InitializeFlags();
//this.m_fp = 0UL;
//this.m_dataSetInfo = node.OutputDataSetInfo;
}
/// <summary>
/// Create a new job executor object.
/// </summary>
public DryadLinqJobExecutor(DryadLinqContext context)
{
// use a new job submission object for each query
this.m_context = context;
this.m_currentStatus = JobStatus.NotSubmitted;
if (context.PlatformKind == PlatformKind.LOCAL)
{
this.m_jobSubmission = new LocalJobSubmission(context);
}
else
{
this.m_jobSubmission = new YarnJobSubmission(
context, context.Cluster.MakeInternalClusterUri("tmp", "staging"));
}
}
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" BasicAPITests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"ToStoreThrowsForNonQuery", () => ToStoreThrowsForNonQuery(context) },
{"ToStoreGetEnumerator", () => ToStoreGetEnumerator(context) },
{"GetEnumeratorNonToStoreTerminated", () => GetEnumeratorNonToStoreTerminated(context) },
{"ToStoreSubmitGetEnumerator", () => ToStoreSubmitGetEnumerator(context) },
{"SubmitNonToStoreTerminated", () => SubmitNonToStoreTerminated(context) },
{"MaterializeToStoreTerminated", () => MaterializeToStoreTerminated(context) },
{"MaterializeNonToStoreTerminated", () => MaterializeNonToStoreTerminated(context) },
{"EnumeratePlainData", () => EnumeratePlainData(context) },
{"CopyPlainDataViaToStoreSubmit", () => CopyPlainDataViaToStoreSubmit(context) },
{"CopyPlainDataViaToStoreMaterialize", () => CopyPlainDataViaToStoreMaterialize(context) },
{"PlainEnumerableAsDryadQueryToStoreSubmit", () => PlainEnumerableAsDryadQueryToStoreSubmit(context) },
{"RepeatSubmit", () => RepeatSubmit(context) },
// ToDo {"RepeatMaterialize", () => RepeatMaterialize(context) }, hangs
{"MaterializeMentionsSameQueryTwice", () => MaterializeMentionsSameQueryTwice(context) },
{"QueryOnDataBackedDLQ", () => QueryOnDataBackedDLQ(context) },
{"Bug11781_CountandFirstOrDefault", () => Bug11781_CountandFirstOrDefault(context) },
{"Bug11782_Aggregate", () => Bug11782_Aggregate(context) },
{"Bug11782_LowLevelQueryableManipulation", () => Bug11782_LowLevelQueryableManipulation(context) },
{"Bug11638_LongWhere", () => Bug11638_LongWhere(context) },
{"AssumeRangePartition", () => AssumeRangePartition(context) },
{"Bug11638_LongMethods", () => Bug11638_LongMethods(context) },
{"ContextConfigIsReadOnly", () => ContextConfigIsReadOnly(context) },
// ToDo{"ToggleSpeculativeDuplication", () => ToggleSpeculativeDuplication(context) },
{"Bug15068_ConfigResourcesAPI", () => Bug15068_ConfigResourcesAPI(context) },
{"Bug14449_ContextShouldExposeVersionIDs", () => Bug14449_ContextShouldExposeVersionIDs(context) }, // not valid anymore
{"Bug_16341_SubmitThrowsForDifferentContexts", () => Bug_16341_SubmitThrowsForDifferentContexts(context) },
{"Bug_16341_VariousTestsForSubmit", () => Bug_16341_VariousTestsForSubmit(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
public static bool Bug11447_GroupByWithComparer(DryadLinqContext context)
{
string testName = "Bug11447_GroupByWithComparer";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
result[0] = a;
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var a = pt1.GroupBy(x => x, EqualityComparer<int>.Default).SelectMany(x => x).ToList();
result[1] = a;
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static void WordCountExample()
{
#if local
// This overload runs the computation on your local computer using a single worker
var config = new DryadLinqContext(1);
var lines = new LineRecord[] { new LineRecord("This is a dummy line for a short job") };
// You can create inputs from any IEnumerable source using this method
var input = config.FromEnumerable(lines);
#else
string clusterName = "Replace with your HDInsight 3.0 cluster name";
// to use the davinci.txt example input below, select your cluster's default
// storage account and container, which automatically includes the sample text
string accountName = "Replace with a storage account name";
string containerName = "Replace with a storage container name";
// This overload creates an Azure-based computation
var config = new DryadLinqContext(clusterName);
config.JobFriendlyName = "DryadLINQ Sample Wordcount";
// plain text files should be read as type LineRecord
var input = config.FromStore<LineRecord>(AzureUtils.ToAzureUri(accountName, containerName,
"example/data/gutenberg/davinci.txt"));
#endif
var words = input.SelectMany(x => x.Line.Split(' '));
var groups = words.GroupBy(x => x);
var counts = groups.Select(x => new KeyValuePair<string, int>(x.Key, x.Count()));
var toOutput = counts.Select(x => new LineRecord(String.Format("{0}: {1}", x.Key, x.Value)));
#if local
// any collection computed by the query can be materialized back at the client,
// not just the 'output' collection. For large collections this is expensive!
foreach (LineRecord line in toOutput)
{
Console.WriteLine(line.Line);
}
#else
// the 'true' parameter to ToStore means the output will be over-written if you run
// the job more than once
var info = toOutput.ToStore(AzureUtils.ToAzureUri(accountName, containerName,
"wc-out.txt"), true).SubmitAndWait();
#endif
}
public static void Run(DryadLinqContext context, string matchPattern)
{
TestLog.Message(" **********************");
TestLog.Message(" MiscBugFixTests ");
TestLog.Message(" **********************");
var tests = new Dictionary<string, Action>()
{
{"Bug11447_GroupByWithComparer", () => Bug11447_GroupByWithComparer(context) },
{"Bug12584_HashPartitionOutputCount", () => Bug12584_HashPartitionOutputCount(context) },
{"Bug13108_DisableSequenceEquals", () => Bug13108_DisableSequenceEquals(context) },
{"Bug13529_and_Bug13593_IndexedOperatorCompilation", () => Bug13529_and_Bug13593_IndexedOperatorCompilation(context) },
{"Bug13130_ReverseOperator", () => Bug13130_ReverseOperator(context) },
{"Bug13736_IndexedTakeWhile", () => Bug13736_IndexedTakeWhile(context) },
{"Bug13534_HashPartitionNegIndexIsError", () => Bug13534_HashPartitionNegIndexIsError(context) },
{"Bug13474_and_Bug13483_FromDscOnBadFileSet", () => Bug13474_and_Bug13483_FromDscOnBadFileSet(context) },
{"Bug13637_EmptyFilesInFilesets", () => Bug13637_EmptyFilesInFilesets(context) },
{"Bug13637_LocalDebugProducingZeroRecords", () => Bug13637_LocalDebugProducingZeroRecords(context) },
{"Bug13245_FromDsc_Submit_throws", () => Bug13245_FromDsc_Submit_throws(context) },
{"Bug13245_QueryUsingNonHpcLinqOperator", () => Bug13245_QueryUsingNonHpcLinqOperator(context) },
{"Bug13302_ConfigEnvironmentCleanup", () => Bug13302_ConfigEnvironmentCleanup(context) },
{"Bug13970_MismatchedDataTypes", () => Bug13970_MismatchedDataTypes(context) },
{"Bug14010_AlreadyDisposedContext", () => Bug14010_AlreadyDisposedContext(context) },
{"Bug14256_LeaseOnTempDscFileset", () => Bug14256_LeaseOnTempDscFileset(context) },
{"Bug14189_OrderPreservation", () => Bug14189_OrderPreservation(context) },
{"Bug14190_MergeJoin_DecreasingOrder", () => Bug14190_MergeJoin_DecreasingOrder(context) },
{"Bug14192_MultiApplySubExpressionReuse", () => Bug14192_MultiApplySubExpressionReuse(context) },
{"Bug14870_LongIndexTakeWhile", () => Bug14870_LongIndexTakeWhile(context) },
{"Bug15159_NotOperatorForNullableBool", () => Bug15159_NotOperatorForNullableBool(context) },
{"Bug15371_NoDataMembersForSerialization", () => Bug15371_NoDataMembersForSerialization(context) },
{"Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes", () => Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(context) },
};
foreach (var test in tests)
{
if (Regex.IsMatch(test.Key, matchPattern, System.Text.RegularExpressions.RegexOptions.IgnoreCase))
{
test.Value.Invoke();
}
}
}
// create DryadLinqMetaData from a query OutputNode
internal static DryadLinqMetaData Get(DryadLinqContext context, DLinqOutputNode node)
{
DryadLinqMetaData metaData = new DryadLinqMetaData();
if (!DataPath.IsValidDataPath(node.OutputUri))
{
throw new InvalidOperationException();
}
metaData.m_context = context;
metaData.m_dataSetUri = node.OutputUri;
metaData.m_elemType = node.OutputTypes[0];
metaData.m_compressionScheme = node.OutputCompressionScheme;
//metaData.m_version = context.ClientVersion();
//metaData.InitializeFlags();
//metaData.m_fp = 0UL;
//metaData.m_dataSetInfo = node.OutputDataSetInfo;
return metaData;
}
public static bool ToStoreThrowsForNonQuery(DryadLinqContext context)
{
string testName = "ToStoreThrowsForNonQuery";
TestLog.TestStart(testName);
bool passed = true;
try
{
int[] data = new[] { 1, 2, 3 };
var q1 = data.AsQueryable().Select(x => 100 + x).ToStore<int>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName,
"dummy")).ToArray();
//Should throw as we got into DryadLinq via AsQueryable() rather than via context.
passed &= false;
}
catch (Exception Ex)
{
//expected
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14870_LongIndexTakeWhile(DryadLinqContext context)
{
string testName = "Bug14870_LongIndexTakeWhile";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
// ToDo - move to data generator
int[][] data = new[] {
new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { }, new int[] { } };
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.LongTakeWhile((x, i) => i < 8).ToArray();
passed &= (output.Length == 8); // "Only eight items should be returned."
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug15159_NotOperatorForNullableBool(DryadLinqContext context)
{
string testName = "Bug15159_NotOperatorForNullableBool";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<bool?>[] result = new IEnumerable<bool?>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
result[0] = output.ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var output = pt1.Select<int, bool?>(x => x == 1 ? (bool?)null : true).Select(x => !x);
result[1] = output.ToArray();
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
internal VertexCodeGen(DryadLinqContext context)
{
this.m_context = context;
}
internal static DryadLinqMetaData Get(DryadLinqContext context, Uri dataSetUri)
{
string scheme = DataPath.GetScheme(dataSetUri);
DataProvider dataProvider = DataProvider.GetDataProvider(scheme);
return dataProvider.GetMetaData(context, dataSetUri);
}
/// <summary>
/// Gets the <see cref="DryadLinqFactory{T}"/> for a specified type. If a factory doesn't exist,
/// the method generates the serialization code and creates a new factory for the type.
/// </summary>
/// <param name="context">An instnance of <see cref="DryadLinqContext"/></param>
/// <param name="type">A specified type</param>
/// <returns>A <see cref="DryadLinqFactory{T}"/> for the type</returns>
public static object GetFactory(DryadLinqContext context, Type type)
{
lock (s_codeGenLock)
{
if (s_TypeToFactory.ContainsKey(type))
{
return s_TypeToFactory[type];
}
DryadLinqCodeGen codeGen = new DryadLinqCodeGen(context, new VertexCodeGen(context));
codeGen.AddDryadCodeForType(type);
// build assembly, and load into memory, because we'll next instantiate
// the factory type out of the generated assembly.
codeGen.BuildAssembly(true);
string factoryTypeFullName = TargetNamespace + "." + DryadLinqFactoryClassName(type);
object factory = codeGen.m_loadedVertexAssembly.CreateInstance(factoryTypeFullName);
s_TypeToFactory.Add(type, factory);
return factory;
}
}
internal DryadLinqCodeGen(DryadLinqContext context, VertexCodeGen vertexCodeGen)
{
this.m_context = context;
this.m_vertexCodeGen = vertexCodeGen;
this.m_loadedVertexAssembly = null;
this.m_dryadLinqUnit = new CodeCompileUnit();
// Create a namespace
this.m_dryadCodeSpace = new CodeNamespace(TargetNamespace);
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Collections.Generic"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Text"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Linq.Expressions"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Diagnostics"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Runtime.Serialization"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.SqlTypes"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("System.Data.Linq.Mapping"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq"));
this.m_dryadCodeSpace.Imports.Add(new CodeNamespaceImport("Microsoft.Research.DryadLinq.Internal"));
this.m_dryadLinqUnit.Namespaces.Add(this.m_dryadCodeSpace);
// Create the class for all the DryadLinq extension methods
this.m_dryadExtensionClass = new CodeTypeDeclaration(ExtensionClassName);
this.m_dryadExtensionClass.IsClass = true;
this.m_dryadExtensionClass.IsPartial = true;
this.m_dryadExtensionClass.TypeAttributes = TypeAttributes.Public;
this.m_dryadCodeSpace.Types.Add(this.m_dryadExtensionClass);
// Create the static constructor for the vertex extension class
this.m_extensionStaticCtor = new CodeTypeConstructor();
this.m_dryadExtensionClass.Members.Add(this.m_extensionStaticCtor);
// Create the class for all the DryadLinq vertex methods
this.m_dryadVertexClass = new CodeTypeDeclaration(VertexClassName);
this.m_dryadVertexClass.IsClass = true;
this.m_dryadVertexClass.TypeAttributes = TypeAttributes.Public | TypeAttributes.Sealed;
this.m_dryadCodeSpace.Types.Add(this.m_dryadVertexClass);
this.AddCopyResourcesMethod();
// The set of input/output channel datatypes
this.m_dryadDataTypes = new HashSet<Type>();
this.m_dryadDataTypes.Add(typeof(byte));
this.m_dryadDataTypes.Add(typeof(sbyte));
this.m_dryadDataTypes.Add(typeof(bool));
this.m_dryadDataTypes.Add(typeof(char));
this.m_dryadDataTypes.Add(typeof(short));
this.m_dryadDataTypes.Add(typeof(ushort));
this.m_dryadDataTypes.Add(typeof(int));
this.m_dryadDataTypes.Add(typeof(uint));
this.m_dryadDataTypes.Add(typeof(long));
this.m_dryadDataTypes.Add(typeof(ulong));
this.m_dryadDataTypes.Add(typeof(float));
this.m_dryadDataTypes.Add(typeof(decimal));
this.m_dryadDataTypes.Add(typeof(double));
this.m_dryadDataTypes.Add(typeof(DateTime));
this.m_dryadDataTypes.Add(typeof(string));
this.m_dryadDataTypes.Add(typeof(LineRecord));
this.m_dryadDataTypes.Add(typeof(SqlDateTime));
this.m_dryadDataTypes.Add(typeof(Guid));
// The set of datatypes we have added serialization methods
this.m_serializationDatatypes = new HashSet<Type>();
this.m_fieldToStaticName = new Dictionary<FieldInfo, string>();
this.m_staticFieldDefined = new HashSet<string>();
this.m_typeToSerializerName = new Dictionary<Type, string>();
this.m_anonymousTypeToName = new Dictionary<Type, string>();
this.m_nameToAlias = new Dictionary<string, string>();
}
public YarnJobSubmission(DryadLinqContext context, Uri baseStagingUri) : base(context)
{
this.baseStagingUri = baseStagingUri;
}
public static bool Bug14190_MergeJoin_DecreasingOrder(DryadLinqContext context) // TODO
{
string testName = "Bug14190_MergeJoin_DecreasingOrder";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug14190_MergeJoin_DecreasingOrder";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
}
// compare result
try
{
Validate.Check(result);
}
catch (Exception ex)
{
TestLog.Message("Error: " + ex.Message);
passed &= false;
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
internal static bool DoAutoTypeInference(DryadLinqContext context, Type type)
{
if (!StaticConfig.AllowAutoTypeInference) return false;
object[] a = type.GetCustomAttributes(typeof(AutoTypeInferenceAttribute), true);
return (a.Length != 0);
}
public static bool Bug15371_NoDataMembersForSerialization(DryadLinqContext context)
{
string testName = "Bug15371_NoDataMembersForSerialization";
TestLog.TestStart(testName);
bool passed = true;
try
{
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug15371_NoDataMembersForSerialization";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
IQueryable<NoDataMembersClass> output = pt1.Select(x => new NoDataMembersClass());
var jobInfo = output.ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
jobInfo.Wait();
var result = context.FromStore<NoDataMembersClass>(AzureUtils.ToAzureUri(Config.accountName, Config.storageKey, Config.containerName, outFile)).ToArray();
passed &= false;
}
}
catch (DryadLinqException Ex)
{
passed &= (Ex.ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers") ||
Ex.InnerException != null && ((DryadLinqException)Ex.InnerException).ErrorCode == ReflectionHelper.GetDryadLinqErrorCode("TypeMustHaveDataMembers")); // "exception should have been thrown.
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14192_MultiApplySubExpressionReuse(DryadLinqContext context)
{
string testName = "Bug14192_MultiApplySubExpressionReuse";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
string outFile = "unittest/output/Bug14192_MultiApplySubExpressionReuse";
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
var data = pt1.Select(x => x);
var info = pt1.Apply(new[] { pt1, pt1 }, (sources) => new int[] { 1, 2, 3 }).ToStore(AzureUtils.ToAzureUri(Config.accountName, Config.containerName, outFile), true).Submit();
info.Wait();
// ToDo
//string queryPlan = TestUtils.GetRecentQueryXmlAsText();
//int nVerticesInPlan = TestUtils.GetVertexStageCount(queryPlan);
//passed &= (nVerticesInPlan == 7); // "Only seven vertices should appear (before bug, there were 10 of which the last three were extraneous.");
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
public static bool Bug14189_OrderPreservation(DryadLinqContext context)
{
string testName = "Bug14189_OrderPreservation";
TestLog.TestStart(testName);
bool passed = true;
try
{
IGrouping<int, int>[] clusterSorted, localSorted;
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var output = pt1.OrderBy(x => x % 4)
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
.GroupBy(x => x % 4)
.ToArray();
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
// sort back on the key for deterministic output.
clusterSorted = output.OrderBy(x => x.Key).ToArray();
}
// local
{
context.LocalDebug = true;
IQueryable<int> pt1 = DataGenerator.GetRangePartitionDataSet(context);
var output = pt1.OrderBy(x => x % 4)
.Select(x => x).Select(x => x).Where(x => true) // this pipeline was not preserving order correctly.
.GroupBy(x => x % 4)
.ToArray();
passed &= (output.Count() == output.Select(x => x.Key).Distinct().Count()); // "each group should have a distinct key");
// sort back on the key for deterministic output.
localSorted = output.OrderBy(x => x.Key).ToArray();
}
// check that each group of output has the same elements as the LINQ groups.
for (int i = 0; i < 4; i++)
{
var a = clusterSorted[i].OrderBy(x => x);
var b = localSorted[i].OrderBy(x => x);
passed &= a.SequenceEqual(b); //the output should match linq. Error for group: + i);
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}
internal static bool RecordCanBeNull(DryadLinqContext context, Type type)
{
if (type == null || type.IsValueType) return false;
object[] attribs = type.GetCustomAttributes(typeof(NullableAttribute), true);
if (attribs.Length == 0)
{
return StaticConfig.AllowNullRecords;
}
return ((NullableAttribute)attribs[0]).CanBeNull;
}
public static bool Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes(DryadLinqContext context)
{
string testName = "Bug15570_GetHashCodeAndEqualsForNullableFieldsOfAnonymousTypes";
TestLog.TestStart(testName);
bool passed = true;
try
{
IEnumerable<int>[] result = new IEnumerable<int>[2];
// cluster
{
context.LocalDebug = false;
IQueryable<int> pt1 = DataGenerator.GetSimpleFileSets(context);
// use an anonymous type with nullable fields, and make sure we at least have some null values in the stream
var output = pt1.Select(x => new { FirstField = (x % 2 == 0) ? new int?(x) : default(int?), SecondField = x.ToString() })
.GroupBy(x => x.FirstField, y => y.SecondField); // use of GB ensures we exercise the emitted GetHashCode() overload
passed &= (output.Count() != 0); // "Query return 0 length output"
}
}
catch (Exception Ex)
{
TestLog.Message("Error: " + Ex.Message);
passed &= false;
}
TestLog.LogResult(new TestResult(testName, context, passed));
return passed;
}