/// <summary>Return true iff the tuple is full (all data sources contain this key).</summary> protected internal override bool Combine(object[] srcs, TupleWritable dst) { System.Diagnostics.Debug.Assert(srcs.Length == dst.Size()); for (int i = 0; i < srcs.Length; ++i) { if (!dst.Has(i)) { return(false); } } return(true); }
/// <exception cref="System.Exception"/> public virtual void TestWideTuple2() { Text emptyText = new Text("Should be empty"); Writable[] values = new Writable[64]; Arrays.Fill(values, emptyText); values[9] = new Text("Number 9"); TupleWritable tuple = new TupleWritable(values); tuple.SetWritten(9); for (int pos = 0; pos < tuple.Size(); pos++) { bool has = tuple.Has(pos); if (pos == 9) { NUnit.Framework.Assert.IsTrue(has); } else { NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: " + pos, has); } } }
/// <summary>Tests that we can write more than 64 values.</summary> /// <exception cref="System.Exception"/> public virtual void TestWideTupleBoundary() { Text emptyText = new Text("Should not be set written"); Writable[] values = new Writable[65]; Arrays.Fill(values, emptyText); values[64] = new Text("Should be the only value set written"); TupleWritable tuple = new TupleWritable(values); tuple.SetWritten(64); for (int pos = 0; pos < tuple.Size(); pos++) { bool has = tuple.Has(pos); if (pos == 64) { NUnit.Framework.Assert.IsTrue(has); } else { NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: " + pos, has); } } }
private void ValidateOuterKeyValue(IntWritable k, TupleWritable v, int tupleSize, bool firstTuple, bool secondTuple) { string kvstr = "Unexpected tuple: " + Stringify(k, v); NUnit.Framework.Assert.IsTrue(kvstr, v.Size() == tupleSize); int key = k.Get(); IntWritable val0 = null; IntWritable val1 = null; LongWritable val2 = null; if (firstTuple) { TupleWritable v0 = ((TupleWritable)v.Get(0)); if (key % 2 == 0 && key / 2 <= Items) { val0 = (IntWritable)v0.Get(0); } else { NUnit.Framework.Assert.IsFalse(kvstr, v0.Has(0)); } if (key % 3 == 0 && key / 3 <= Items) { val1 = (IntWritable)v0.Get(1); } else { NUnit.Framework.Assert.IsFalse(kvstr, v0.Has(1)); } if (key % 4 == 0 && key / 4 <= Items) { val2 = (LongWritable)v.Get(1); } else { NUnit.Framework.Assert.IsFalse(kvstr, v.Has(2)); } } else { if (secondTuple) { if (key % 2 == 0 && key / 2 <= Items) { val0 = (IntWritable)v.Get(0); } else { NUnit.Framework.Assert.IsFalse(kvstr, v.Has(0)); } TupleWritable v1 = ((TupleWritable)v.Get(1)); if (key % 3 == 0 && key / 3 <= Items) { val1 = (IntWritable)v1.Get(0); } else { NUnit.Framework.Assert.IsFalse(kvstr, v1.Has(0)); } if (key % 4 == 0 && key / 4 <= Items) { val2 = (LongWritable)v1.Get(1); } else { NUnit.Framework.Assert.IsFalse(kvstr, v1.Has(1)); } } else { if (key % 2 == 0 && key / 2 <= Items) { val0 = (IntWritable)v.Get(0); } else { NUnit.Framework.Assert.IsFalse(kvstr, v.Has(0)); } if (key % 3 == 0 && key / 3 <= Items) { val1 = (IntWritable)v.Get(1); } else { NUnit.Framework.Assert.IsFalse(kvstr, v.Has(1)); } if (key % 4 == 0 && key / 4 <= Items) { val2 = (LongWritable)v.Get(2); } else { NUnit.Framework.Assert.IsFalse(kvstr, v.Has(2)); } } } if (val0 != null) { NUnit.Framework.Assert.IsTrue(kvstr, val0.Get() == 0); } if (val1 != null) { NUnit.Framework.Assert.IsTrue(kvstr, val1.Get() == 1); } if (val2 != null) { NUnit.Framework.Assert.IsTrue(kvstr, val2.Get() == 2); } }
/// <exception cref="System.Exception"/> public virtual void TestNestedJoin() { // outer(inner(S1,...,Sn),outer(S1,...Sn)) int Sources = 3; int Items = (Sources + 1) * (Sources + 1); Configuration conf = new Configuration(); Path @base = cluster.GetFileSystem().MakeQualified(new Path("/nested")); int[][] source = new int[Sources][]; for (int i = 0; i < Sources; ++i) { source[i] = new int[Items]; for (int j = 0; j < Items; ++j) { source[i][j] = (i + 2) * (j + 1); } } Path[] src = new Path[Sources]; SequenceFile.Writer[] @out = CreateWriters(@base, conf, Sources, src); IntWritable k = new IntWritable(); for (int i_1 = 0; i_1 < Sources; ++i_1) { IntWritable v = new IntWritable(); v.Set(i_1); for (int j = 0; j < Items; ++j) { k.Set(source[i_1][j]); @out[i_1].Append(k, v); } @out[i_1].Close(); } @out = null; StringBuilder sb = new StringBuilder(); sb.Append("outer(inner("); for (int i_2 = 0; i_2 < Sources; ++i_2) { sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_2]. ToString())); if (i_2 + 1 != Sources) { sb.Append(","); } } sb.Append("),outer("); sb.Append(CompositeInputFormat.Compose(typeof(MapReduceTestUtil.Fake_IF), "foobar" )); sb.Append(","); for (int i_3 = 0; i_3 < Sources; ++i_3) { sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_3]. ToString())); sb.Append(","); } sb.Append(CompositeInputFormat.Compose(typeof(MapReduceTestUtil.Fake_IF), "raboof" ) + "))"); conf.Set(CompositeInputFormat.JoinExpr, sb.ToString()); MapReduceTestUtil.Fake_IF.SetKeyClass(conf, typeof(IntWritable)); MapReduceTestUtil.Fake_IF.SetValClass(conf, typeof(IntWritable)); Job job = Job.GetInstance(conf); Path outf = new Path(@base, "out"); FileOutputFormat.SetOutputPath(job, outf); job.SetInputFormatClass(typeof(CompositeInputFormat)); job.SetMapperClass(typeof(Mapper)); job.SetReducerClass(typeof(Reducer)); job.SetNumReduceTasks(0); job.SetOutputKeyClass(typeof(IntWritable)); job.SetOutputValueClass(typeof(TupleWritable)); job.SetOutputFormatClass(typeof(SequenceFileOutputFormat)); job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful()); FileStatus[] outlist = cluster.GetFileSystem().ListStatus(outf, new Utils.OutputFileUtils.OutputFilesFilter ()); NUnit.Framework.Assert.AreEqual(1, outlist.Length); NUnit.Framework.Assert.IsTrue(0 < outlist[0].GetLen()); SequenceFile.Reader r = new SequenceFile.Reader(cluster.GetFileSystem(), outlist[ 0].GetPath(), conf); TupleWritable v_1 = new TupleWritable(); while (r.Next(k, v_1)) { NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(0)); NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(Sources + 1)); bool chk = true; int ki = k.Get(); for (int i_4 = 2; i_4 < Sources + 2; ++i_4) { if ((ki % i_4) == 0 && ki <= i_4 * Items) { NUnit.Framework.Assert.AreEqual(i_4 - 2, ((IntWritable)((TupleWritable)v_1.Get(1) ).Get((i_4 - 1))).Get()); } else { chk = false; } } if (chk) { // present in all sources; chk inner NUnit.Framework.Assert.IsTrue(v_1.Has(0)); for (int i_5 = 0; i_5 < Sources; ++i_5) { NUnit.Framework.Assert.IsTrue(((TupleWritable)v_1.Get(0)).Has(i_5)); } } else { // should not be present in inner join NUnit.Framework.Assert.IsFalse(v_1.Has(0)); } } r.Close(); @base.GetFileSystem(conf).Delete(@base, true); }