/// <summary>Return true iff the tuple is full (all data sources contain this key).</summary>
protected internal override bool Combine(object[] srcs, TupleWritable dst)
{
System.Diagnostics.Debug.Assert(srcs.Length == dst.Size());
for (int i = 0; i < srcs.Length; ++i)
{
if (!dst.Has(i))
{
return(false);
}
}
return(true);
}
/// <exception cref="System.Exception"/>
public virtual void TestWideTuple2()
{
Text emptyText = new Text("Should be empty");
Writable[] values = new Writable[64];
Arrays.Fill(values, emptyText);
values[9] = new Text("Number 9");
TupleWritable tuple = new TupleWritable(values);
tuple.SetWritten(9);
for (int pos = 0; pos < tuple.Size(); pos++)
{
bool has = tuple.Has(pos);
if (pos == 9)
{
NUnit.Framework.Assert.IsTrue(has);
}
else
{
NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: "
+ pos, has);
}
}
}
/// <summary>Tests that we can write more than 64 values.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestWideTupleBoundary()
{
Text emptyText = new Text("Should not be set written");
Writable[] values = new Writable[65];
Arrays.Fill(values, emptyText);
values[64] = new Text("Should be the only value set written");
TupleWritable tuple = new TupleWritable(values);
tuple.SetWritten(64);
for (int pos = 0; pos < tuple.Size(); pos++)
{
bool has = tuple.Has(pos);
if (pos == 64)
{
NUnit.Framework.Assert.IsTrue(has);
}
else
{
NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: "
+ pos, has);
}
}
}
private void ValidateOuterKeyValue(IntWritable k, TupleWritable v, int tupleSize,
bool firstTuple, bool secondTuple)
{
string kvstr = "Unexpected tuple: " + Stringify(k, v);
NUnit.Framework.Assert.IsTrue(kvstr, v.Size() == tupleSize);
int key = k.Get();
IntWritable val0 = null;
IntWritable val1 = null;
LongWritable val2 = null;
if (firstTuple)
{
TupleWritable v0 = ((TupleWritable)v.Get(0));
if (key % 2 == 0 && key / 2 <= Items)
{
val0 = (IntWritable)v0.Get(0);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v0.Has(0));
}
if (key % 3 == 0 && key / 3 <= Items)
{
val1 = (IntWritable)v0.Get(1);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v0.Has(1));
}
if (key % 4 == 0 && key / 4 <= Items)
{
val2 = (LongWritable)v.Get(1);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v.Has(2));
}
}
else
{
if (secondTuple)
{
if (key % 2 == 0 && key / 2 <= Items)
{
val0 = (IntWritable)v.Get(0);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v.Has(0));
}
TupleWritable v1 = ((TupleWritable)v.Get(1));
if (key % 3 == 0 && key / 3 <= Items)
{
val1 = (IntWritable)v1.Get(0);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v1.Has(0));
}
if (key % 4 == 0 && key / 4 <= Items)
{
val2 = (LongWritable)v1.Get(1);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v1.Has(1));
}
}
else
{
if (key % 2 == 0 && key / 2 <= Items)
{
val0 = (IntWritable)v.Get(0);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v.Has(0));
}
if (key % 3 == 0 && key / 3 <= Items)
{
val1 = (IntWritable)v.Get(1);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v.Has(1));
}
if (key % 4 == 0 && key / 4 <= Items)
{
val2 = (LongWritable)v.Get(2);
}
else
{
NUnit.Framework.Assert.IsFalse(kvstr, v.Has(2));
}
}
}
if (val0 != null)
{
NUnit.Framework.Assert.IsTrue(kvstr, val0.Get() == 0);
}
if (val1 != null)
{
NUnit.Framework.Assert.IsTrue(kvstr, val1.Get() == 1);
}
if (val2 != null)
{
NUnit.Framework.Assert.IsTrue(kvstr, val2.Get() == 2);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNestedJoin()
{
// outer(inner(S1,...,Sn),outer(S1,...Sn))
int Sources = 3;
int Items = (Sources + 1) * (Sources + 1);
Configuration conf = new Configuration();
Path @base = cluster.GetFileSystem().MakeQualified(new Path("/nested"));
int[][] source = new int[Sources][];
for (int i = 0; i < Sources; ++i)
{
source[i] = new int[Items];
for (int j = 0; j < Items; ++j)
{
source[i][j] = (i + 2) * (j + 1);
}
}
Path[] src = new Path[Sources];
SequenceFile.Writer[] @out = CreateWriters(@base, conf, Sources, src);
IntWritable k = new IntWritable();
for (int i_1 = 0; i_1 < Sources; ++i_1)
{
IntWritable v = new IntWritable();
v.Set(i_1);
for (int j = 0; j < Items; ++j)
{
k.Set(source[i_1][j]);
@out[i_1].Append(k, v);
}
@out[i_1].Close();
}
@out = null;
StringBuilder sb = new StringBuilder();
sb.Append("outer(inner(");
for (int i_2 = 0; i_2 < Sources; ++i_2)
{
sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_2].
ToString()));
if (i_2 + 1 != Sources)
{
sb.Append(",");
}
}
sb.Append("),outer(");
sb.Append(CompositeInputFormat.Compose(typeof(MapReduceTestUtil.Fake_IF), "foobar"
));
sb.Append(",");
for (int i_3 = 0; i_3 < Sources; ++i_3)
{
sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_3].
ToString()));
sb.Append(",");
}
sb.Append(CompositeInputFormat.Compose(typeof(MapReduceTestUtil.Fake_IF), "raboof"
) + "))");
conf.Set(CompositeInputFormat.JoinExpr, sb.ToString());
MapReduceTestUtil.Fake_IF.SetKeyClass(conf, typeof(IntWritable));
MapReduceTestUtil.Fake_IF.SetValClass(conf, typeof(IntWritable));
Job job = Job.GetInstance(conf);
Path outf = new Path(@base, "out");
FileOutputFormat.SetOutputPath(job, outf);
job.SetInputFormatClass(typeof(CompositeInputFormat));
job.SetMapperClass(typeof(Mapper));
job.SetReducerClass(typeof(Reducer));
job.SetNumReduceTasks(0);
job.SetOutputKeyClass(typeof(IntWritable));
job.SetOutputValueClass(typeof(TupleWritable));
job.SetOutputFormatClass(typeof(SequenceFileOutputFormat));
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
FileStatus[] outlist = cluster.GetFileSystem().ListStatus(outf, new Utils.OutputFileUtils.OutputFilesFilter
());
NUnit.Framework.Assert.AreEqual(1, outlist.Length);
NUnit.Framework.Assert.IsTrue(0 < outlist[0].GetLen());
SequenceFile.Reader r = new SequenceFile.Reader(cluster.GetFileSystem(), outlist[
0].GetPath(), conf);
TupleWritable v_1 = new TupleWritable();
while (r.Next(k, v_1))
{
NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(0));
NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(Sources + 1));
bool chk = true;
int ki = k.Get();
for (int i_4 = 2; i_4 < Sources + 2; ++i_4)
{
if ((ki % i_4) == 0 && ki <= i_4 * Items)
{
NUnit.Framework.Assert.AreEqual(i_4 - 2, ((IntWritable)((TupleWritable)v_1.Get(1)
).Get((i_4 - 1))).Get());
}
else
{
chk = false;
}
}
if (chk)
{
// present in all sources; chk inner
NUnit.Framework.Assert.IsTrue(v_1.Has(0));
for (int i_5 = 0; i_5 < Sources; ++i_5)
{
NUnit.Framework.Assert.IsTrue(((TupleWritable)v_1.Get(0)).Has(i_5));
}
}
else
{
// should not be present in inner join
NUnit.Framework.Assert.IsFalse(v_1.Has(0));
}
}
r.Close();
@base.GetFileSystem(conf).Delete(@base, true);
}
