/// <summary>Return true iff the tuple is full (all data sources contain this key).</summary>
protected internal override bool Combine(object[] srcs, TupleWritable dst)
{
System.Diagnostics.Debug.Assert(srcs.Length == dst.Size());
for (int i = 0; i < srcs.Length; ++i)
{
if (!dst.Has(i))
{
return(false);
}
}
return(true);
}
public virtual bool IsCompatible(TupleWritable that)
{
if (this.Size() != that.Size())
{
return(false);
}
for (int i = 0; i < values.Length; ++i)
{
if (Has(i) != that.Has(i))
{
return(false);
}
if (Has(i) && !values[i].Equals(that.Get(i)))
{
return(false);
}
}
return(true);
}
/// <summary>Tests that we can write more than 64 values.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestWideTupleBoundary()
{
Text emptyText = new Text("Should not be set written");
Writable[] values = new Writable[65];
Arrays.Fill(values, emptyText);
values[64] = new Text("Should be the only value set written");
TupleWritable tuple = new TupleWritable(values);
tuple.SetWritten(64);
for (int pos = 0; pos < tuple.Size(); pos++)
{
bool has = tuple.Has(pos);
if (pos == 64)
{
NUnit.Framework.Assert.IsTrue(has);
}
else
{
NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: "
+ pos, has);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestWideTuple2()
{
Text emptyText = new Text("Should be empty");
Writable[] values = new Writable[64];
Arrays.Fill(values, emptyText);
values[9] = new Text("Number 9");
TupleWritable tuple = new TupleWritable(values);
tuple.SetWritten(9);
for (int pos = 0; pos < tuple.Size(); pos++)
{
bool has = tuple.Has(pos);
if (pos == 9)
{
NUnit.Framework.Assert.IsTrue(has);
}
else
{
NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: "
+ pos, has);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNestedJoin()
{
// outer(inner(S1,...,Sn),outer(S1,...Sn))
int Sources = 3;
int Items = (Sources + 1) * (Sources + 1);
JobConf job = new JobConf();
Path @base = cluster.GetFileSystem().MakeQualified(new Path("/nested"));
int[][] source = new int[Sources][];
for (int i = 0; i < Sources; ++i)
{
source[i] = new int[Items];
for (int j = 0; j < Items; ++j)
{
source[i][j] = (i + 2) * (j + 1);
}
}
Path[] src = new Path[Sources];
SequenceFile.Writer[] @out = CreateWriters(@base, job, Sources, src);
IntWritable k = new IntWritable();
for (int i_1 = 0; i_1 < Sources; ++i_1)
{
IntWritable v = new IntWritable();
v.Set(i_1);
for (int j = 0; j < Items; ++j)
{
k.Set(source[i_1][j]);
@out[i_1].Append(k, v);
}
@out[i_1].Close();
}
@out = null;
StringBuilder sb = new StringBuilder();
sb.Append("outer(inner(");
for (int i_2 = 0; i_2 < Sources; ++i_2)
{
sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_2].
ToString()));
if (i_2 + 1 != Sources)
{
sb.Append(",");
}
}
sb.Append("),outer(");
sb.Append(CompositeInputFormat.Compose(typeof(TestDatamerge.Fake_IF), "foobar"));
sb.Append(",");
for (int i_3 = 0; i_3 < Sources; ++i_3)
{
sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_3].
ToString()));
sb.Append(",");
}
sb.Append(CompositeInputFormat.Compose(typeof(TestDatamerge.Fake_IF), "raboof") +
"))");
job.Set("mapreduce.join.expr", sb.ToString());
job.SetInputFormat(typeof(CompositeInputFormat));
Path outf = new Path(@base, "out");
FileOutputFormat.SetOutputPath(job, outf);
TestDatamerge.Fake_IF.SetKeyClass(job, typeof(IntWritable));
TestDatamerge.Fake_IF.SetValClass(job, typeof(IntWritable));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
job.SetOutputKeyClass(typeof(IntWritable));
job.SetOutputValueClass(typeof(TupleWritable));
job.SetOutputFormat(typeof(SequenceFileOutputFormat));
JobClient.RunJob(job);
FileStatus[] outlist = cluster.GetFileSystem().ListStatus(outf, new Utils.OutputFileUtils.OutputFilesFilter
());
NUnit.Framework.Assert.AreEqual(1, outlist.Length);
NUnit.Framework.Assert.IsTrue(0 < outlist[0].GetLen());
SequenceFile.Reader r = new SequenceFile.Reader(cluster.GetFileSystem(), outlist[
0].GetPath(), job);
TupleWritable v_1 = new TupleWritable();
while (r.Next(k, v_1))
{
NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(0));
NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(Sources + 1));
bool chk = true;
int ki = k.Get();
for (int i_4 = 2; i_4 < Sources + 2; ++i_4)
{
if ((ki % i_4) == 0 && ki <= i_4 * Items)
{
NUnit.Framework.Assert.AreEqual(i_4 - 2, ((IntWritable)((TupleWritable)v_1.Get(1)
).Get((i_4 - 1))).Get());
}
else
{
chk = false;
}
}
if (chk)
{
// present in all sources; chk inner
NUnit.Framework.Assert.IsTrue(v_1.Has(0));
for (int i_5 = 0; i_5 < Sources; ++i_5)
{
NUnit.Framework.Assert.IsTrue(((TupleWritable)v_1.Get(0)).Has(i_5));
}
}
else
{
// should not be present in inner join
NUnit.Framework.Assert.IsFalse(v_1.Has(0));
}
}
r.Close();
@base.GetFileSystem(job).Delete(@base, true);
}
