/// <summary>Populate Tuple from iterators.</summary>
/// <remarks>
/// Populate Tuple from iterators.
/// It should be the case that, given iterators i_1...i_n over values from
/// sources s_1...s_n sharing key k, repeated calls to next should yield
/// I x I.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
protected internal virtual bool Next(TupleWritable val)
{
// No static typeinfo on Tuples
if (this.first)
{
int i = -1;
for (this.pos = 0; this.pos < this.iters.Length; ++this.pos)
{
if (this.iters[this.pos].HasNext() && this.iters[this.pos].Next((X)val.Get(this.pos
)))
{
i = this.pos;
val.SetWritten(i);
}
}
this.pos = i;
this.first = false;
if (this.pos < 0)
{
this.Clear();
return(false);
}
return(true);
}
while (0 <= this.pos && !(this.iters[this.pos].HasNext() && this.iters[this.pos].
Next((X)val.Get(this.pos))))
{
--this.pos;
}
if (this.pos < 0)
{
this.Clear();
return(false);
}
val.SetWritten(this.pos);
for (int i_1 = 0; i_1 < this.pos; ++i_1)
{
if (this.iters[i_1].Replay((X)val.Get(i_1)))
{
val.SetWritten(i_1);
}
}
while (this.pos + 1 < this.iters.Length)
{
++this.pos;
this.iters[this.pos].Reset();
if (this.iters[this.pos].HasNext() && this.iters[this.pos].Next((X)val.Get(this.pos
)))
{
val.SetWritten(this.pos);
}
}
return(true);
}
/// <summary>
/// Write the next value into key, value as accepted by the operation
/// associated with this set of RecordReaders.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual bool Flush(TupleWritable value)
{
while (this.HasNext())
{
value.ClearWritten();
if (this.Next(value) && this._enclosing.Combine(this._enclosing.kids, value))
{
return(true);
}
}
return(false);
}
/// <summary>Return true iff the tuple is full (all data sources contain this key).</summary>
protected internal override bool Combine(object[] srcs, TupleWritable dst)
{
System.Diagnostics.Debug.Assert(srcs.Length == dst.Size());
for (int i = 0; i < srcs.Length; ++i)
{
if (!dst.Has(i))
{
return(false);
}
}
return(true);
}
private int VerifIter(Writable[] writs, TupleWritable t, int i)
{
foreach (Writable w in t)
{
if (w is TupleWritable)
{
i = VerifIter(writs, ((TupleWritable)w), i);
continue;
}
NUnit.Framework.Assert.IsTrue("Bad value", w.Equals(writs[i++]));
}
return(i);
}
/// <exception cref="System.Exception"/>
public virtual void TestNestedIterable()
{
Random r = new Random();
Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable
(r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new
LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString(
"dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable
(Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) };
TupleWritable sTuple = MakeTuple(writs);
NUnit.Framework.Assert.IsTrue("Bad count", writs.Length == VerifIter(writs, sTuple
, 0));
}
/// <exception cref="System.Exception"/>
public virtual void TestIterable()
{
Random r = new Random();
Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable
(r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new
LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString(
"dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable
(Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) };
TupleWritable t = new TupleWritable(writs);
for (int i = 0; i < 6; ++i)
{
t.SetWritten(i);
}
VerifIter(writs, t, 0);
}
/// <summary>
/// <inheritDoc/>
///
/// </summary>
public virtual V CreateValue()
{
// Explicit check for value class agreement
if (null == valueclass)
{
Type cls = kids[0].CreateValue().GetType();
foreach (RecordReader <K, V> rr in kids)
{
if (!cls.Equals(rr.CreateValue().GetType()))
{
throw new InvalidCastException("Child value classes fail to agree");
}
}
valueclass = cls.AsSubclass <Writable>();
ivalue = CreateInternalValue();
}
return((V)ReflectionUtils.NewInstance(valueclass, null));
}
/// <exception cref="System.Exception"/>
public virtual void TestPreVersion21CompatibilityEmptyTuple()
{
Writable[] manyWrits = new Writable[0];
TestTupleWritable.PreVersion21TupleWritable oldTuple = new TestTupleWritable.PreVersion21TupleWritable
(manyWrits);
// don't set any values written
ByteArrayOutputStream @out = new ByteArrayOutputStream();
oldTuple.Write(new DataOutputStream(@out));
ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.ReadFields(new DataInputStream(@in));
NUnit.Framework.Assert.IsTrue("Tuple writable is unable to read pre-0.21 versions of TupleWritable"
, oldTuple.IsCompatible(dTuple));
NUnit.Framework.Assert.AreEqual("All tuple data has not been read from the stream"
, -1, @in.Read());
}
public virtual bool IsCompatible(TupleWritable that)
{
if (this.Size() != that.Size())
{
return(false);
}
for (int i = 0; i < values.Length; ++i)
{
if (Has(i) != that.Has(i))
{
return(false);
}
if (Has(i) && !values[i].Equals(that.Get(i)))
{
return(false);
}
}
return(true);
}
/// <summary>Replay the last Tuple emitted.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual bool Replay(TupleWritable val)
{
// No static typeinfo on Tuples
// The last emitted tuple might have drawn on an empty source;
// it can't be cleared prematurely, b/c there may be more duplicate
// keys in iterator positions < pos
System.Diagnostics.Debug.Assert(!this.first);
bool ret = false;
for (int i = 0; i < this.iters.Length; ++i)
{
if (this.iters[i].Replay((X)val.Get(i)))
{
val.SetWritten(i);
ret = true;
}
}
return(ret);
}
/// <exception cref="System.Exception"/>
public virtual void TestWritable()
{
Random r = new Random();
Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable
(r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new
LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString(
"dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable
(Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) };
TupleWritable sTuple = MakeTuple(writs);
ByteArrayOutputStream @out = new ByteArrayOutputStream();
sTuple.Write(new DataOutputStream(@out));
ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.ReadFields(new DataInputStream(@in));
NUnit.Framework.Assert.IsTrue("Failed to write/read tuple", sTuple.Equals(dTuple)
);
}
/// <exception cref="System.Exception"/>
public virtual void TestWideWritable2()
{
Writable[] manyWrits = MakeRandomWritables(71);
TupleWritable sTuple = new TupleWritable(manyWrits);
for (int i = 0; i < manyWrits.Length; i++)
{
sTuple.SetWritten(i);
}
ByteArrayOutputStream @out = new ByteArrayOutputStream();
sTuple.Write(new DataOutputStream(@out));
ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.ReadFields(new DataInputStream(@in));
NUnit.Framework.Assert.IsTrue("Failed to write/read tuple", sTuple.Equals(dTuple)
);
NUnit.Framework.Assert.AreEqual("All tuple data has not been read from the stream"
, -1, @in.Read());
}
/// <summary>Tests that we can write more than 64 values.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestWideTupleBoundary()
{
Text emptyText = new Text("Should not be set written");
Writable[] values = new Writable[65];
Arrays.Fill(values, emptyText);
values[64] = new Text("Should be the only value set written");
TupleWritable tuple = new TupleWritable(values);
tuple.SetWritten(64);
for (int pos = 0; pos < tuple.Size(); pos++)
{
bool has = tuple.Has(pos);
if (pos == 64)
{
NUnit.Framework.Assert.IsTrue(has);
}
else
{
NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: "
+ pos, has);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestWideTuple2()
{
Text emptyText = new Text("Should be empty");
Writable[] values = new Writable[64];
Arrays.Fill(values, emptyText);
values[9] = new Text("Number 9");
TupleWritable tuple = new TupleWritable(values);
tuple.SetWritten(9);
for (int pos = 0; pos < tuple.Size(); pos++)
{
bool has = tuple.Has(pos);
if (pos == 9)
{
NUnit.Framework.Assert.IsTrue(has);
}
else
{
NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: "
+ pos, has);
}
}
}
/// <summary>Tests compatibility with pre-0.21 versions of TupleWritable</summary>
/// <exception cref="System.Exception"/>
public virtual void TestPreVersion21Compatibility()
{
Writable[] manyWrits = MakeRandomWritables(64);
TestTupleWritable.PreVersion21TupleWritable oldTuple = new TestTupleWritable.PreVersion21TupleWritable
(manyWrits);
for (int i = 0; i < manyWrits.Length; i++)
{
if (i % 3 == 0)
{
oldTuple.SetWritten(i);
}
}
ByteArrayOutputStream @out = new ByteArrayOutputStream();
oldTuple.Write(new DataOutputStream(@out));
ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.ReadFields(new DataInputStream(@in));
NUnit.Framework.Assert.IsTrue("Tuple writable is unable to read pre-0.21 versions of TupleWritable"
, oldTuple.IsCompatible(dTuple));
NUnit.Framework.Assert.AreEqual("All tuple data has not been read from the stream"
, -1, @in.Read());
}
/// <summary>
/// Emit the next set of key, value pairs as defined by the child
/// RecordReaders and operation associated with this composite RR.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual bool Next(K key, TupleWritable value)
{
if (jc.Flush(value))
{
WritableUtils.CloneInto(key, jc.Key());
return(true);
}
jc.Clear();
K iterkey = CreateKey();
PriorityQueue <ComposableRecordReader <K, object> > q = GetRecordReaderQueue();
while (!q.IsEmpty())
{
FillJoinCollector(iterkey);
jc.Reset(iterkey);
if (jc.Flush(value))
{
WritableUtils.CloneInto(key, jc.Key());
return(true);
}
jc.Clear();
}
return(false);
}
private TupleWritable MakeTuple(Writable[] writs)
{
Writable[] sub1 = new Writable[] { writs[1], writs[2] };
Writable[] sub3 = new Writable[] { writs[4], writs[5] };
Writable[] sub2 = new Writable[] { writs[3], new TupleWritable(sub3), writs[6] };
Writable[] vals = new Writable[] { writs[0], new TupleWritable(sub1), new TupleWritable
(sub2), writs[7], writs[8], writs[9] };
// [v0, [v1, v2], [v3, [v4, v5], v6], v7, v8, v9]
TupleWritable ret = new TupleWritable(vals);
for (int i = 0; i < 6; ++i)
{
ret.SetWritten(i);
}
((TupleWritable)sub2[1]).SetWritten(0);
((TupleWritable)sub2[1]).SetWritten(1);
((TupleWritable)vals[1]).SetWritten(0);
((TupleWritable)vals[1]).SetWritten(1);
for (int i_1 = 0; i_1 < 3; ++i_1)
{
((TupleWritable)vals[2]).SetWritten(i_1);
}
return(ret);
}
/// <summary>Emit everything from the collector.</summary>
protected internal override bool Combine(object[] srcs, TupleWritable dst)
{
System.Diagnostics.Debug.Assert(srcs.Length == dst.Size());
return(true);
}
// key type // accepts RecordReader<K,V> as children // emits Writables of this type protected internal abstract bool Combine(object[] srcs, TupleWritable value);
/// <summary>Emit the value with the highest position in the tuple.</summary>
protected internal override V Emit(TupleWritable dst)
{
// No static typeinfo on Tuples
return((V)dst.GetEnumerator().Next());
}
/// <exception cref="System.Exception"/>
public virtual void TestNestedJoin()
{
// outer(inner(S1,...,Sn),outer(S1,...Sn))
int Sources = 3;
int Items = (Sources + 1) * (Sources + 1);
JobConf job = new JobConf();
Path @base = cluster.GetFileSystem().MakeQualified(new Path("/nested"));
int[][] source = new int[Sources][];
for (int i = 0; i < Sources; ++i)
{
source[i] = new int[Items];
for (int j = 0; j < Items; ++j)
{
source[i][j] = (i + 2) * (j + 1);
}
}
Path[] src = new Path[Sources];
SequenceFile.Writer[] @out = CreateWriters(@base, job, Sources, src);
IntWritable k = new IntWritable();
for (int i_1 = 0; i_1 < Sources; ++i_1)
{
IntWritable v = new IntWritable();
v.Set(i_1);
for (int j = 0; j < Items; ++j)
{
k.Set(source[i_1][j]);
@out[i_1].Append(k, v);
}
@out[i_1].Close();
}
@out = null;
StringBuilder sb = new StringBuilder();
sb.Append("outer(inner(");
for (int i_2 = 0; i_2 < Sources; ++i_2)
{
sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_2].
ToString()));
if (i_2 + 1 != Sources)
{
sb.Append(",");
}
}
sb.Append("),outer(");
sb.Append(CompositeInputFormat.Compose(typeof(TestDatamerge.Fake_IF), "foobar"));
sb.Append(",");
for (int i_3 = 0; i_3 < Sources; ++i_3)
{
sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_3].
ToString()));
sb.Append(",");
}
sb.Append(CompositeInputFormat.Compose(typeof(TestDatamerge.Fake_IF), "raboof") +
"))");
job.Set("mapreduce.join.expr", sb.ToString());
job.SetInputFormat(typeof(CompositeInputFormat));
Path outf = new Path(@base, "out");
FileOutputFormat.SetOutputPath(job, outf);
TestDatamerge.Fake_IF.SetKeyClass(job, typeof(IntWritable));
TestDatamerge.Fake_IF.SetValClass(job, typeof(IntWritable));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
job.SetOutputKeyClass(typeof(IntWritable));
job.SetOutputValueClass(typeof(TupleWritable));
job.SetOutputFormat(typeof(SequenceFileOutputFormat));
JobClient.RunJob(job);
FileStatus[] outlist = cluster.GetFileSystem().ListStatus(outf, new Utils.OutputFileUtils.OutputFilesFilter
());
NUnit.Framework.Assert.AreEqual(1, outlist.Length);
NUnit.Framework.Assert.IsTrue(0 < outlist[0].GetLen());
SequenceFile.Reader r = new SequenceFile.Reader(cluster.GetFileSystem(), outlist[
0].GetPath(), job);
TupleWritable v_1 = new TupleWritable();
while (r.Next(k, v_1))
{
NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(0));
NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(Sources + 1));
bool chk = true;
int ki = k.Get();
for (int i_4 = 2; i_4 < Sources + 2; ++i_4)
{
if ((ki % i_4) == 0 && ki <= i_4 * Items)
{
NUnit.Framework.Assert.AreEqual(i_4 - 2, ((IntWritable)((TupleWritable)v_1.Get(1)
).Get((i_4 - 1))).Get());
}
else
{
chk = false;
}
}
if (chk)
{
// present in all sources; chk inner
NUnit.Framework.Assert.IsTrue(v_1.Has(0));
for (int i_5 = 0; i_5 < Sources; ++i_5)
{
NUnit.Framework.Assert.IsTrue(((TupleWritable)v_1.Get(0)).Has(i_5));
}
}
else
{
// should not be present in inner join
NUnit.Framework.Assert.IsFalse(v_1.Has(0));
}
}
r.Close();
@base.GetFileSystem(job).Delete(@base, true);
}
/// <summary> /// For each tuple emitted, return a value (typically one of the values /// in the tuple). /// </summary> /// <remarks> /// For each tuple emitted, return a value (typically one of the values /// in the tuple). /// Modifying the Writables in the tuple is permitted and unlikely to affect /// join behavior in most cases, but it is not recommended. It's safer to /// clone first. /// </remarks> /// <exception cref="System.IO.IOException"/> protected internal abstract V Emit(TupleWritable dst);
/// <summary>
/// Default implementation offers
/// <see cref="MultiFilterRecordReader{K, V}.Emit(TupleWritable)"/>
/// every Tuple from the
/// collector (the outer join of child RRs).
/// </summary>
protected internal override bool Combine(object[] srcs, TupleWritable dst)
{
return(true);
}