public _IEnumerator_123(TupleWritable _enclosing, Org.Apache.Hadoop.Mapreduce.Lib.Join.TupleWritable
t)
{
this._enclosing = _enclosing;
this.t = t;
this.bitIndex = this._enclosing.written.NextSetBit(0);
}
/// <summary>Populate Tuple from iterators.</summary>
/// <remarks>
/// Populate Tuple from iterators.
/// It should be the case that, given iterators i_1...i_n over values from
/// sources s_1...s_n sharing key k, repeated calls to next should yield
/// I x I.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
protected internal virtual bool Next(TupleWritable val)
{
// No static type info on Tuples
if (this.first)
{
int i = -1;
for (this.pos = 0; this.pos < this.iters.Length; ++this.pos)
{
if (this.iters[this.pos].HasNext() && this.iters[this.pos].Next((X)val.Get(this.pos
)))
{
i = this.pos;
val.SetWritten(i);
}
}
this.pos = i;
this.first = false;
if (this.pos < 0)
{
this.Clear();
return(false);
}
return(true);
}
while (0 <= this.pos && !(this.iters[this.pos].HasNext() && this.iters[this.pos].
Next((X)val.Get(this.pos))))
{
--this.pos;
}
if (this.pos < 0)
{
this.Clear();
return(false);
}
val.SetWritten(this.pos);
for (int i_1 = 0; i_1 < this.pos; ++i_1)
{
if (this.iters[i_1].Replay((X)val.Get(i_1)))
{
val.SetWritten(i_1);
}
}
while (this.pos + 1 < this.iters.Length)
{
++this.pos;
this.iters[this.pos].Reset();
if (this.iters[this.pos].HasNext() && this.iters[this.pos].Next((X)val.Get(this.pos
)))
{
val.SetWritten(this.pos);
}
}
return(true);
}
/// <summary>Return true iff the tuple is full (all data sources contain this key).</summary>
protected internal override bool Combine(object[] srcs, TupleWritable dst)
{
System.Diagnostics.Debug.Assert(srcs.Length == dst.Size());
for (int i = 0; i < srcs.Length; ++i)
{
if (!dst.Has(i))
{
return(false);
}
}
return(true);
}
/// <summary>
/// Write the next value into key, value as accepted by the operation
/// associated with this set of RecordReaders.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual bool Flush(TupleWritable value)
{
while (this.HasNext())
{
value.ClearWritten();
if (this.Next(value) && this._enclosing.Combine(this._enclosing.kids, value))
{
return(true);
}
}
return(false);
}
private int VerifIter(Writable[] writs, TupleWritable t, int i)
{
foreach (Writable w in t)
{
if (w is TupleWritable)
{
i = VerifIter(writs, ((TupleWritable)w), i);
continue;
}
NUnit.Framework.Assert.IsTrue("Bad value", w.Equals(writs[i++]));
}
return(i);
}
/// <exception cref="System.Exception"/>
public virtual void TestNestedIterable()
{
Random r = new Random();
Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable
(r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new
LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString(
"dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable
(Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) };
TupleWritable sTuple = MakeTuple(writs);
NUnit.Framework.Assert.IsTrue("Bad count", writs.Length == VerifIter(writs, sTuple
, 0));
}
/// <exception cref="System.Exception"/>
public virtual void TestIterable()
{
Random r = new Random();
Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable
(r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new
LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString(
"dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable
(Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) };
TupleWritable t = new TupleWritable(writs);
for (int i = 0; i < 6; ++i)
{
t.SetWritten(i);
}
VerifIter(writs, t, 0);
}
/// <summary>Replay the last Tuple emitted.</summary>
/// <exception cref="System.IO.IOException"/>
public virtual bool Replay(TupleWritable val)
{
// No static typeinfo on Tuples
// The last emitted tuple might have drawn on an empty source;
// it can't be cleared prematurely, b/c there may be more duplicate
// keys in iterator positions < pos
System.Diagnostics.Debug.Assert(!this.first);
bool ret = false;
for (int i = 0; i < this.iters.Length; ++i)
{
if (this.iters[i].Replay((X)val.Get(i)))
{
val.SetWritten(i);
ret = true;
}
}
return(ret);
}
/// <exception cref="System.Exception"/>
public virtual void TestWritable()
{
Random r = new Random();
Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable
(r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new
LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString(
"dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable
(Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) };
TupleWritable sTuple = MakeTuple(writs);
ByteArrayOutputStream @out = new ByteArrayOutputStream();
sTuple.Write(new DataOutputStream(@out));
ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.ReadFields(new DataInputStream(@in));
NUnit.Framework.Assert.IsTrue("Failed to write/read tuple", sTuple.Equals(dTuple)
);
}
private void ValidateInnerKeyValue(IntWritable k, TupleWritable v, int tupleSize,
bool firstTuple, bool secondTuple)
{
string kvstr = "Unexpected tuple: " + Stringify(k, v);
NUnit.Framework.Assert.IsTrue(kvstr, v.Size() == tupleSize);
int key = k.Get();
IntWritable val0 = null;
IntWritable val1 = null;
LongWritable val2 = null;
NUnit.Framework.Assert.IsTrue(kvstr, key % 2 == 0 && key / 2 <= Items);
NUnit.Framework.Assert.IsTrue(kvstr, key % 3 == 0 && key / 3 <= Items);
NUnit.Framework.Assert.IsTrue(kvstr, key % 4 == 0 && key / 4 <= Items);
if (firstTuple)
{
TupleWritable v0 = ((TupleWritable)v.Get(0));
val0 = (IntWritable)v0.Get(0);
val1 = (IntWritable)v0.Get(1);
val2 = (LongWritable)v.Get(1);
}
else
{
if (secondTuple)
{
val0 = (IntWritable)v.Get(0);
TupleWritable v1 = ((TupleWritable)v.Get(1));
val1 = (IntWritable)v1.Get(0);
val2 = (LongWritable)v1.Get(1);
}
else
{
val0 = (IntWritable)v.Get(0);
val1 = (IntWritable)v.Get(1);
val2 = (LongWritable)v.Get(2);
}
}
NUnit.Framework.Assert.IsTrue(kvstr, val0.Get() == 0);
NUnit.Framework.Assert.IsTrue(kvstr, val1.Get() == 1);
NUnit.Framework.Assert.IsTrue(kvstr, val2.Get() == 2);
}
/// <exception cref="System.Exception"/>
public virtual void TestWideWritable2()
{
Writable[] manyWrits = MakeRandomWritables(71);
TupleWritable sTuple = new TupleWritable(manyWrits);
for (int i = 0; i < manyWrits.Length; i++)
{
sTuple.SetWritten(i);
}
ByteArrayOutputStream @out = new ByteArrayOutputStream();
sTuple.Write(new DataOutputStream(@out));
ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray());
TupleWritable dTuple = new TupleWritable();
dTuple.ReadFields(new DataInputStream(@in));
NUnit.Framework.Assert.IsTrue("Failed to write/read tuple", sTuple.Equals(dTuple)
);
NUnit.Framework.Assert.AreEqual("All tuple data has not been read from the stream"
, -1, @in.Read());
}
/// <summary>
/// <inheritDoc/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override bool NextKeyValue()
{
if (key == null)
{
key = CreateKey();
}
if (value == null)
{
value = CreateValue();
}
if (jc.Flush(ivalue))
{
ReflectionUtils.Copy(conf, jc.Key(), key);
ReflectionUtils.Copy(conf, Emit(ivalue), value);
return(true);
}
if (ivalue == null)
{
ivalue = CreateTupleWritable();
}
jc.Clear();
PriorityQueue <ComposableRecordReader <K, object> > q = GetRecordReaderQueue();
K iterkey = CreateKey();
while (q != null && !q.IsEmpty())
{
FillJoinCollector(iterkey);
jc.Reset(iterkey);
if (jc.Flush(ivalue))
{
ReflectionUtils.Copy(conf, jc.Key(), key);
ReflectionUtils.Copy(conf, Emit(ivalue), value);
return(true);
}
jc.Clear();
}
return(false);
}
private TupleWritable MakeTuple(Writable[] writs)
{
Writable[] sub1 = new Writable[] { writs[1], writs[2] };
Writable[] sub3 = new Writable[] { writs[4], writs[5] };
Writable[] sub2 = new Writable[] { writs[3], new TupleWritable(sub3), writs[6] };
Writable[] vals = new Writable[] { writs[0], new TupleWritable(sub1), new TupleWritable
(sub2), writs[7], writs[8], writs[9] };
// [v0, [v1, v2], [v3, [v4, v5], v6], v7, v8, v9]
TupleWritable ret = new TupleWritable(vals);
for (int i = 0; i < 6; ++i)
{
ret.SetWritten(i);
}
((TupleWritable)sub2[1]).SetWritten(0);
((TupleWritable)sub2[1]).SetWritten(1);
((TupleWritable)vals[1]).SetWritten(0);
((TupleWritable)vals[1]).SetWritten(1);
for (int i_1 = 0; i_1 < 3; ++i_1)
{
((TupleWritable)vals[2]).SetWritten(i_1);
}
return(ret);
}
/// <summary>Tests that we can write more than 64 values.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestWideTupleBoundary()
{
Text emptyText = new Text("Should not be set written");
Writable[] values = new Writable[65];
Arrays.Fill(values, emptyText);
values[64] = new Text("Should be the only value set written");
TupleWritable tuple = new TupleWritable(values);
tuple.SetWritten(64);
for (int pos = 0; pos < tuple.Size(); pos++)
{
bool has = tuple.Has(pos);
if (pos == 64)
{
NUnit.Framework.Assert.IsTrue(has);
}
else
{
NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: "
+ pos, has);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestWideTuple2()
{
Text emptyText = new Text("Should be empty");
Writable[] values = new Writable[64];
Arrays.Fill(values, emptyText);
values[9] = new Text("Number 9");
TupleWritable tuple = new TupleWritable(values);
tuple.SetWritten(9);
for (int pos = 0; pos < tuple.Size(); pos++)
{
bool has = tuple.Has(pos);
if (pos == 9)
{
NUnit.Framework.Assert.IsTrue(has);
}
else
{
NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: "
+ pos, has);
}
}
}
/// <summary>Emit the value with the highest position in the tuple.</summary>
protected internal override V Emit(TupleWritable dst)
{
// No static typeinfo on Tuples
return((V)dst.GetEnumerator().Next());
}
// key type // accepts RecordReader<K,V> as children // emits Writables of this type protected internal abstract bool Combine(object[] srcs, TupleWritable value);
/// <summary>
/// Default implementation offers
/// <see cref="MultiFilterRecordReader{K, V}.Emit(TupleWritable)"/>
/// every Tuple from the
/// collector (the outer join of child RRs).
/// </summary>
protected internal override bool Combine(object[] srcs, TupleWritable dst)
{
return(true);
}
/// <summary> /// For each tuple emitted, return a value (typically one of the values /// in the tuple). /// </summary> /// <remarks> /// For each tuple emitted, return a value (typically one of the values /// in the tuple). /// Modifying the Writables in the tuple is permitted and unlikely to affect /// join behavior in most cases, but it is not recommended. It's safer to /// clone first. /// </remarks> /// <exception cref="System.IO.IOException"/> protected internal abstract V Emit(TupleWritable dst);
/// <exception cref="System.Exception"/>
public virtual void TestNestedJoin()
{
// outer(inner(S1,...,Sn),outer(S1,...Sn))
int Sources = 3;
int Items = (Sources + 1) * (Sources + 1);
Configuration conf = new Configuration();
Path @base = cluster.GetFileSystem().MakeQualified(new Path("/nested"));
int[][] source = new int[Sources][];
for (int i = 0; i < Sources; ++i)
{
source[i] = new int[Items];
for (int j = 0; j < Items; ++j)
{
source[i][j] = (i + 2) * (j + 1);
}
}
Path[] src = new Path[Sources];
SequenceFile.Writer[] @out = CreateWriters(@base, conf, Sources, src);
IntWritable k = new IntWritable();
for (int i_1 = 0; i_1 < Sources; ++i_1)
{
IntWritable v = new IntWritable();
v.Set(i_1);
for (int j = 0; j < Items; ++j)
{
k.Set(source[i_1][j]);
@out[i_1].Append(k, v);
}
@out[i_1].Close();
}
@out = null;
StringBuilder sb = new StringBuilder();
sb.Append("outer(inner(");
for (int i_2 = 0; i_2 < Sources; ++i_2)
{
sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_2].
ToString()));
if (i_2 + 1 != Sources)
{
sb.Append(",");
}
}
sb.Append("),outer(");
sb.Append(CompositeInputFormat.Compose(typeof(MapReduceTestUtil.Fake_IF), "foobar"
));
sb.Append(",");
for (int i_3 = 0; i_3 < Sources; ++i_3)
{
sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_3].
ToString()));
sb.Append(",");
}
sb.Append(CompositeInputFormat.Compose(typeof(MapReduceTestUtil.Fake_IF), "raboof"
) + "))");
conf.Set(CompositeInputFormat.JoinExpr, sb.ToString());
MapReduceTestUtil.Fake_IF.SetKeyClass(conf, typeof(IntWritable));
MapReduceTestUtil.Fake_IF.SetValClass(conf, typeof(IntWritable));
Job job = Job.GetInstance(conf);
Path outf = new Path(@base, "out");
FileOutputFormat.SetOutputPath(job, outf);
job.SetInputFormatClass(typeof(CompositeInputFormat));
job.SetMapperClass(typeof(Mapper));
job.SetReducerClass(typeof(Reducer));
job.SetNumReduceTasks(0);
job.SetOutputKeyClass(typeof(IntWritable));
job.SetOutputValueClass(typeof(TupleWritable));
job.SetOutputFormatClass(typeof(SequenceFileOutputFormat));
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
FileStatus[] outlist = cluster.GetFileSystem().ListStatus(outf, new Utils.OutputFileUtils.OutputFilesFilter
());
NUnit.Framework.Assert.AreEqual(1, outlist.Length);
NUnit.Framework.Assert.IsTrue(0 < outlist[0].GetLen());
SequenceFile.Reader r = new SequenceFile.Reader(cluster.GetFileSystem(), outlist[
0].GetPath(), conf);
TupleWritable v_1 = new TupleWritable();
while (r.Next(k, v_1))
{
NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(0));
NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(Sources + 1));
bool chk = true;
int ki = k.Get();
for (int i_4 = 2; i_4 < Sources + 2; ++i_4)
{
if ((ki % i_4) == 0 && ki <= i_4 * Items)
{
NUnit.Framework.Assert.AreEqual(i_4 - 2, ((IntWritable)((TupleWritable)v_1.Get(1)
).Get((i_4 - 1))).Get());
}
else
{
chk = false;
}
}
if (chk)
{
// present in all sources; chk inner
NUnit.Framework.Assert.IsTrue(v_1.Has(0));
for (int i_5 = 0; i_5 < Sources; ++i_5)
{
NUnit.Framework.Assert.IsTrue(((TupleWritable)v_1.Get(0)).Has(i_5));
}
}
else
{
// should not be present in inner join
NUnit.Framework.Assert.IsFalse(v_1.Has(0));
}
}
r.Close();
@base.GetFileSystem(conf).Delete(@base, true);
}
/// <summary>Emit everything from the collector.</summary>
protected internal override bool Combine(object[] srcs, TupleWritable dst)
{
System.Diagnostics.Debug.Assert(srcs.Length == dst.Size());
return(true);
}