/// <summary>Populate Tuple from iterators.</summary> /// <remarks> /// Populate Tuple from iterators. /// It should be the case that, given iterators i_1...i_n over values from /// sources s_1...s_n sharing key k, repeated calls to next should yield /// I x I. /// </remarks> /// <exception cref="System.IO.IOException"/> protected internal virtual bool Next(TupleWritable val) { // No static typeinfo on Tuples if (this.first) { int i = -1; for (this.pos = 0; this.pos < this.iters.Length; ++this.pos) { if (this.iters[this.pos].HasNext() && this.iters[this.pos].Next((X)val.Get(this.pos ))) { i = this.pos; val.SetWritten(i); } } this.pos = i; this.first = false; if (this.pos < 0) { this.Clear(); return(false); } return(true); } while (0 <= this.pos && !(this.iters[this.pos].HasNext() && this.iters[this.pos]. Next((X)val.Get(this.pos)))) { --this.pos; } if (this.pos < 0) { this.Clear(); return(false); } val.SetWritten(this.pos); for (int i_1 = 0; i_1 < this.pos; ++i_1) { if (this.iters[i_1].Replay((X)val.Get(i_1))) { val.SetWritten(i_1); } } while (this.pos + 1 < this.iters.Length) { ++this.pos; this.iters[this.pos].Reset(); if (this.iters[this.pos].HasNext() && this.iters[this.pos].Next((X)val.Get(this.pos ))) { val.SetWritten(this.pos); } } return(true); }
/// <summary> /// Write the next value into key, value as accepted by the operation /// associated with this set of RecordReaders. /// </summary> /// <exception cref="System.IO.IOException"/> public virtual bool Flush(TupleWritable value) { while (this.HasNext()) { value.ClearWritten(); if (this.Next(value) && this._enclosing.Combine(this._enclosing.kids, value)) { return(true); } } return(false); }
/// <summary>Return true iff the tuple is full (all data sources contain this key).</summary> protected internal override bool Combine(object[] srcs, TupleWritable dst) { System.Diagnostics.Debug.Assert(srcs.Length == dst.Size()); for (int i = 0; i < srcs.Length; ++i) { if (!dst.Has(i)) { return(false); } } return(true); }
private int VerifIter(Writable[] writs, TupleWritable t, int i) { foreach (Writable w in t) { if (w is TupleWritable) { i = VerifIter(writs, ((TupleWritable)w), i); continue; } NUnit.Framework.Assert.IsTrue("Bad value", w.Equals(writs[i++])); } return(i); }
/// <exception cref="System.Exception"/> public virtual void TestNestedIterable() { Random r = new Random(); Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable (r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString( "dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable (Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) }; TupleWritable sTuple = MakeTuple(writs); NUnit.Framework.Assert.IsTrue("Bad count", writs.Length == VerifIter(writs, sTuple , 0)); }
/// <exception cref="System.Exception"/> public virtual void TestIterable() { Random r = new Random(); Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable (r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString( "dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable (Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) }; TupleWritable t = new TupleWritable(writs); for (int i = 0; i < 6; ++i) { t.SetWritten(i); } VerifIter(writs, t, 0); }
/// <summary> /// <inheritDoc/> /// /// </summary> public virtual V CreateValue() { // Explicit check for value class agreement if (null == valueclass) { Type cls = kids[0].CreateValue().GetType(); foreach (RecordReader <K, V> rr in kids) { if (!cls.Equals(rr.CreateValue().GetType())) { throw new InvalidCastException("Child value classes fail to agree"); } } valueclass = cls.AsSubclass <Writable>(); ivalue = CreateInternalValue(); } return((V)ReflectionUtils.NewInstance(valueclass, null)); }
/// <exception cref="System.Exception"/> public virtual void TestPreVersion21CompatibilityEmptyTuple() { Writable[] manyWrits = new Writable[0]; TestTupleWritable.PreVersion21TupleWritable oldTuple = new TestTupleWritable.PreVersion21TupleWritable (manyWrits); // don't set any values written ByteArrayOutputStream @out = new ByteArrayOutputStream(); oldTuple.Write(new DataOutputStream(@out)); ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray()); TupleWritable dTuple = new TupleWritable(); dTuple.ReadFields(new DataInputStream(@in)); NUnit.Framework.Assert.IsTrue("Tuple writable is unable to read pre-0.21 versions of TupleWritable" , oldTuple.IsCompatible(dTuple)); NUnit.Framework.Assert.AreEqual("All tuple data has not been read from the stream" , -1, @in.Read()); }
public virtual bool IsCompatible(TupleWritable that) { if (this.Size() != that.Size()) { return(false); } for (int i = 0; i < values.Length; ++i) { if (Has(i) != that.Has(i)) { return(false); } if (Has(i) && !values[i].Equals(that.Get(i))) { return(false); } } return(true); }
/// <summary>Replay the last Tuple emitted.</summary> /// <exception cref="System.IO.IOException"/> public virtual bool Replay(TupleWritable val) { // No static typeinfo on Tuples // The last emitted tuple might have drawn on an empty source; // it can't be cleared prematurely, b/c there may be more duplicate // keys in iterator positions < pos System.Diagnostics.Debug.Assert(!this.first); bool ret = false; for (int i = 0; i < this.iters.Length; ++i) { if (this.iters[i].Replay((X)val.Get(i))) { val.SetWritten(i); ret = true; } } return(ret); }
/// <exception cref="System.Exception"/> public virtual void TestWritable() { Random r = new Random(); Writable[] writs = new Writable[] { new BooleanWritable(r.NextBoolean()), new FloatWritable (r.NextFloat()), new FloatWritable(r.NextFloat()), new IntWritable(r.Next()), new LongWritable(r.NextLong()), new BytesWritable(Sharpen.Runtime.GetBytesForString( "dingo")), new LongWritable(r.NextLong()), new IntWritable(r.Next()), new BytesWritable (Sharpen.Runtime.GetBytesForString("yak")), new IntWritable(r.Next()) }; TupleWritable sTuple = MakeTuple(writs); ByteArrayOutputStream @out = new ByteArrayOutputStream(); sTuple.Write(new DataOutputStream(@out)); ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray()); TupleWritable dTuple = new TupleWritable(); dTuple.ReadFields(new DataInputStream(@in)); NUnit.Framework.Assert.IsTrue("Failed to write/read tuple", sTuple.Equals(dTuple) ); }
/// <exception cref="System.Exception"/> public virtual void TestWideWritable2() { Writable[] manyWrits = MakeRandomWritables(71); TupleWritable sTuple = new TupleWritable(manyWrits); for (int i = 0; i < manyWrits.Length; i++) { sTuple.SetWritten(i); } ByteArrayOutputStream @out = new ByteArrayOutputStream(); sTuple.Write(new DataOutputStream(@out)); ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray()); TupleWritable dTuple = new TupleWritable(); dTuple.ReadFields(new DataInputStream(@in)); NUnit.Framework.Assert.IsTrue("Failed to write/read tuple", sTuple.Equals(dTuple) ); NUnit.Framework.Assert.AreEqual("All tuple data has not been read from the stream" , -1, @in.Read()); }
/// <summary>Tests that we can write more than 64 values.</summary> /// <exception cref="System.Exception"/> public virtual void TestWideTupleBoundary() { Text emptyText = new Text("Should not be set written"); Writable[] values = new Writable[65]; Arrays.Fill(values, emptyText); values[64] = new Text("Should be the only value set written"); TupleWritable tuple = new TupleWritable(values); tuple.SetWritten(64); for (int pos = 0; pos < tuple.Size(); pos++) { bool has = tuple.Has(pos); if (pos == 64) { NUnit.Framework.Assert.IsTrue(has); } else { NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: " + pos, has); } } }
/// <exception cref="System.Exception"/> public virtual void TestWideTuple2() { Text emptyText = new Text("Should be empty"); Writable[] values = new Writable[64]; Arrays.Fill(values, emptyText); values[9] = new Text("Number 9"); TupleWritable tuple = new TupleWritable(values); tuple.SetWritten(9); for (int pos = 0; pos < tuple.Size(); pos++) { bool has = tuple.Has(pos); if (pos == 9) { NUnit.Framework.Assert.IsTrue(has); } else { NUnit.Framework.Assert.IsFalse("Tuple position is incorrectly labelled as set: " + pos, has); } } }
/// <summary>Tests compatibility with pre-0.21 versions of TupleWritable</summary> /// <exception cref="System.Exception"/> public virtual void TestPreVersion21Compatibility() { Writable[] manyWrits = MakeRandomWritables(64); TestTupleWritable.PreVersion21TupleWritable oldTuple = new TestTupleWritable.PreVersion21TupleWritable (manyWrits); for (int i = 0; i < manyWrits.Length; i++) { if (i % 3 == 0) { oldTuple.SetWritten(i); } } ByteArrayOutputStream @out = new ByteArrayOutputStream(); oldTuple.Write(new DataOutputStream(@out)); ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray()); TupleWritable dTuple = new TupleWritable(); dTuple.ReadFields(new DataInputStream(@in)); NUnit.Framework.Assert.IsTrue("Tuple writable is unable to read pre-0.21 versions of TupleWritable" , oldTuple.IsCompatible(dTuple)); NUnit.Framework.Assert.AreEqual("All tuple data has not been read from the stream" , -1, @in.Read()); }
/// <summary> /// Emit the next set of key, value pairs as defined by the child /// RecordReaders and operation associated with this composite RR. /// </summary> /// <exception cref="System.IO.IOException"/> public virtual bool Next(K key, TupleWritable value) { if (jc.Flush(value)) { WritableUtils.CloneInto(key, jc.Key()); return(true); } jc.Clear(); K iterkey = CreateKey(); PriorityQueue <ComposableRecordReader <K, object> > q = GetRecordReaderQueue(); while (!q.IsEmpty()) { FillJoinCollector(iterkey); jc.Reset(iterkey); if (jc.Flush(value)) { WritableUtils.CloneInto(key, jc.Key()); return(true); } jc.Clear(); } return(false); }
private TupleWritable MakeTuple(Writable[] writs) { Writable[] sub1 = new Writable[] { writs[1], writs[2] }; Writable[] sub3 = new Writable[] { writs[4], writs[5] }; Writable[] sub2 = new Writable[] { writs[3], new TupleWritable(sub3), writs[6] }; Writable[] vals = new Writable[] { writs[0], new TupleWritable(sub1), new TupleWritable (sub2), writs[7], writs[8], writs[9] }; // [v0, [v1, v2], [v3, [v4, v5], v6], v7, v8, v9] TupleWritable ret = new TupleWritable(vals); for (int i = 0; i < 6; ++i) { ret.SetWritten(i); } ((TupleWritable)sub2[1]).SetWritten(0); ((TupleWritable)sub2[1]).SetWritten(1); ((TupleWritable)vals[1]).SetWritten(0); ((TupleWritable)vals[1]).SetWritten(1); for (int i_1 = 0; i_1 < 3; ++i_1) { ((TupleWritable)vals[2]).SetWritten(i_1); } return(ret); }
/// <summary>Emit everything from the collector.</summary> protected internal override bool Combine(object[] srcs, TupleWritable dst) { System.Diagnostics.Debug.Assert(srcs.Length == dst.Size()); return(true); }
// key type // accepts RecordReader<K,V> as children // emits Writables of this type protected internal abstract bool Combine(object[] srcs, TupleWritable value);
/// <summary>Emit the value with the highest position in the tuple.</summary> protected internal override V Emit(TupleWritable dst) { // No static typeinfo on Tuples return((V)dst.GetEnumerator().Next()); }
/// <exception cref="System.Exception"/> public virtual void TestNestedJoin() { // outer(inner(S1,...,Sn),outer(S1,...Sn)) int Sources = 3; int Items = (Sources + 1) * (Sources + 1); JobConf job = new JobConf(); Path @base = cluster.GetFileSystem().MakeQualified(new Path("/nested")); int[][] source = new int[Sources][]; for (int i = 0; i < Sources; ++i) { source[i] = new int[Items]; for (int j = 0; j < Items; ++j) { source[i][j] = (i + 2) * (j + 1); } } Path[] src = new Path[Sources]; SequenceFile.Writer[] @out = CreateWriters(@base, job, Sources, src); IntWritable k = new IntWritable(); for (int i_1 = 0; i_1 < Sources; ++i_1) { IntWritable v = new IntWritable(); v.Set(i_1); for (int j = 0; j < Items; ++j) { k.Set(source[i_1][j]); @out[i_1].Append(k, v); } @out[i_1].Close(); } @out = null; StringBuilder sb = new StringBuilder(); sb.Append("outer(inner("); for (int i_2 = 0; i_2 < Sources; ++i_2) { sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_2]. ToString())); if (i_2 + 1 != Sources) { sb.Append(","); } } sb.Append("),outer("); sb.Append(CompositeInputFormat.Compose(typeof(TestDatamerge.Fake_IF), "foobar")); sb.Append(","); for (int i_3 = 0; i_3 < Sources; ++i_3) { sb.Append(CompositeInputFormat.Compose(typeof(SequenceFileInputFormat), src[i_3]. ToString())); sb.Append(","); } sb.Append(CompositeInputFormat.Compose(typeof(TestDatamerge.Fake_IF), "raboof") + "))"); job.Set("mapreduce.join.expr", sb.ToString()); job.SetInputFormat(typeof(CompositeInputFormat)); Path outf = new Path(@base, "out"); FileOutputFormat.SetOutputPath(job, outf); TestDatamerge.Fake_IF.SetKeyClass(job, typeof(IntWritable)); TestDatamerge.Fake_IF.SetValClass(job, typeof(IntWritable)); job.SetMapperClass(typeof(IdentityMapper)); job.SetReducerClass(typeof(IdentityReducer)); job.SetNumReduceTasks(0); job.SetOutputKeyClass(typeof(IntWritable)); job.SetOutputValueClass(typeof(TupleWritable)); job.SetOutputFormat(typeof(SequenceFileOutputFormat)); JobClient.RunJob(job); FileStatus[] outlist = cluster.GetFileSystem().ListStatus(outf, new Utils.OutputFileUtils.OutputFilesFilter ()); NUnit.Framework.Assert.AreEqual(1, outlist.Length); NUnit.Framework.Assert.IsTrue(0 < outlist[0].GetLen()); SequenceFile.Reader r = new SequenceFile.Reader(cluster.GetFileSystem(), outlist[ 0].GetPath(), job); TupleWritable v_1 = new TupleWritable(); while (r.Next(k, v_1)) { NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(0)); NUnit.Framework.Assert.IsFalse(((TupleWritable)v_1.Get(1)).Has(Sources + 1)); bool chk = true; int ki = k.Get(); for (int i_4 = 2; i_4 < Sources + 2; ++i_4) { if ((ki % i_4) == 0 && ki <= i_4 * Items) { NUnit.Framework.Assert.AreEqual(i_4 - 2, ((IntWritable)((TupleWritable)v_1.Get(1) ).Get((i_4 - 1))).Get()); } else { chk = false; } } if (chk) { // present in all sources; chk inner NUnit.Framework.Assert.IsTrue(v_1.Has(0)); for (int i_5 = 0; i_5 < Sources; ++i_5) { NUnit.Framework.Assert.IsTrue(((TupleWritable)v_1.Get(0)).Has(i_5)); } } else { // should not be present in inner join NUnit.Framework.Assert.IsFalse(v_1.Has(0)); } } r.Close(); @base.GetFileSystem(job).Delete(@base, true); }
/// <summary> /// For each tuple emitted, return a value (typically one of the values /// in the tuple). /// </summary> /// <remarks> /// For each tuple emitted, return a value (typically one of the values /// in the tuple). /// Modifying the Writables in the tuple is permitted and unlikely to affect /// join behavior in most cases, but it is not recommended. It's safer to /// clone first. /// </remarks> /// <exception cref="System.IO.IOException"/> protected internal abstract V Emit(TupleWritable dst);
/// <summary> /// Default implementation offers /// <see cref="MultiFilterRecordReader{K, V}.Emit(TupleWritable)"/> /// every Tuple from the /// collector (the outer join of child RRs). /// </summary> protected internal override bool Combine(object[] srcs, TupleWritable dst) { return(true); }