/// <exception cref="System.IO.IOException"/>
internal WrappedRecordReader(int id, RecordReader <K, U> rr, Type cmpcl, Configuration
conf)
{
// index at which values will be inserted in collector
// key at the top of this RR
// value assoc with khead
this.id = id;
this.rr = rr;
this.conf = (conf == null) ? new Configuration() : conf;
khead = rr.CreateKey();
vhead = rr.CreateValue();
try
{
cmp = (null == cmpcl) ? WritableComparator.Get(khead.GetType(), this.conf) : System.Activator.CreateInstance
(cmpcl);
}
catch (InstantiationException e)
{
throw (IOException)Sharpen.Extensions.InitCause(new IOException(), e);
}
catch (MemberAccessException e)
{
throw (IOException)Sharpen.Extensions.InitCause(new IOException(), e);
}
vjoin = new StreamBackedIterator <U>();
Next();
}
public virtual void TestBakedUserComparator()
{
TestComparators.MyWritable a = new TestComparators.MyWritable(8, 8);
TestComparators.MyWritable b = new TestComparators.MyWritable(7, 9);
NUnit.Framework.Assert.IsTrue(a.CompareTo(b) > 0);
NUnit.Framework.Assert.IsTrue(WritableComparator.Get(typeof(TestComparators.MyWritable
)).Compare(a, b) < 0);
}
public virtual void TestCompareUnequalWritables()
{
var bTrue = WriteWritable(new BooleanWritable(true));
var bFalse = WriteWritable(new BooleanWritable(false));
WritableComparator writableComparator = WritableComparator.Get(typeof(BooleanWritable));
0.ShouldEqual(Compare(writableComparator, bTrue, bTrue));
0.ShouldEqual(Compare(writableComparator, bTrue, bTrue));
0.ShouldEqual(Compare(writableComparator, bFalse, bFalse));
1.ShouldEqual(Compare(writableComparator, bTrue, bFalse));
(-1).ShouldEqual(Compare(writableComparator, bFalse, bTrue));
}
/// <summary>Add a RecordReader to this collection.</summary>
/// <remarks>
/// Add a RecordReader to this collection.
/// The id() of a RecordReader determines where in the Tuple its
/// entry will appear. Adding RecordReaders with the same id has
/// undefined behavior.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void Add(ComposableRecordReader <K, V> rr)
{
kids[rr.Id()] = rr;
if (null == q)
{
cmp = WritableComparator.Get(rr.CreateKey().GetType(), conf);
q = new PriorityQueue <ComposableRecordReader <K, object> >(3, new _IComparer_136(this
));
}
if (rr.HasNext())
{
q.AddItem(rr);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Initialize(InputSplit split, TaskAttemptContext context)
{
rr.Initialize(split, context);
conf = context.GetConfiguration();
NextKeyValue();
if (!empty)
{
keyclass = key.GetType().AsSubclass <WritableComparable>();
valueclass = value.GetType();
if (cmp == null)
{
cmp = WritableComparator.Get(keyclass, conf);
}
}
}
/// <exception cref="System.IO.IOException"/>
public WritableSortable(int j)
{
seed = r.NextLong();
r.SetSeed(seed);
Text t = new Text();
StringBuilder sb = new StringBuilder();
indices = new int[j];
offsets = new int[j];
check = new string[j];
DataOutputBuffer dob = new DataOutputBuffer();
for (int i = 0; i < j; ++i)
{
indices[i] = i;
offsets[i] = dob.GetLength();
GenRandom(t, r.Next(15) + 1, sb);
t.Write(dob);
check[i] = t.ToString();
}
eob = dob.GetLength();
bytes = dob.GetData();
comparator = WritableComparator.Get(typeof(Org.Apache.Hadoop.IO.Text));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Initialize(InputSplit split, TaskAttemptContext context)
{
if (kids != null)
{
for (int i = 0; i < kids.Length; ++i)
{
kids[i].Initialize(((CompositeInputSplit)split).Get(i), context);
if (kids[i].Key() == null)
{
continue;
}
// get keyclass
if (keyclass == null)
{
keyclass = kids[i].CreateKey().GetType().AsSubclass <WritableComparable>();
}
// create priority queue
if (null == q)
{
cmp = WritableComparator.Get(keyclass, conf);
q = new PriorityQueue <ComposableRecordReader <K, object> >(3, new _IComparer_114(this
));
}
// Explicit check for key class agreement
if (!keyclass.Equals(kids[i].Key().GetType()))
{
throw new InvalidCastException("Child key classes fail to agree");
}
// add the kid to priority queue if it has any elements
if (kids[i].HasNext())
{
q.AddItem(kids[i]);
}
}
}
}
/// <summary>Create a set naming the element class and compression type.</summary>
/// <exception cref="System.IO.IOException"/>
public Writer(Configuration conf, FileSystem fs, string dirName, Type keyClass, SequenceFile.CompressionType
compress)
: this(conf, fs, dirName, WritableComparator.Get(keyClass, conf), compress)
{
}
/// <exception cref="System.IO.IOException"/>
public virtual void RunValueIterator(Path tmpDir, TestReduceTask.Pair[] vals, Configuration
conf, CompressionCodec codec)
{
FileSystem localFs = FileSystem.GetLocal(conf);
FileSystem rfs = ((LocalFileSystem)localFs).GetRaw();
Path path = new Path(tmpDir, "data.in");
IFile.Writer <Text, Text> writer = new IFile.Writer <Text, Text>(conf, rfs.Create(path
), typeof(Text), typeof(Text), codec, null);
foreach (TestReduceTask.Pair p in vals)
{
writer.Append(new Text(p.key), new Text(p.value));
}
writer.Close();
RawKeyValueIterator rawItr = Merger.Merge <Text, Text>(conf, rfs, codec, new Path[]
{ path }, false, conf.GetInt(JobContext.IoSortFactor, 100), tmpDir, new Text.Comparator
(), new TestReduceTask.NullProgress(), null, null, null);
Task.ValuesIterator valItr = new Task.ValuesIterator <Text, Text>(rawItr, WritableComparator
.Get(typeof(Text)), typeof(Text), typeof(Text), conf, new TestReduceTask.NullProgress
());
// WritableComparators are not generic
int i = 0;
while (valItr.More())
{
object key = valItr.GetKey();
string keyString = key.ToString();
// make sure it matches!
NUnit.Framework.Assert.AreEqual(vals[i].key, keyString);
// must have at least 1 value!
NUnit.Framework.Assert.IsTrue(valItr.HasNext());
while (valItr.HasNext())
{
string valueString = valItr.Next().ToString();
// make sure the values match
NUnit.Framework.Assert.AreEqual(vals[i].value, valueString);
// make sure the keys match
NUnit.Framework.Assert.AreEqual(vals[i].key, valItr.GetKey().ToString());
i += 1;
}
// make sure the key hasn't changed under the hood
NUnit.Framework.Assert.AreEqual(keyString, valItr.GetKey().ToString());
valItr.NextKey();
}
NUnit.Framework.Assert.AreEqual(vals.Length, i);
// make sure we have progress equal to 1.0
NUnit.Framework.Assert.AreEqual(1.0f, rawItr.GetProgress().Get());
}