/// <exception cref="System.IO.IOException"/>
public SequenceFileAsTextRecordReader(Configuration conf, FileSplit split)
{
sequenceFileRecordReader = new SequenceFileRecordReader <WritableComparable, Writable
>(conf, split);
innerKey = sequenceFileRecordReader.CreateKey();
innerValue = sequenceFileRecordReader.CreateValue();
}
/// <exception cref="System.Exception"/>
public virtual int TestFormat(Configuration conf, int tupleSize, bool firstTuple,
bool secondTuple, TestJoinProperties.TestType ttype)
{
Job job = Job.GetInstance(conf);
CompositeInputFormat format = new CompositeInputFormat();
int count = 0;
foreach (InputSplit split in (IList <InputSplit>)format.GetSplits(job))
{
TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(conf
);
RecordReader reader = format.CreateRecordReader(split, context);
MapContext mcontext = new MapContextImpl(conf, context.GetTaskAttemptID(), reader
, null, null, MapReduceTestUtil.CreateDummyReporter(), split);
reader.Initialize(split, mcontext);
WritableComparable key = null;
Writable value = null;
while (reader.NextKeyValue())
{
key = (WritableComparable)reader.GetCurrentKey();
value = (Writable)reader.GetCurrentValue();
ValidateKeyValue(key, value, tupleSize, firstTuple, secondTuple, ttype);
count++;
}
}
return(count);
}
// Mapper that sleeps for a long time.
// Used for running a job that will be killed
/// <exception cref="System.IO.IOException"/>
protected override void Map <_T0>(WritableComparable <_T0> key, Writable value, Mapper.Context
context)
{
try
{
Sharpen.Thread.Sleep(1000000);
}
catch (Exception)
{
}
}
/// <summary>
/// Break the prefix string into moves (a sequence of integer row ids that
/// will be selected for each column in order).
/// </summary>
/// <remarks>
/// Break the prefix string into moves (a sequence of integer row ids that
/// will be selected for each column in order). Find all solutions with
/// that prefix.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
protected override void Map <_T0>(WritableComparable <_T0> key, Text value, Mapper.Context
context)
{
prefixString = value;
StringTokenizer itr = new StringTokenizer(prefixString.ToString(), ",");
int[] prefix = new int[depth];
int idx = 0;
while (itr.HasMoreTokens())
{
string num = itr.NextToken();
prefix[idx++] = System.Convert.ToInt32(num);
}
pent.Solve(prefix);
}
public virtual void TestSortedMapWritable()
{
Text[] keys = new Text[] { new Text("key1"), new Text("key2"), new Text("key3") };
BytesWritable[] values = new BytesWritable[] { new BytesWritable(Runtime.GetBytesForString
("value1")), new BytesWritable(Runtime.GetBytesForString("value2")), new
BytesWritable(Runtime.GetBytesForString("value3")) };
SortedMapWritable inMap = new SortedMapWritable();
for (int i = 0; i < keys.Length; i++)
{
inMap[keys[i]] = values[i];
}
Assert.Equal(0, inMap.FirstKey().CompareTo(keys[0]));
Assert.Equal(0, inMap.LastKey().CompareTo(keys[2]));
SortedMapWritable outMap = new SortedMapWritable(inMap);
Assert.Equal(inMap.Count, outMap.Count);
foreach (KeyValuePair <WritableComparable, Writable> e in inMap)
{
Assert.True(outMap.Contains(e.Key));
Assert.Equal(0, ((WritableComparable)outMap[e.Key]).CompareTo(
e.Value));
}
// Now for something a little harder...
Text[] maps = new Text[] { new Text("map1"), new Text("map2") };
SortedMapWritable mapOfMaps = new SortedMapWritable();
mapOfMaps[maps[0]] = inMap;
mapOfMaps[maps[1]] = outMap;
SortedMapWritable copyOfMapOfMaps = new SortedMapWritable(mapOfMaps);
for (int i_1 = 0; i_1 < maps.Length; i_1++)
{
Assert.True(copyOfMapOfMaps.Contains(maps[i_1]));
SortedMapWritable a = (SortedMapWritable)mapOfMaps[maps[i_1]];
SortedMapWritable b = (SortedMapWritable)copyOfMapOfMaps[maps[i_1]];
Assert.Equal(a.Count, b.Count);
foreach (Writable key in a.Keys)
{
Assert.True(b.Contains(key));
// This will work because we know what we put into each set
WritableComparable aValue = (WritableComparable)a[key];
WritableComparable bValue = (WritableComparable)b[key];
Assert.Equal(0, aValue.CompareTo(bValue));
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Reduce(WritableComparable key, IEnumerator values, OutputCollector
output, Reporter reporter)
{
if (first)
{
first = false;
MapOutputFile mapOutputFile = new MROutputFiles();
mapOutputFile.SetConf(conf);
Path input = mapOutputFile.GetInputFile(0);
FileSystem fs = FileSystem.Get(conf);
NUnit.Framework.Assert.IsTrue("reduce input exists " + input, fs.Exists(input));
SequenceFile.Reader rdr = new SequenceFile.Reader(fs, input, conf);
NUnit.Framework.Assert.AreEqual("is reduce input compressed " + input, compressInput
, rdr.IsCompressed());
rdr.Close();
}
}
public override int Compare(WritableComparable v1, WritableComparable v2)
{
int val1 = ((IntWritable)(v1)).Get() / 100;
int val2 = ((IntWritable)(v2)).Get() / 100;
if (val1 < val2)
{
return(1);
}
else
{
if (val1 > val2)
{
return(-1);
}
else
{
return(0);
}
}
}
/// <exception cref="System.IO.IOException"/>
private void ValidateKeyValue <_T0>(WritableComparable <_T0> k, Writable v, int tupleSize
, bool firstTuple, bool secondTuple, TestJoinProperties.TestType ttype)
{
System.Console.Out.WriteLine("out k:" + k + " v:" + v);
if (ttype.Equals(TestJoinProperties.TestType.OuterAssociativity))
{
ValidateOuterKeyValue((IntWritable)k, (TupleWritable)v, tupleSize, firstTuple, secondTuple
);
}
else
{
if (ttype.Equals(TestJoinProperties.TestType.InnerAssociativity))
{
ValidateInnerKeyValue((IntWritable)k, (TupleWritable)v, tupleSize, firstTuple, secondTuple
);
}
}
if (ttype.Equals(TestJoinProperties.TestType.InnerIdentity))
{
ValidateKeyValue_INNER_IDENTITY((IntWritable)k, (IntWritable)v);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map <_T0>(WritableComparable <_T0> key, Text val, Mapper.Context
context)
{
context.Write(new IntWritable(System.Convert.ToInt32(val.ToString().Trim())), new
IntWritable(1));
}
/// <summary>Read the next key in a set into <code>key</code>.</summary>
/// <remarks>
/// Read the next key in a set into <code>key</code>. Returns
/// true if such a key exists and false when at the end of the set.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual bool Next(WritableComparable key)
{
return(Next(key, NullWritable.Get()));
}
// javadoc inherited
/// <exception cref="System.IO.IOException"/>
public override bool Seek(WritableComparable key)
{
return(base.Seek(key));
}
// no public ctor
/// <summary>Append a key to a set.</summary>
/// <remarks>
/// Append a key to a set. The key must be strictly greater than the
/// previous key added to the set.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void Append(WritableComparable key)
{
Append(key, NullWritable.Get());
}
public override int Compare(WritableComparable a, WritableComparable b)
{
TestComparators.MyWritable aa = (TestComparators.MyWritable)a;
TestComparators.MyWritable bb = (TestComparators.MyWritable)b;
return(aa.j - bb.j);
}
public override int Compare(WritableComparable v1, WritableComparable v2)
{
return(-base.Compare(v1, v2));
}
/// <exception cref="System.IO.IOException"/>
public virtual void MapItem(WritableComparable key, Writable value)
{
WritableUtils.WriteVInt(stream, BinaryProtocol.MessageType.MapItem.code);
WriteObject(key);
WriteObject(value);
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReduceKey(WritableComparable key)
{
WritableUtils.WriteVInt(stream, BinaryProtocol.MessageType.ReduceKey.code);
WriteObject(key);
}
// Mapper that fails
/// <exception cref="System.IO.IOException"/>
protected override void Map <_T0>(WritableComparable <_T0> key, Writable value, Mapper.Context
context)
{
throw new RuntimeException("failing map");
}
/// <exception cref="System.IO.IOException"/>
public override void Write <_T0>(WritableComparable <_T0> key, Writable value)
{
@out.Append(key, value);
}