public virtual bool Equals(IntWritable v1, IntWritable v2)
{
int val1 = v1.Get();
int val2 = v2.Get();
return((val1 / 100) == (val2 / 100));
}
/// <summary>read the int value</summary>
/// <exception cref="System.IO.IOException"/>
internal static int ReadInt(DataInput @in)
{
IntWritable uInt = TlData.Get().UInt;
uInt.ReadFields(@in);
return(uInt.Get());
}
/// <exception cref="System.IO.IOException"/>
public override void Map(IntWritable key, TupleWritable val, OutputCollector <IntWritable
, IntWritable> @out, Reporter reporter)
{
int k = key.Get();
string kvstr = "Unexpected tuple: " + Stringify(key, val);
if (0 == k % (srcs * srcs))
{
for (int i = 0; i < val.Size(); ++i)
{
NUnit.Framework.Assert.IsTrue(kvstr, val.Get(i) is IntWritable);
int vali = ((IntWritable)val.Get(i)).Get();
NUnit.Framework.Assert.IsTrue(kvstr, (vali - i) * srcs == 10 * k);
}
}
else
{
for (int i = 0; i < val.Size(); ++i)
{
if (i == k % srcs)
{
NUnit.Framework.Assert.IsTrue(kvstr, val.Get(i) is IntWritable);
int vali = ((IntWritable)val.Get(i)).Get();
NUnit.Framework.Assert.IsTrue(kvstr, srcs * (vali - i) == 10 * (k - i));
}
else
{
NUnit.Framework.Assert.IsTrue(kvstr, !val.Has(i));
}
}
}
@out.Collect(key, one);
}
// Extracts a block (data enclosed within delimeters) ignoring escape
// sequences. Throws ParseException if an incomplete block is found else
// returns null.
/// <exception cref="Sharpen.ParseException"/>
private static string GetBlock(string str, char open, char close, IntWritable index
)
{
StringBuilder split = new StringBuilder();
int next = StringUtils.FindNext(str, open, StringUtils.EscapeChar, index.Get(), split
);
split.Length = 0;
// clear the buffer
if (next >= 0)
{
++next;
// move over '('
next = StringUtils.FindNext(str, close, StringUtils.EscapeChar, next, split);
if (next >= 0)
{
++next;
// move over ')'
index.Set(next);
return(split.ToString());
}
else
{
// found a block
throw new ParseException("Unexpected end of block", next);
}
}
return(null);
}
/// <summary>Test the most common use case.</summary>
/// <remarks>
/// Test the most common use case. Mark before start of the iteration and
/// reset at the end to go over the entire list
/// </remarks>
/// <param name="key"/>
/// <param name="values"/>
/// <returns/>
/// <exception cref="System.IO.IOException"/>
private static int Test0(IntWritable key, MarkableIterator <IntWritable> values)
{
int errors = 0;
IntWritable i;
AList <IntWritable> expectedValues = new AList <IntWritable>();
Log.Info("Executing TEST:0 for Key:" + key.ToString());
values.Mark();
Log.Info("TEST:0. Marking");
while (values.HasNext())
{
i = values.Next();
expectedValues.AddItem(i);
Log.Info(key + ":" + i);
}
values.Reset();
Log.Info("TEST:0. Reset");
int count = 0;
while (values.HasNext())
{
i = values.Next();
Log.Info(key + ":" + i);
if (i != expectedValues[count])
{
Log.Info("TEST:0. Check:1 Expected: " + expectedValues[count] + ", Got: " + i);
errors++;
return(errors);
}
count++;
}
Log.Info("TEST:0 Done");
return(errors);
}
/// <exception cref="System.IO.IOException"/>
private static int CountProduct(IntWritable key, Path[] src, Configuration conf)
{
int product = 1;
foreach (Path p in src)
{
int count = 0;
SequenceFile.Reader r = new SequenceFile.Reader(cluster.GetFileSystem(), p, conf);
IntWritable k = new IntWritable();
IntWritable v = new IntWritable();
while (r.Next(k, v))
{
if (k.Equals(key))
{
count++;
}
}
r.Close();
if (count != 0)
{
product *= count;
}
}
return(product);
}
/// <exception cref="System.IO.IOException"/>
private static void CreateFiles(int length, int numFiles, Random random, Job job)
{
TestCombineSequenceFileInputFormat.Range[] ranges = CreateRanges(length, numFiles
, random);
for (int i = 0; i < numFiles; i++)
{
Path file = new Path(workDir, "test_" + i + ".seq");
// create a file with length entries
SequenceFile.Writer writer = SequenceFile.CreateWriter(localFs, job.GetConfiguration
(), file, typeof(IntWritable), typeof(BytesWritable));
TestCombineSequenceFileInputFormat.Range range = ranges[i];
try
{
for (int j = range.start; j < range.end; j++)
{
IntWritable key = new IntWritable(j);
byte[] data = new byte[random.Next(10)];
random.NextBytes(data);
BytesWritable value = new BytesWritable(data);
writer.Append(key, value);
}
}
finally
{
writer.Close();
}
}
}
public virtual void TestOnFinalKey()
{
string TestMethodKey = "testOnFinalKey.mapfile";
int Size = 10;
MapFile.Writer writer = null;
MapFile.Reader reader = null;
try
{
writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(IntWritable));
for (int i = 0; i < Size; i++)
{
writer.Append(new IntWritable(i), new IntWritable(i));
}
writer.Close();
reader = CreateReader(TestMethodKey, typeof(IntWritable));
IntWritable expectedKey = new IntWritable(0);
reader.FinalKey(expectedKey);
Assert.Equal("testOnFinalKey not same !!!", expectedKey, new IntWritable
(9));
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("testOnFinalKey error !!!");
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
/// <summary>write the int value</summary>
/// <exception cref="System.IO.IOException"/>
internal static void WriteInt(int value, DataOutputStream @out)
{
IntWritable uInt = TlData.Get().UInt;
uInt.Set(value);
uInt.Write(@out);
}
private static string Stringify(IntWritable key, Writable val)
{
StringBuilder sb = new StringBuilder();
sb.Append("(" + key);
sb.Append("," + val + ")");
return(sb.ToString());
}
private void ValidateKeyValue_INNER_IDENTITY(IntWritable k, IntWritable v)
{
string kvstr = "Unexpected tuple: " + Stringify(k, v);
int key = k.Get();
NUnit.Framework.Assert.IsTrue(kvstr, (key % 2 == 0 && key / 2 <= Items));
NUnit.Framework.Assert.IsTrue(kvstr, v.Get() == 0);
}
public virtual void BinaryProtocolStub()
{
try
{
InitSoket();
// output code
WritableUtils.WriteVInt(dataOut, 50);
IntWritable wt = new IntWritable();
wt.Set(123);
WriteObject(wt, dataOut);
WriteObject(new Text("value"), dataOut);
// PARTITIONED_OUTPUT
WritableUtils.WriteVInt(dataOut, 51);
WritableUtils.WriteVInt(dataOut, 0);
WriteObject(wt, dataOut);
WriteObject(new Text("value"), dataOut);
// STATUS
WritableUtils.WriteVInt(dataOut, 52);
Text.WriteString(dataOut, "PROGRESS");
dataOut.Flush();
// progress
WritableUtils.WriteVInt(dataOut, 53);
dataOut.WriteFloat(0.55f);
// register counter
WritableUtils.WriteVInt(dataOut, 55);
// id
WritableUtils.WriteVInt(dataOut, 0);
Text.WriteString(dataOut, "group");
Text.WriteString(dataOut, "name");
// increment counter
WritableUtils.WriteVInt(dataOut, 56);
WritableUtils.WriteVInt(dataOut, 0);
WritableUtils.WriteVLong(dataOut, 2);
// map item
int intValue = WritableUtils.ReadVInt(dataInput);
System.Console.Out.WriteLine("intValue:" + intValue);
IntWritable iw = new IntWritable();
ReadObject(iw, dataInput);
System.Console.Out.WriteLine("key:" + iw.Get());
Text txt = new Text();
ReadObject(txt, dataInput);
System.Console.Out.WriteLine("value:" + txt.ToString());
// done
// end of session
WritableUtils.WriteVInt(dataOut, 54);
System.Console.Out.WriteLine("finish");
dataOut.Flush();
dataOut.Close();
}
catch (Exception x)
{
Sharpen.Runtime.PrintStackTrace(x);
}
finally
{
CloseSoket();
}
}
void next(OrcStruct next)
{
if (recordReader.hasNext())
{
long nextRowId = recordReader.getRowNumber();
// have to do initialization here, because the super's constructor
// calls next and thus we need to initialize before our constructor
// runs
if (next == null)
{
nextRecord = new OrcStruct(OrcRecordUpdater.FIELDS);
IntWritable operation =
new IntWritable(OrcRecordUpdater.INSERT_OPERATION);
nextRecord.setFieldValue(OrcRecordUpdater.OPERATION, operation);
nextRecord.setFieldValue(OrcRecordUpdater.CURRENT_TRANSACTION,
new LongWritable(0));
nextRecord.setFieldValue(OrcRecordUpdater.ORIGINAL_TRANSACTION,
new LongWritable(0));
nextRecord.setFieldValue(OrcRecordUpdater.BUCKET,
new IntWritable(bucket));
nextRecord.setFieldValue(OrcRecordUpdater.ROW_ID,
new LongWritable(nextRowId));
nextRecord.setFieldValue(OrcRecordUpdater.ROW,
recordReader.next(null));
}
else
{
nextRecord = next;
((IntWritable)next.getFieldValue(OrcRecordUpdater.OPERATION))
.set(OrcRecordUpdater.INSERT_OPERATION);
((LongWritable)next.getFieldValue(OrcRecordUpdater.ORIGINAL_TRANSACTION))
.set(0);
((IntWritable)next.getFieldValue(OrcRecordUpdater.BUCKET))
.set(bucket);
((LongWritable)next.getFieldValue(OrcRecordUpdater.CURRENT_TRANSACTION))
.set(0);
((LongWritable)next.getFieldValue(OrcRecordUpdater.ROW_ID))
.set(0);
nextRecord.setFieldValue(OrcRecordUpdater.ROW,
recordReader.next(OrcRecordUpdater.getRow(next)));
}
key.setValues(0L, bucket, nextRowId, 0L, 0);
if (maxKey != null && key.compareRow(maxKey) > 0)
{
if (LOG.isDebugEnabled())
{
LOG.debug("key " + key + " > maxkey " + maxKey);
}
nextRecord = null;
recordReader.close();
}
}
else
{
nextRecord = null;
recordReader.close();
}
}
public virtual void TestMerge()
{
string TestMethodKey = "testMerge.mapfile";
int Size = 10;
int Iterations = 5;
Path[] @in = new Path[5];
IList <int> expected = new AList <int>();
for (int j = 0; j < 5; j++)
{
using (MapFile.Writer writer = CreateWriter(TestMethodKey + "." + j, typeof(IntWritable
), typeof(Text)))
{
@in[j] = new Path(TestDir, TestMethodKey + "." + j);
for (int i = 0; i < Size; i++)
{
expected.AddItem(i + j);
writer.Append(new IntWritable(i + j), new Text("Value:" + (i + j)));
}
}
}
// Sort expected values
expected.Sort();
// Merge all 5 files
MapFile.Merger merger = new MapFile.Merger(conf);
merger.Merge(@in, true, new Path(TestDir, TestMethodKey));
using (MapFile.Reader reader = CreateReader(TestMethodKey, typeof(IntWritable)))
{
int start = 0;
// test iteration
Text startValue = new Text("Value:" + start);
int i = 0;
while (i++ < Iterations)
{
IEnumerator <int> expectedIterator = expected.GetEnumerator();
IntWritable key = new IntWritable(start);
Text value = startValue;
IntWritable prev = new IntWritable(start);
while (reader.Next(key, value))
{
Assert.True("Next key should be always equal or more", prev.Get
() <= key.Get());
Assert.Equal(expectedIterator.Next(), key.Get());
prev.Set(key.Get());
}
reader.Reset();
}
}
// inputs should be deleted
for (int j_1 = 0; j_1 < @in.Length; j_1++)
{
Path path = @in[j_1];
NUnit.Framework.Assert.IsFalse("inputs should be deleted", path.GetFileSystem(conf
).Exists(path));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestMembershipTest()
{
// write the file
FileSystem fs = FileSystem.GetLocal(conf);
Path qualifiedDirName = fs.MakeQualified(TestDir);
conf.SetInt("io.mapfile.bloom.size", 2048);
BloomMapFile.Writer writer = null;
BloomMapFile.Reader reader = null;
try
{
writer = new BloomMapFile.Writer(conf, fs, qualifiedDirName.ToString(), typeof(IntWritable
), typeof(Text));
IntWritable key = new IntWritable();
Text value = new Text();
for (int i = 0; i < 2000; i += 2)
{
key.Set(i);
value.Set("00" + i);
writer.Append(key, value);
}
writer.Close();
reader = new BloomMapFile.Reader(fs, qualifiedDirName.ToString(), conf);
// check false positives rate
int falsePos = 0;
int falseNeg = 0;
for (int i_1 = 0; i_1 < 2000; i_1++)
{
key.Set(i_1);
bool exists = reader.ProbablyHasKey(key);
if (i_1 % 2 == 0)
{
if (!exists)
{
falseNeg++;
}
}
else
{
if (exists)
{
falsePos++;
}
}
}
reader.Close();
fs.Delete(qualifiedDirName, true);
System.Console.Out.WriteLine("False negatives: " + falseNeg);
Assert.Equal(0, falseNeg);
System.Console.Out.WriteLine("False positives: " + falsePos);
Assert.True(falsePos < 2);
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestFormat()
{
JobConf job = new JobConf(conf);
Reporter reporter = Reporter.Null;
Random random = new Random();
long seed = random.NextLong();
Log.Info("seed = " + seed);
random.SetSeed(seed);
localFs.Delete(workDir, true);
FileInputFormat.SetInputPaths(job, workDir);
int length = 10000;
int numFiles = 10;
// create a file with various lengths
CreateFiles(length, numFiles, random);
// create a combine split for the files
InputFormat <IntWritable, BytesWritable> format = new CombineSequenceFileInputFormat
<IntWritable, BytesWritable>();
IntWritable key = new IntWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < 3; i++)
{
int numSplits = random.Next(length / (SequenceFile.SyncInterval / 20)) + 1;
Log.Info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.GetSplits(job, numSplits);
Log.Info("splitting: got = " + splits.Length);
// we should have a single split as the length is comfortably smaller than
// the block size
NUnit.Framework.Assert.AreEqual("We got more than one splits!", 1, splits.Length);
InputSplit split = splits[0];
NUnit.Framework.Assert.AreEqual("It should be CombineFileSplit", typeof(CombineFileSplit
), split.GetType());
// check each split
BitSet bits = new BitSet(length);
RecordReader <IntWritable, BytesWritable> reader = format.GetRecordReader(split, job
, reporter);
try
{
while (reader.Next(key, value))
{
NUnit.Framework.Assert.IsFalse("Key in multiple partitions.", bits.Get(key.Get())
);
bits.Set(key.Get());
}
}
finally
{
reader.Close();
}
NUnit.Framework.Assert.AreEqual("Some keys in no partition.", length, bits.Cardinality
());
}
}
/// <exception cref="System.IO.IOException"/>
private static void ForOffset(SequenceFile.Reader reader, IntWritable key, Text val
, int iter, long off, int expectedRecord)
{
val.Clear();
reader.Sync(off);
reader.Next(key, val);
Assert.Equal(key.Get(), expectedRecord);
string test = string.Format(RecFmt, expectedRecord, expectedRecord);
Assert.Equal("Invalid value " + val, 0, val.Find(test, 0));
}
/// <exception cref="System.IO.IOException"/>
public virtual void Reduce(IntWritable key, IEnumerator <IntWritable> values, OutputCollector
<Org.Apache.Hadoop.IO.Text, Org.Apache.Hadoop.IO.Text> output, Reporter reporter
)
{
int seen = 0;
while (values.HasNext())
{
seen += values.Next().Get();
}
NUnit.Framework.Assert.IsTrue("Bad count for " + key.Get(), Verify(key.Get(), seen
));
}
/// <exception cref="System.IO.IOException"/>
public static void WriteSequenceFile(SequenceFile.Writer writer, int numRecords)
{
IntWritable key = new IntWritable();
Text val = new Text();
for (int numWritten = 0; numWritten < numRecords; ++numWritten)
{
key.Set(numWritten);
RandomText(val, numWritten, Recordsize);
writer.Append(key, val);
}
writer.Close();
}
// found nothing
/// <summary>Parse a pre 0.21 counters string into a counter object.</summary>
/// <?/>
/// <?/>
/// <?/>
/// <param name="compactString">to parse</param>
/// <param name="counters">an empty counters object to hold the result</param>
/// <returns>the counters object holding the result</returns>
/// <exception cref="Sharpen.ParseException"/>
public static T ParseEscapedCompactString <C, G, T>(string compactString, T counters
)
where C : Counter
where G : CounterGroupBase <C>
where T : AbstractCounters <C, G>
{
IntWritable index = new IntWritable(0);
// Get the group to work on
string groupString = GetBlock(compactString, GroupOpen, GroupClose, index);
while (groupString != null)
{
IntWritable groupIndex = new IntWritable(0);
// Get the actual name
string groupName = StringInterner.WeakIntern(GetBlock(groupString, UnitOpen, UnitClose
, groupIndex));
groupName = StringInterner.WeakIntern(Unescape(groupName));
// Get the display name
string groupDisplayName = StringInterner.WeakIntern(GetBlock(groupString, UnitOpen
, UnitClose, groupIndex));
groupDisplayName = StringInterner.WeakIntern(Unescape(groupDisplayName));
// Get the counters
G group = counters.GetGroup(groupName);
group.SetDisplayName(groupDisplayName);
string counterString = GetBlock(groupString, CounterOpen, CounterClose, groupIndex
);
while (counterString != null)
{
IntWritable counterIndex = new IntWritable(0);
// Get the actual name
string counterName = StringInterner.WeakIntern(GetBlock(counterString, UnitOpen,
UnitClose, counterIndex));
counterName = StringInterner.WeakIntern(Unescape(counterName));
// Get the display name
string counterDisplayName = StringInterner.WeakIntern(GetBlock(counterString, UnitOpen
, UnitClose, counterIndex));
counterDisplayName = StringInterner.WeakIntern(Unescape(counterDisplayName));
// Get the value
long value = long.Parse(GetBlock(counterString, UnitOpen, UnitClose, counterIndex
));
// Add the counter
Counter counter = group.FindCounter(counterName);
counter.SetDisplayName(counterDisplayName);
counter.Increment(value);
// Get the next counter
counterString = GetBlock(groupString, CounterOpen, CounterClose, groupIndex);
}
groupString = GetBlock(compactString, GroupOpen, GroupClose, index);
}
return(counters);
}
public virtual void CopyFailed(TaskAttemptID mapId, MapHost host, bool readError,
bool connectExcpt)
{
lock (this)
{
host.Penalize();
int failures = 1;
if (failureCounts.Contains(mapId))
{
IntWritable x = failureCounts[mapId];
x.Set(x.Get() + 1);
failures = x.Get();
}
else
{
failureCounts[mapId] = new IntWritable(1);
}
string hostname = host.GetHostName();
IntWritable hostFailedNum = hostFailures[hostname];
// MAPREDUCE-6361: hostname could get cleanup from hostFailures in another
// thread with copySucceeded.
// In this case, add back hostname to hostFailures to get rid of NPE issue.
if (hostFailedNum == null)
{
hostFailures[hostname] = new IntWritable(1);
}
//report failure if already retried maxHostFailures times
bool hostFail = hostFailures[hostname].Get() > GetMaxHostFailures() ? true : false;
if (failures >= abortFailureLimit)
{
try
{
throw new IOException(failures + " failures downloading " + mapId);
}
catch (IOException ie)
{
reporter.ReportException(ie);
}
}
CheckAndInformMRAppMaster(failures, mapId, readError, connectExcpt, hostFail);
CheckReducerHealth();
long delay = (long)(InitialPenalty * Math.Pow(PenaltyGrowthRate, failures));
if (delay > maxDelay)
{
delay = maxDelay;
}
penalties.AddItem(new ShuffleSchedulerImpl.Penalty(host, delay));
failedShuffleCounter.Increment(1);
}
}
public virtual void HostFailed(string hostname)
{
lock (this)
{
if (hostFailures.Contains(hostname))
{
IntWritable x = hostFailures[hostname];
x.Set(x.Get() + 1);
}
else
{
hostFailures[hostname] = new IntWritable(1);
}
}
}
public virtual void TestLowSyncpoint()
{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.GetLocal(conf);
Path path = new Path(Runtime.GetProperty("test.build.data", "/tmp"), "sequencefile.sync.test"
);
IntWritable input = new IntWritable();
Text val = new Text();
SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, typeof(IntWritable
), typeof(Text));
try
{
WriteSequenceFile(writer, Numrecords);
for (int i = 0; i < 5; i++)
{
SequenceFile.Reader reader;
//try different SequenceFile.Reader constructors
if (i % 2 == 0)
{
reader = new SequenceFile.Reader(fs, path, conf);
}
else
{
FSDataInputStream @in = fs.Open(path);
long length = fs.GetFileStatus(path).GetLen();
int buffersize = conf.GetInt("io.file.buffer.size", 4096);
reader = new SequenceFile.Reader(@in, buffersize, 0L, length, conf);
}
try
{
ForOffset(reader, input, val, i, 0, 0);
ForOffset(reader, input, val, i, 65, 0);
ForOffset(reader, input, val, i, 2000, 21);
ForOffset(reader, input, val, i, 0, 0);
}
finally
{
reader.Close();
}
}
}
finally
{
fs.Delete(path, false);
}
}
public virtual void TestReaderKeyIteration()
{
string TestMethodKey = "testReaderKeyIteration.mapfile";
int Size = 10;
int Iterations = 5;
MapFile.Writer writer = null;
MapFile.Reader reader = null;
try
{
writer = CreateWriter(TestMethodKey, typeof(IntWritable), typeof(Text));
int start = 0;
for (int i = 0; i < Size; i++)
{
writer.Append(new IntWritable(i), new Text("Value:" + i));
}
writer.Close();
reader = CreateReader(TestMethodKey, typeof(IntWritable));
// test iteration
Writable startValue = new Text("Value:" + start);
int i_1 = 0;
while (i_1++ < Iterations)
{
IntWritable key = new IntWritable(start);
Writable value = startValue;
while (reader.Next(key, value))
{
NUnit.Framework.Assert.IsNotNull(key);
NUnit.Framework.Assert.IsNotNull(value);
}
reader.Reset();
}
Assert.True("reader seek error !!!", reader.Seek(new IntWritable
(Size / 2)));
NUnit.Framework.Assert.IsFalse("reader seek error !!!", reader.Seek(new IntWritable
(Size * 2)));
}
catch (IOException)
{
NUnit.Framework.Assert.Fail("reader seek error !!!");
}
finally
{
IOUtils.Cleanup(null, writer, reader);
}
}
public virtual void TestPipesPartitioner()
{
PipesPartitioner <IntWritable, Text> partitioner = new PipesPartitioner <IntWritable
, Text>();
JobConf configuration = new JobConf();
Submitter.GetJavaPartitioner(configuration);
partitioner.Configure(new JobConf());
IntWritable iw = new IntWritable(4);
// the cache empty
NUnit.Framework.Assert.AreEqual(0, partitioner.GetPartition(iw, new Text("test"),
2));
// set data into cache
PipesPartitioner.SetNextPartition(3);
// get data from cache
NUnit.Framework.Assert.AreEqual(3, partitioner.GetPartition(iw, new Text("test"),
2));
}
private void ValidateInnerKeyValue(IntWritable k, TupleWritable v, int tupleSize,
bool firstTuple, bool secondTuple)
{
string kvstr = "Unexpected tuple: " + Stringify(k, v);
NUnit.Framework.Assert.IsTrue(kvstr, v.Size() == tupleSize);
int key = k.Get();
IntWritable val0 = null;
IntWritable val1 = null;
LongWritable val2 = null;
NUnit.Framework.Assert.IsTrue(kvstr, key % 2 == 0 && key / 2 <= Items);
NUnit.Framework.Assert.IsTrue(kvstr, key % 3 == 0 && key / 3 <= Items);
NUnit.Framework.Assert.IsTrue(kvstr, key % 4 == 0 && key / 4 <= Items);
if (firstTuple)
{
TupleWritable v0 = ((TupleWritable)v.Get(0));
val0 = (IntWritable)v0.Get(0);
val1 = (IntWritable)v0.Get(1);
val2 = (LongWritable)v.Get(1);
}
else
{
if (secondTuple)
{
val0 = (IntWritable)v.Get(0);
TupleWritable v1 = ((TupleWritable)v.Get(1));
val1 = (IntWritable)v1.Get(0);
val2 = (LongWritable)v1.Get(1);
}
else
{
val0 = (IntWritable)v.Get(0);
val1 = (IntWritable)v.Get(1);
val2 = (LongWritable)v.Get(2);
}
}
NUnit.Framework.Assert.IsTrue(kvstr, val0.Get() == 0);
NUnit.Framework.Assert.IsTrue(kvstr, val1.Get() == 1);
NUnit.Framework.Assert.IsTrue(kvstr, val2.Get() == 2);
}
public virtual void TestEquality()
{
MapWritable map1 = new MapWritable();
MapWritable map2 = new MapWritable();
MapWritable map3 = new MapWritable();
IntWritable k1 = new IntWritable(5);
IntWritable k2 = new IntWritable(10);
Text value = new Text("value");
map1[k1] = value;
// equal
map2[k1] = value;
// equal
map3[k2] = value;
// not equal
Assert.True(map1.Equals(map2));
Assert.True(map2.Equals(map1));
NUnit.Framework.Assert.IsFalse(map1.Equals(map3));
Assert.Equal(map1.GetHashCode(), map2.GetHashCode());
NUnit.Framework.Assert.IsFalse(map1.GetHashCode() == map3.GetHashCode());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(IntWritable key, TupleWritable val, Mapper.Context context
)
{
int k = key.Get();
string kvstr = "Unexpected tuple: " + Stringify(key, val);
NUnit.Framework.Assert.IsTrue(kvstr, 0 == k % (srcs * srcs));
for (int i = 0; i < val.Size(); ++i)
{
int vali = ((IntWritable)val.Get(i)).Get();
NUnit.Framework.Assert.IsTrue(kvstr, (vali - i) * srcs == 10 * k);
}
context.Write(key, one);
// If the user modifies the key or any of the values in the tuple, it
// should not affect the rest of the join.
key.Set(-1);
if (val.Has(0))
{
((IntWritable)val.Get(0)).Set(0);
}
}
/// <exception cref="System.IO.IOException"/>
private static Path[] WriteSimpleSrc(Path testdir, Configuration conf, int srcs)
{
SequenceFile.Writer[] @out = null;
Path[] src = new Path[srcs];
try
{
@out = CreateWriters(testdir, conf, srcs, src);
int capacity = srcs * 2 + 1;
IntWritable key = new IntWritable();
IntWritable val = new IntWritable();
for (int k = 0; k < capacity; ++k)
{
for (int i = 0; i < srcs; ++i)
{
key.Set(k % srcs == 0 ? k * srcs : k * srcs + i);
val.Set(10 * k + i);
@out[i].Append(key, val);
if (i == k)
{
// add duplicate key
@out[i].Append(key, val);
}
}
}
}
finally
{
if (@out != null)
{
for (int i = 0; i < srcs; ++i)
{
if (@out[i] != null)
{
@out[i].Close();
}
}
}
}
return(src);
}
/// <exception cref="System.IO.IOException"/>
private static void CheckOuterConsistency(Job job, Path[] src)
{
Path outf = FileOutputFormat.GetOutputPath(job);
FileStatus[] outlist = cluster.GetFileSystem().ListStatus(outf, new Utils.OutputFileUtils.OutputFilesFilter
());
NUnit.Framework.Assert.AreEqual("number of part files is more than 1. It is" + outlist
.Length, 1, outlist.Length);
NUnit.Framework.Assert.IsTrue("output file with zero length" + outlist[0].GetLen(
), 0 < outlist[0].GetLen());
SequenceFile.Reader r = new SequenceFile.Reader(cluster.GetFileSystem(), outlist[
0].GetPath(), job.GetConfiguration());
IntWritable k = new IntWritable();
IntWritable v = new IntWritable();
while (r.Next(k, v))
{
NUnit.Framework.Assert.AreEqual("counts does not match", v.Get(), CountProduct(k,
src, job.GetConfiguration()));
}
r.Close();
}
void next(OrcStruct next)
{
if (recordReader.hasNext())
{
long nextRowId = recordReader.getRowNumber();
// have to do initialization here, because the super's constructor
// calls next and thus we need to initialize before our constructor
// runs
if (next == null)
{
nextRecord = new OrcStruct(OrcRecordUpdater.FIELDS);
IntWritable operation =
new IntWritable(OrcRecordUpdater.INSERT_OPERATION);
nextRecord.setFieldValue(OrcRecordUpdater.OPERATION, operation);
nextRecord.setFieldValue(OrcRecordUpdater.CURRENT_TRANSACTION,
new LongWritable(0));
nextRecord.setFieldValue(OrcRecordUpdater.ORIGINAL_TRANSACTION,
new LongWritable(0));
nextRecord.setFieldValue(OrcRecordUpdater.BUCKET,
new IntWritable(bucket));
nextRecord.setFieldValue(OrcRecordUpdater.ROW_ID,
new LongWritable(nextRowId));
nextRecord.setFieldValue(OrcRecordUpdater.ROW,
recordReader.next(null));
}
else
{
nextRecord = next;
((IntWritable)next.getFieldValue(OrcRecordUpdater.OPERATION))
.set(OrcRecordUpdater.INSERT_OPERATION);
((LongWritable)next.getFieldValue(OrcRecordUpdater.ORIGINAL_TRANSACTION))
.set(0);
((IntWritable)next.getFieldValue(OrcRecordUpdater.BUCKET))
.set(bucket);
((LongWritable)next.getFieldValue(OrcRecordUpdater.CURRENT_TRANSACTION))
.set(0);
((LongWritable)next.getFieldValue(OrcRecordUpdater.ROW_ID))
.set(0);
nextRecord.setFieldValue(OrcRecordUpdater.ROW,
recordReader.next(OrcRecordUpdater.getRow(next)));
}
key.setValues(0L, bucket, nextRowId, 0L, 0);
if (maxKey != null && key.compareRow(maxKey) > 0)
{
if (LOG.isDebugEnabled())
{
LOG.debug("key " + key + " > maxkey " + maxKey);
}
nextRecord = null;
recordReader.close();
}
}
else
{
nextRecord = null;
recordReader.close();
}
}
public void reduce(IntWritable key, Iterable<Text> values, Context context)
