/// <exception cref="System.IO.IOException"/>
public static void OutputRecords(OutputStream @out, bool useAscii, Unsigned16 firstRecordNumber
, Unsigned16 recordsToGenerate, Unsigned16 checksum)
{
byte[] row = new byte[100];
Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
Checksum crc = new PureJavaCrc32();
Unsigned16 tmp = new Unsigned16();
lastRecordNumber.Add(recordsToGenerate);
Unsigned16 One = new Unsigned16(1);
Unsigned16 rand = Random16.SkipAhead(firstRecordNumber);
while (!recordNumber.Equals(lastRecordNumber))
{
Random16.NextRand(rand);
if (useAscii)
{
GenerateAsciiRecord(row, rand, recordNumber);
}
else
{
GenerateRecord(row, rand, recordNumber);
}
if (checksum != null)
{
crc.Reset();
crc.Update(row, 0, row.Length);
tmp.Set(crc.GetValue());
checksum.Add(tmp);
}
recordNumber.Add(One);
@out.Write(row);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(Text key, Text value, Mapper.Context context)
{
if (lastKey == null)
{
FileSplit fs = (FileSplit)context.GetInputSplit();
filename = GetFilename(fs);
context.Write(new Text(filename + ":begin"), key);
lastKey = new Text();
}
else
{
if (key.CompareTo(lastKey) < 0)
{
context.Write(Error, new Text("misorder in " + filename + " between " + TextifyBytes
(lastKey) + " and " + TextifyBytes(key)));
}
}
// compute the crc of the key and value and add it to the sum
crc32.Reset();
crc32.Update(key.GetBytes(), 0, key.GetLength());
crc32.Update(value.GetBytes(), 0, value.GetLength());
tmp.Set(crc32.GetValue());
checksum.Add(tmp);
lastKey.Set(key);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
if (Error.Equals(key))
{
foreach (Text val in values)
{
context.Write(key, val);
}
}
else
{
if (Checksum.Equals(key))
{
Unsigned16 tmp = new Unsigned16();
Unsigned16 sum = new Unsigned16();
foreach (Text val in values)
{
tmp.Set(val.ToString());
sum.Add(tmp);
}
context.Write(Checksum, new Text(sum.ToString()));
}
else
{
Text value = values.GetEnumerator().Next();
if (firstKey)
{
firstKey = false;
}
else
{
if (value.CompareTo(lastValue) < 0)
{
context.Write(Error, new Text("bad key partitioning:\n file " + lastKey + " key "
+ TextifyBytes(lastValue) + "\n file " + key + " key " + TextifyBytes(value)));
}
}
lastKey.Set(key);
lastValue.Set(value);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable row, NullWritable ignored, Mapper.Context
context)
{
if (rand == null)
{
rowId = new Unsigned16(row.Get());
rand = Random16.SkipAhead(rowId);
checksumCounter = context.GetCounter(TeraGen.Counters.Checksum);
}
Random16.NextRand(rand);
GenSort.GenerateRecord(buffer, rand, rowId);
key.Set(buffer, 0, TeraInputFormat.KeyLength);
value.Set(buffer, TeraInputFormat.KeyLength, TeraInputFormat.ValueLength);
context.Write(key, value);
crc32.Reset();
crc32.Update(buffer, 0, TeraInputFormat.KeyLength + TeraInputFormat.ValueLength);
checksum.Set(crc32.GetValue());
total.Add(checksum);
rowId.Add(One);
}