/// <exception cref="System.IO.IOException"/>
public static void OutputRecords(OutputStream @out, bool useAscii, Unsigned16 firstRecordNumber
, Unsigned16 recordsToGenerate, Unsigned16 checksum)
{
byte[] row = new byte[100];
Unsigned16 recordNumber = new Unsigned16(firstRecordNumber);
Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
Checksum crc = new PureJavaCrc32();
Unsigned16 tmp = new Unsigned16();
lastRecordNumber.Add(recordsToGenerate);
Unsigned16 One = new Unsigned16(1);
Unsigned16 rand = Random16.SkipAhead(firstRecordNumber);
while (!recordNumber.Equals(lastRecordNumber))
{
Random16.NextRand(rand);
if (useAscii)
{
GenerateAsciiRecord(row, rand, recordNumber);
}
else
{
GenerateRecord(row, rand, recordNumber);
}
if (checksum != null)
{
crc.Reset();
crc.Update(row, 0, row.Length);
tmp.Set(crc.GetValue());
checksum.Add(tmp);
}
recordNumber.Add(One);
@out.Write(row);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
context)
{
if (Error.Equals(key))
{
foreach (Text val in values)
{
context.Write(key, val);
}
}
else
{
if (Checksum.Equals(key))
{
Unsigned16 tmp = new Unsigned16();
Unsigned16 sum = new Unsigned16();
foreach (Text val in values)
{
tmp.Set(val.ToString());
sum.Add(tmp);
}
context.Write(Checksum, new Text(sum.ToString()));
}
else
{
Text value = values.GetEnumerator().Next();
if (firstKey)
{
firstKey = false;
}
else
{
if (value.CompareTo(lastValue) < 0)
{
context.Write(Error, new Text("bad key partitioning:\n file " + lastKey + " key "
+ TextifyBytes(lastValue) + "\n file " + key + " key " + TextifyBytes(value)));
}
}
lastKey.Set(key);
lastValue.Set(value);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable row, NullWritable ignored, Mapper.Context
context)
{
if (rand == null)
{
rowId = new Unsigned16(row.Get());
rand = Random16.SkipAhead(rowId);
checksumCounter = context.GetCounter(TeraGen.Counters.Checksum);
}
Random16.NextRand(rand);
GenSort.GenerateRecord(buffer, rand, rowId);
key.Set(buffer, 0, TeraInputFormat.KeyLength);
value.Set(buffer, TeraInputFormat.KeyLength, TeraInputFormat.ValueLength);
context.Write(key, value);
crc32.Reset();
crc32.Update(buffer, 0, TeraInputFormat.KeyLength + TeraInputFormat.ValueLength);
checksum.Set(crc32.GetValue());
total.Add(checksum);
rowId.Add(One);
}
/// <summary>
/// Generate a "binary" record suitable for all sort benchmarks *except*
/// PennySort.
/// </summary>
internal static void GenerateRecord(byte[] recBuf, Unsigned16 rand, Unsigned16 recordNumber
)
{
/* generate the 10-byte key using the high 10 bytes of the 128-bit
* random number
*/
for (int i = 0; i < 10; ++i)
{
recBuf[i] = rand.GetByte(i);
}
/* add 2 bytes of "break" */
recBuf[10] = unchecked ((int)(0x00));
recBuf[11] = unchecked ((int)(0x11));
/* convert the 128-bit record number to 32 bits of ascii hexadecimal
* as the next 32 bytes of the record.
*/
for (int i_1 = 0; i_1 < 32; i_1++)
{
recBuf[12 + i_1] = unchecked ((byte)recordNumber.GetHexDigit(i_1));
}
/* add 4 bytes of "break" data */
recBuf[44] = unchecked ((byte)unchecked ((int)(0x88)));
recBuf[45] = unchecked ((byte)unchecked ((int)(0x99)));
recBuf[46] = unchecked ((byte)unchecked ((int)(0xAA)));
recBuf[47] = unchecked ((byte)unchecked ((int)(0xBB)));
/* add 48 bytes of filler based on low 48 bits of random number */
for (int i_2 = 0; i_2 < 12; ++i_2)
{
recBuf[48 + i_2 * 4] = recBuf[49 + i_2 * 4] = recBuf[50 + i_2 * 4] = recBuf[51 +
i_2 * 4] = unchecked ((byte)rand.GetHexDigit(20 + i_2));
}
/* add 4 bytes of "break" data */
recBuf[96] = unchecked ((byte)unchecked ((int)(0xCC)));
recBuf[97] = unchecked ((byte)unchecked ((int)(0xDD)));
recBuf[98] = unchecked ((byte)unchecked ((int)(0xEE)));
recBuf[99] = unchecked ((byte)unchecked ((int)(0xFF)));
}
/// <summary>
/// Generate an ascii record suitable for all sort benchmarks including
/// PennySort.
/// </summary>
internal static void GenerateAsciiRecord(byte[] recBuf, Unsigned16 rand, Unsigned16
recordNumber)
{
/* generate the 10-byte ascii key using mostly the high 64 bits.
*/
long temp = rand.GetHigh8();
if (temp < 0)
{
// use biginteger to avoid the negative sign problem
BigInteger bigTemp = MakeBigInteger(temp);
recBuf[0] = unchecked ((byte)((byte)(' ') + (bigTemp.Mod(NinetyFive))));
temp = bigTemp.Divide(NinetyFive);
}
else
{
recBuf[0] = unchecked ((byte)((byte)(' ') + (temp % 95)));
temp /= 95;
}
for (int i = 1; i < 8; ++i)
{
recBuf[i] = unchecked ((byte)((byte)(' ') + (temp % 95)));
temp /= 95;
}
temp = rand.GetLow8();
if (temp < 0)
{
BigInteger bigTemp = MakeBigInteger(temp);
recBuf[8] = unchecked ((byte)((byte)(' ') + (bigTemp.Mod(NinetyFive))));
temp = bigTemp.Divide(NinetyFive);
}
else
{
recBuf[8] = unchecked ((byte)((byte)(' ') + (temp % 95)));
temp /= 95;
}
recBuf[9] = unchecked ((byte)((byte)(' ') + (temp % 95)));
/* add 2 bytes of "break" */
recBuf[10] = (byte)(' ');
recBuf[11] = (byte)(' ');
/* convert the 128-bit record number to 32 bits of ascii hexadecimal
* as the next 32 bytes of the record.
*/
for (int i_1 = 0; i_1 < 32; i_1++)
{
recBuf[12 + i_1] = unchecked ((byte)recordNumber.GetHexDigit(i_1));
}
/* add 2 bytes of "break" data */
recBuf[44] = (byte)(' ');
recBuf[45] = (byte)(' ');
/* add 52 bytes of filler based on low 48 bits of random number */
for (int i_2 = 0; i_2 < 13; ++i_2)
{
recBuf[46 + i_2 * 4] = recBuf[47 + i_2 * 4] = recBuf[48 + i_2 * 4] = recBuf[49 +
i_2 * 4] = unchecked ((byte)rand.GetHexDigit(19 + i_2));
}
/* add 2 bytes of "break" data */
recBuf[98] = (byte)('\r');
/* nice for Windows */
recBuf[99] = (byte)('\n');
}
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Unsigned16 startingRecord = new Unsigned16();
Unsigned16 numberOfRecords;
OutputStream @out;
bool useAscii = false;
Unsigned16 checksum = null;
int i;
for (i = 0; i < args.Length; ++i)
{
string arg = args[i];
int argLength = arg.Length;
if (argLength >= 1 && arg[0] == '-')
{
if (argLength < 2)
{
Usage();
}
switch (arg[1])
{
case 'a':
{
useAscii = true;
break;
}
case 'b':
{
startingRecord = Unsigned16.FromDecimal(Sharpen.Runtime.Substring(arg, 2));
break;
}
case 'c':
{
checksum = new Unsigned16();
break;
}
default:
{
Usage();
break;
}
}
}
else
{
break;
}
}
if (args.Length - i != 2)
{
Usage();
}
numberOfRecords = Unsigned16.FromDecimal(args[i]);
@out = new FileOutputStream(args[i + 1]);
OutputRecords(@out, useAscii, startingRecord, numberOfRecords, checksum);
@out.Close();
if (checksum != null)
{
System.Console.Out.WriteLine(checksum);
}
}
