/// <exception cref="System.IO.IOException"/> public static void OutputRecords(OutputStream @out, bool useAscii, Unsigned16 firstRecordNumber , Unsigned16 recordsToGenerate, Unsigned16 checksum) { byte[] row = new byte[100]; Unsigned16 recordNumber = new Unsigned16(firstRecordNumber); Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber); Checksum crc = new PureJavaCrc32(); Unsigned16 tmp = new Unsigned16(); lastRecordNumber.Add(recordsToGenerate); Unsigned16 One = new Unsigned16(1); Unsigned16 rand = Random16.SkipAhead(firstRecordNumber); while (!recordNumber.Equals(lastRecordNumber)) { Random16.NextRand(rand); if (useAscii) { GenerateAsciiRecord(row, rand, recordNumber); } else { GenerateRecord(row, rand, recordNumber); } if (checksum != null) { crc.Reset(); crc.Update(row, 0, row.Length); tmp.Set(crc.GetValue()); checksum.Add(tmp); } recordNumber.Add(One); @out.Write(row); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(Text key, Text value, Mapper.Context context) { if (lastKey == null) { FileSplit fs = (FileSplit)context.GetInputSplit(); filename = GetFilename(fs); context.Write(new Text(filename + ":begin"), key); lastKey = new Text(); } else { if (key.CompareTo(lastKey) < 0) { context.Write(Error, new Text("misorder in " + filename + " between " + TextifyBytes (lastKey) + " and " + TextifyBytes(key))); } } // compute the crc of the key and value and add it to the sum crc32.Reset(); crc32.Update(key.GetBytes(), 0, key.GetLength()); crc32.Update(value.GetBytes(), 0, value.GetLength()); tmp.Set(crc32.GetValue()); checksum.Add(tmp); lastKey.Set(key); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context context) { if (Error.Equals(key)) { foreach (Text val in values) { context.Write(key, val); } } else { if (Checksum.Equals(key)) { Unsigned16 tmp = new Unsigned16(); Unsigned16 sum = new Unsigned16(); foreach (Text val in values) { tmp.Set(val.ToString()); sum.Add(tmp); } context.Write(Checksum, new Text(sum.ToString())); } else { Text value = values.GetEnumerator().Next(); if (firstKey) { firstKey = false; } else { if (value.CompareTo(lastValue) < 0) { context.Write(Error, new Text("bad key partitioning:\n file " + lastKey + " key " + TextifyBytes(lastValue) + "\n file " + key + " key " + TextifyBytes(value))); } } lastKey.Set(key); lastValue.Set(value); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable row, NullWritable ignored, Mapper.Context context) { if (rand == null) { rowId = new Unsigned16(row.Get()); rand = Random16.SkipAhead(rowId); checksumCounter = context.GetCounter(TeraGen.Counters.Checksum); } Random16.NextRand(rand); GenSort.GenerateRecord(buffer, rand, rowId); key.Set(buffer, 0, TeraInputFormat.KeyLength); value.Set(buffer, TeraInputFormat.KeyLength, TeraInputFormat.ValueLength); context.Write(key, value); crc32.Reset(); crc32.Update(buffer, 0, TeraInputFormat.KeyLength + TeraInputFormat.ValueLength); checksum.Set(crc32.GetValue()); total.Add(checksum); rowId.Add(One); }