Пример #1
0
        /// <exception cref="System.IO.IOException"/>
        public static void OutputRecords(OutputStream @out, bool useAscii, Unsigned16 firstRecordNumber
                                         , Unsigned16 recordsToGenerate, Unsigned16 checksum)
        {
            byte[]     row              = new byte[100];
            Unsigned16 recordNumber     = new Unsigned16(firstRecordNumber);
            Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
            Checksum   crc              = new PureJavaCrc32();
            Unsigned16 tmp              = new Unsigned16();

            lastRecordNumber.Add(recordsToGenerate);
            Unsigned16 One  = new Unsigned16(1);
            Unsigned16 rand = Random16.SkipAhead(firstRecordNumber);

            while (!recordNumber.Equals(lastRecordNumber))
            {
                Random16.NextRand(rand);
                if (useAscii)
                {
                    GenerateAsciiRecord(row, rand, recordNumber);
                }
                else
                {
                    GenerateRecord(row, rand, recordNumber);
                }
                if (checksum != null)
                {
                    crc.Reset();
                    crc.Update(row, 0, row.Length);
                    tmp.Set(crc.GetValue());
                    checksum.Add(tmp);
                }
                recordNumber.Add(One);
                @out.Write(row);
            }
        }
Пример #2
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 protected override void Map(Text key, Text value, Mapper.Context context)
 {
     if (lastKey == null)
     {
         FileSplit fs = (FileSplit)context.GetInputSplit();
         filename = GetFilename(fs);
         context.Write(new Text(filename + ":begin"), key);
         lastKey = new Text();
     }
     else
     {
         if (key.CompareTo(lastKey) < 0)
         {
             context.Write(Error, new Text("misorder in " + filename + " between " + TextifyBytes
                                               (lastKey) + " and " + TextifyBytes(key)));
         }
     }
     // compute the crc of the key and value and add it to the sum
     crc32.Reset();
     crc32.Update(key.GetBytes(), 0, key.GetLength());
     crc32.Update(value.GetBytes(), 0, value.GetLength());
     tmp.Set(crc32.GetValue());
     checksum.Add(tmp);
     lastKey.Set(key);
 }
Пример #3
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
                                context)
 {
     if (Error.Equals(key))
     {
         foreach (Text val in values)
         {
             context.Write(key, val);
         }
     }
     else
     {
         if (Checksum.Equals(key))
         {
             Unsigned16 tmp = new Unsigned16();
             Unsigned16 sum = new Unsigned16();
             foreach (Text val in values)
             {
                 tmp.Set(val.ToString());
                 sum.Add(tmp);
             }
             context.Write(Checksum, new Text(sum.ToString()));
         }
         else
         {
             Text value = values.GetEnumerator().Next();
             if (firstKey)
             {
                 firstKey = false;
             }
             else
             {
                 if (value.CompareTo(lastValue) < 0)
                 {
                     context.Write(Error, new Text("bad key partitioning:\n  file " + lastKey + " key "
                                                   + TextifyBytes(lastValue) + "\n  file " + key + " key " + TextifyBytes(value)));
                 }
             }
             lastKey.Set(key);
             lastValue.Set(value);
         }
     }
 }
Пример #4
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 protected override void Map(LongWritable row, NullWritable ignored, Mapper.Context
                             context)
 {
     if (rand == null)
     {
         rowId           = new Unsigned16(row.Get());
         rand            = Random16.SkipAhead(rowId);
         checksumCounter = context.GetCounter(TeraGen.Counters.Checksum);
     }
     Random16.NextRand(rand);
     GenSort.GenerateRecord(buffer, rand, rowId);
     key.Set(buffer, 0, TeraInputFormat.KeyLength);
     value.Set(buffer, TeraInputFormat.KeyLength, TeraInputFormat.ValueLength);
     context.Write(key, value);
     crc32.Reset();
     crc32.Update(buffer, 0, TeraInputFormat.KeyLength + TeraInputFormat.ValueLength);
     checksum.Set(crc32.GetValue());
     total.Add(checksum);
     rowId.Add(One);
 }