Exemplo n.º 1
0
        /// <exception cref="System.IO.IOException"/>
        public static void OutputRecords(OutputStream @out, bool useAscii, Unsigned16 firstRecordNumber
                                         , Unsigned16 recordsToGenerate, Unsigned16 checksum)
        {
            byte[]     row              = new byte[100];
            Unsigned16 recordNumber     = new Unsigned16(firstRecordNumber);
            Unsigned16 lastRecordNumber = new Unsigned16(firstRecordNumber);
            Checksum   crc              = new PureJavaCrc32();
            Unsigned16 tmp              = new Unsigned16();

            lastRecordNumber.Add(recordsToGenerate);
            Unsigned16 One  = new Unsigned16(1);
            Unsigned16 rand = Random16.SkipAhead(firstRecordNumber);

            while (!recordNumber.Equals(lastRecordNumber))
            {
                Random16.NextRand(rand);
                if (useAscii)
                {
                    GenerateAsciiRecord(row, rand, recordNumber);
                }
                else
                {
                    GenerateRecord(row, rand, recordNumber);
                }
                if (checksum != null)
                {
                    crc.Reset();
                    crc.Update(row, 0, row.Length);
                    tmp.Set(crc.GetValue());
                    checksum.Add(tmp);
                }
                recordNumber.Add(One);
                @out.Write(row);
            }
        }
Exemplo n.º 2
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 protected override void Reduce(Text key, IEnumerable <Text> values, Reducer.Context
                                context)
 {
     if (Error.Equals(key))
     {
         foreach (Text val in values)
         {
             context.Write(key, val);
         }
     }
     else
     {
         if (Checksum.Equals(key))
         {
             Unsigned16 tmp = new Unsigned16();
             Unsigned16 sum = new Unsigned16();
             foreach (Text val in values)
             {
                 tmp.Set(val.ToString());
                 sum.Add(tmp);
             }
             context.Write(Checksum, new Text(sum.ToString()));
         }
         else
         {
             Text value = values.GetEnumerator().Next();
             if (firstKey)
             {
                 firstKey = false;
             }
             else
             {
                 if (value.CompareTo(lastValue) < 0)
                 {
                     context.Write(Error, new Text("bad key partitioning:\n  file " + lastKey + " key "
                                                   + TextifyBytes(lastValue) + "\n  file " + key + " key " + TextifyBytes(value)));
                 }
             }
             lastKey.Set(key);
             lastValue.Set(value);
         }
     }
 }
Exemplo n.º 3
0
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 protected override void Map(LongWritable row, NullWritable ignored, Mapper.Context
                             context)
 {
     if (rand == null)
     {
         rowId           = new Unsigned16(row.Get());
         rand            = Random16.SkipAhead(rowId);
         checksumCounter = context.GetCounter(TeraGen.Counters.Checksum);
     }
     Random16.NextRand(rand);
     GenSort.GenerateRecord(buffer, rand, rowId);
     key.Set(buffer, 0, TeraInputFormat.KeyLength);
     value.Set(buffer, TeraInputFormat.KeyLength, TeraInputFormat.ValueLength);
     context.Write(key, value);
     crc32.Reset();
     crc32.Update(buffer, 0, TeraInputFormat.KeyLength + TeraInputFormat.ValueLength);
     checksum.Set(crc32.GetValue());
     total.Add(checksum);
     rowId.Add(One);
 }
Exemplo n.º 4
0
        /// <summary>
        /// Generate a "binary" record suitable for all sort benchmarks *except*
        /// PennySort.
        /// </summary>
        internal static void GenerateRecord(byte[] recBuf, Unsigned16 rand, Unsigned16 recordNumber
                                            )
        {
            /* generate the 10-byte key using the high 10 bytes of the 128-bit
             * random number
             */
            for (int i = 0; i < 10; ++i)
            {
                recBuf[i] = rand.GetByte(i);
            }
            /* add 2 bytes of "break" */
            recBuf[10] = unchecked ((int)(0x00));
            recBuf[11] = unchecked ((int)(0x11));

            /* convert the 128-bit record number to 32 bits of ascii hexadecimal
             * as the next 32 bytes of the record.
             */
            for (int i_1 = 0; i_1 < 32; i_1++)
            {
                recBuf[12 + i_1] = unchecked ((byte)recordNumber.GetHexDigit(i_1));
            }
            /* add 4 bytes of "break" data */
            recBuf[44] = unchecked ((byte)unchecked ((int)(0x88)));
            recBuf[45] = unchecked ((byte)unchecked ((int)(0x99)));
            recBuf[46] = unchecked ((byte)unchecked ((int)(0xAA)));
            recBuf[47] = unchecked ((byte)unchecked ((int)(0xBB)));
            /* add 48 bytes of filler based on low 48 bits of random number */
            for (int i_2 = 0; i_2 < 12; ++i_2)
            {
                recBuf[48 + i_2 * 4] = recBuf[49 + i_2 * 4] = recBuf[50 + i_2 * 4] = recBuf[51 +
                                                                                            i_2 * 4] = unchecked ((byte)rand.GetHexDigit(20 + i_2));
            }
            /* add 4 bytes of "break" data */
            recBuf[96] = unchecked ((byte)unchecked ((int)(0xCC)));
            recBuf[97] = unchecked ((byte)unchecked ((int)(0xDD)));
            recBuf[98] = unchecked ((byte)unchecked ((int)(0xEE)));
            recBuf[99] = unchecked ((byte)unchecked ((int)(0xFF)));
        }
Exemplo n.º 5
0
        /// <summary>
        /// Generate an ascii record suitable for all sort benchmarks including
        /// PennySort.
        /// </summary>
        internal static void GenerateAsciiRecord(byte[] recBuf, Unsigned16 rand, Unsigned16
                                                 recordNumber)
        {
            /* generate the 10-byte ascii key using mostly the high 64 bits.
             */
            long temp = rand.GetHigh8();

            if (temp < 0)
            {
                // use biginteger to avoid the negative sign problem
                BigInteger bigTemp = MakeBigInteger(temp);
                recBuf[0] = unchecked ((byte)((byte)(' ') + (bigTemp.Mod(NinetyFive))));
                temp      = bigTemp.Divide(NinetyFive);
            }
            else
            {
                recBuf[0] = unchecked ((byte)((byte)(' ') + (temp % 95)));
                temp     /= 95;
            }
            for (int i = 1; i < 8; ++i)
            {
                recBuf[i] = unchecked ((byte)((byte)(' ') + (temp % 95)));
                temp     /= 95;
            }
            temp = rand.GetLow8();
            if (temp < 0)
            {
                BigInteger bigTemp = MakeBigInteger(temp);
                recBuf[8] = unchecked ((byte)((byte)(' ') + (bigTemp.Mod(NinetyFive))));
                temp      = bigTemp.Divide(NinetyFive);
            }
            else
            {
                recBuf[8] = unchecked ((byte)((byte)(' ') + (temp % 95)));
                temp     /= 95;
            }
            recBuf[9] = unchecked ((byte)((byte)(' ') + (temp % 95)));
            /* add 2 bytes of "break" */
            recBuf[10] = (byte)(' ');
            recBuf[11] = (byte)(' ');

            /* convert the 128-bit record number to 32 bits of ascii hexadecimal
             * as the next 32 bytes of the record.
             */
            for (int i_1 = 0; i_1 < 32; i_1++)
            {
                recBuf[12 + i_1] = unchecked ((byte)recordNumber.GetHexDigit(i_1));
            }
            /* add 2 bytes of "break" data */
            recBuf[44] = (byte)(' ');
            recBuf[45] = (byte)(' ');
            /* add 52 bytes of filler based on low 48 bits of random number */
            for (int i_2 = 0; i_2 < 13; ++i_2)
            {
                recBuf[46 + i_2 * 4] = recBuf[47 + i_2 * 4] = recBuf[48 + i_2 * 4] = recBuf[49 +
                                                                                            i_2 * 4] = unchecked ((byte)rand.GetHexDigit(19 + i_2));
            }
            /* add 2 bytes of "break" data */
            recBuf[98] = (byte)('\r');
            /* nice for Windows */
            recBuf[99] = (byte)('\n');
        }
Exemplo n.º 6
0
        /// <exception cref="System.Exception"/>
        public static void Main(string[] args)
        {
            Unsigned16   startingRecord = new Unsigned16();
            Unsigned16   numberOfRecords;
            OutputStream @out;
            bool         useAscii = false;
            Unsigned16   checksum = null;
            int          i;

            for (i = 0; i < args.Length; ++i)
            {
                string arg       = args[i];
                int    argLength = arg.Length;
                if (argLength >= 1 && arg[0] == '-')
                {
                    if (argLength < 2)
                    {
                        Usage();
                    }
                    switch (arg[1])
                    {
                    case 'a':
                    {
                        useAscii = true;
                        break;
                    }

                    case 'b':
                    {
                        startingRecord = Unsigned16.FromDecimal(Sharpen.Runtime.Substring(arg, 2));
                        break;
                    }

                    case 'c':
                    {
                        checksum = new Unsigned16();
                        break;
                    }

                    default:
                    {
                        Usage();
                        break;
                    }
                    }
                }
                else
                {
                    break;
                }
            }
            if (args.Length - i != 2)
            {
                Usage();
            }
            numberOfRecords = Unsigned16.FromDecimal(args[i]);
            @out            = new FileOutputStream(args[i + 1]);
            OutputRecords(@out, useAscii, startingRecord, numberOfRecords, checksum);
            @out.Close();
            if (checksum != null)
            {
                System.Console.Out.WriteLine(checksum);
            }
        }