Пример #1
0
        private void FillKey(BytesWritable o)
        {
            int len = keyLenRNG.NextInt();

            if (len < MinKeyLen)
            {
                len = MinKeyLen;
            }
            o.SetSize(len);
            int n = MinKeyLen;

            while (n < len)
            {
                byte[] word = dict[random.Next(dict.Length)];
                int    l    = Math.Min(word.Length, len - n);
                System.Array.Copy(word, 0, o.Get(), n, l);
                n += l;
            }
            if (sorted && WritableComparator.CompareBytes(lastKey.Get(), MinKeyLen, lastKey.GetSize
                                                              () - MinKeyLen, o.Get(), MinKeyLen, o.GetSize() - MinKeyLen) > 0)
            {
                IncrementPrefix();
            }
            System.Array.Copy(prefix, 0, o.Get(), 0, MinKeyLen);
            lastKey.Set(o);
        }
Пример #2
0
        public virtual void Next(BytesWritable key)
        {
            key.SetSize(Math.Max(MinKeyLen, keyLenRNG.NextInt()));
            random.NextBytes(key.Get());
            int n = random.Next(max - min) + min;

            byte[] b = key.Get();
            b[0] = unchecked ((byte)(n >> 24));
            b[1] = unchecked ((byte)(n >> 16));
            b[2] = unchecked ((byte)(n >> 8));
            b[3] = unchecked ((byte)n);
        }
Пример #3
0
        private void FillValue(BytesWritable o)
        {
            int len = valLenRNG.NextInt();

            o.SetSize(len);
            int n = 0;

            while (n < len)
            {
                byte[] word = dict[random.Next(dict.Length)];
                int    l    = Math.Min(word.Length, len - n);
                System.Array.Copy(word, 0, o.Get(), n, l);
                n += l;
            }
        }
Пример #4
0
        /// <exception cref="System.IO.IOException"/>
        private static void CreateBigMapInputFile(Configuration conf, FileSystem fs, Path
                                                  dir, long fileSizeInMB)
        {
            // Check if the input path exists and is non-empty
            if (fs.Exists(dir))
            {
                FileStatus[] list = fs.ListStatus(dir);
                if (list.Length > 0)
                {
                    throw new IOException("Input path: " + dir + " already exists... ");
                }
            }
            Path file = new Path(dir, "part-0");

            SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(BytesWritable
                                                                                          ), typeof(BytesWritable), SequenceFile.CompressionType.None);
            long          numBytesToWrite = fileSizeInMB * 1024 * 1024;
            int           minKeySize      = conf.GetInt(MinKey, 10);
            int           keySizeRange    = conf.GetInt(MaxKey, 1000) - minKeySize;
            int           minValueSize    = conf.GetInt(MinValue, 0);
            int           valueSizeRange  = conf.GetInt(MaxValue, 20000) - minValueSize;
            BytesWritable randomKey       = new BytesWritable();
            BytesWritable randomValue     = new BytesWritable();

            Log.Info("Writing " + numBytesToWrite + " bytes to " + file + " with " + "minKeySize: "
                     + minKeySize + " keySizeRange: " + keySizeRange + " minValueSize: " + minValueSize
                     + " valueSizeRange: " + valueSizeRange);
            long start = Runtime.CurrentTimeMillis();

            while (numBytesToWrite > 0)
            {
                int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0);
                randomKey.SetSize(keyLength);
                RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength());
                int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange
                                                                                    ) : 0);
                randomValue.SetSize(valueLength);
                RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength());
                writer.Append(randomKey, randomValue);
                numBytesToWrite -= keyLength + valueLength;
            }
            writer.Close();
            long end = Runtime.CurrentTimeMillis();

            Log.Info("Created " + file + " of size: " + fileSizeInMB + "MB in " + (end - start
                                                                                   ) / 1000 + "secs");
        }