private void FillKey(BytesWritable o)
{
int len = keyLenRNG.NextInt();
if (len < MinKeyLen)
{
len = MinKeyLen;
}
o.SetSize(len);
int n = MinKeyLen;
while (n < len)
{
byte[] word = dict[random.Next(dict.Length)];
int l = Math.Min(word.Length, len - n);
System.Array.Copy(word, 0, o.Get(), n, l);
n += l;
}
if (sorted && WritableComparator.CompareBytes(lastKey.Get(), MinKeyLen, lastKey.GetSize
() - MinKeyLen, o.Get(), MinKeyLen, o.GetSize() - MinKeyLen) > 0)
{
IncrementPrefix();
}
System.Array.Copy(prefix, 0, o.Get(), 0, MinKeyLen);
lastKey.Set(o);
}
public virtual void Next(BytesWritable key)
{
key.SetSize(Math.Max(MinKeyLen, keyLenRNG.NextInt()));
random.NextBytes(key.Get());
int n = random.Next(max - min) + min;
byte[] b = key.Get();
b[0] = unchecked ((byte)(n >> 24));
b[1] = unchecked ((byte)(n >> 16));
b[2] = unchecked ((byte)(n >> 8));
b[3] = unchecked ((byte)n);
}
private void FillValue(BytesWritable o)
{
int len = valLenRNG.NextInt();
o.SetSize(len);
int n = 0;
while (n < len)
{
byte[] word = dict[random.Next(dict.Length)];
int l = Math.Min(word.Length, len - n);
System.Array.Copy(word, 0, o.Get(), n, l);
n += l;
}
}
/// <exception cref="System.IO.IOException"/>
private static void CreateBigMapInputFile(Configuration conf, FileSystem fs, Path
dir, long fileSizeInMB)
{
// Check if the input path exists and is non-empty
if (fs.Exists(dir))
{
FileStatus[] list = fs.ListStatus(dir);
if (list.Length > 0)
{
throw new IOException("Input path: " + dir + " already exists... ");
}
}
Path file = new Path(dir, "part-0");
SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(BytesWritable
), typeof(BytesWritable), SequenceFile.CompressionType.None);
long numBytesToWrite = fileSizeInMB * 1024 * 1024;
int minKeySize = conf.GetInt(MinKey, 10);
int keySizeRange = conf.GetInt(MaxKey, 1000) - minKeySize;
int minValueSize = conf.GetInt(MinValue, 0);
int valueSizeRange = conf.GetInt(MaxValue, 20000) - minValueSize;
BytesWritable randomKey = new BytesWritable();
BytesWritable randomValue = new BytesWritable();
Log.Info("Writing " + numBytesToWrite + " bytes to " + file + " with " + "minKeySize: "
+ minKeySize + " keySizeRange: " + keySizeRange + " minValueSize: " + minValueSize
+ " valueSizeRange: " + valueSizeRange);
long start = Runtime.CurrentTimeMillis();
while (numBytesToWrite > 0)
{
int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0);
randomKey.SetSize(keyLength);
RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength());
int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange
) : 0);
randomValue.SetSize(valueLength);
RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength());
writer.Append(randomKey, randomValue);
numBytesToWrite -= keyLength + valueLength;
}
writer.Close();
long end = Runtime.CurrentTimeMillis();
Log.Info("Created " + file + " of size: " + fileSizeInMB + "MB in " + (end - start
) / 1000 + "secs");
}