private void FillKey(BytesWritable o)
{
int len = keyLenRNG.NextInt();
if (len < MinKeyLen)
{
len = MinKeyLen;
}
o.SetSize(len);
int n = MinKeyLen;
while (n < len)
{
byte[] word = dict[random.Next(dict.Length)];
int l = Math.Min(word.Length, len - n);
System.Array.Copy(word, 0, o.Get(), n, l);
n += l;
}
if (sorted && WritableComparator.CompareBytes(lastKey.Get(), MinKeyLen, lastKey.GetSize
() - MinKeyLen, o.Get(), MinKeyLen, o.GetSize() - MinKeyLen) > 0)
{
IncrementPrefix();
}
System.Array.Copy(prefix, 0, o.Get(), 0, MinKeyLen);
lastKey.Set(o);
}
public virtual void Next(BytesWritable key)
{
key.SetSize(Math.Max(MinKeyLen, keyLenRNG.NextInt()));
random.NextBytes(key.Get());
int n = random.Next(max - min) + min;
byte[] b = key.Get();
b[0] = unchecked ((byte)(n >> 24));
b[1] = unchecked ((byte)(n >> 16));
b[2] = unchecked ((byte)(n >> 8));
b[3] = unchecked ((byte)n);
}
/// <exception cref="System.IO.IOException"/>
private void CreateTFile()
{
long totalBytes = 0;
FSDataOutputStream fout = CreateFSOutput(path, fs);
try
{
TFile.Writer writer = new TFile.Writer(fout, options.minBlockSize, options.compress
, "memcmp", conf);
try
{
BytesWritable key = new BytesWritable();
BytesWritable val = new BytesWritable();
timer.Start();
for (long i = 0; true; ++i)
{
if (i % 1000 == 0)
{
// test the size for every 1000 rows.
if (fs.GetFileStatus(path).GetLen() >= options.fileSize)
{
break;
}
}
kvGen.Next(key, val, false);
writer.Append(key.Get(), 0, key.GetSize(), val.Get(), 0, val.GetSize());
totalBytes += key.GetSize();
totalBytes += val.GetSize();
}
timer.Stop();
}
finally
{
writer.Close();
}
}
finally
{
fout.Close();
}
double duration = (double)timer.Read() / 1000;
// in us.
long fsize = fs.GetFileStatus(path).GetLen();
System.Console.Out.Printf("time: %s...uncompressed: %.2fMB...raw thrpt: %.2fMB/s\n"
, timer.ToString(), (double)totalBytes / 1024 / 1024, totalBytes / duration);
System.Console.Out.Printf("time: %s...file size: %.2fMB...disk thrpt: %.2fMB/s\n"
, timer.ToString(), (double)fsize / 1024 / 1024, fsize / duration);
}
private void FillValue(BytesWritable o)
{
int len = valLenRNG.NextInt();
o.SetSize(len);
int n = 0;
while (n < len)
{
byte[] word = dict[random.Next(dict.Length)];
int l = Math.Min(word.Length, len - n);
System.Array.Copy(word, 0, o.Get(), n, l);
n += l;
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void SeekTFile()
{
int miss = 0;
long totalBytes = 0;
FSDataInputStream fsdis = fs.Open(path);
TFile.Reader reader = new TFile.Reader(fsdis, fs.GetFileStatus(path).GetLen(), conf
);
KeySampler kSampler = new KeySampler(rng, reader.GetFirstKey(), reader.GetLastKey
(), keyLenGen);
TFile.Reader.Scanner scanner = reader.CreateScanner();
BytesWritable key = new BytesWritable();
BytesWritable val = new BytesWritable();
timer.Reset();
timer.Start();
for (int i = 0; i < options.seekCount; ++i)
{
kSampler.Next(key);
scanner.LowerBound(key.Get(), 0, key.GetSize());
if (!scanner.AtEnd())
{
scanner.Entry().Get(key, val);
totalBytes += key.GetSize();
totalBytes += val.GetSize();
}
else
{
++miss;
}
}
timer.Stop();
double duration = (double)timer.Read() / 1000;
// in us.
System.Console.Out.Printf("time: %s...avg seek: %s...%d hit...%d miss...avg I/O size: %.2fKB\n"
, timer.ToString(), NanoTimer.NanoTimeToString(timer.Read() / options.seekCount)
, options.seekCount - miss, miss, (double)totalBytes / 1024 / (options.seekCount
- miss));
}
public virtual byte[] GetValue()
{
return(value.Get());
}
public virtual byte[] GetKey()
{
return(key.Get());
}
/// <exception cref="System.IO.IOException"/>
public virtual void Append(BytesWritable key, BytesWritable value)
{
writer.Append(key.Get(), 0, key.GetSize(), value.Get(), 0, value.GetSize());
}