private static RandomDatum[] Generate(int count)
{
Log.Info("generating " + count + " records in memory");
RandomDatum[] data = new RandomDatum[count];
RandomDatum.Generator generator = new RandomDatum.Generator();
for (int i = 0; i < count; i++)
{
generator.Next();
data[i] = generator.GetValue();
}
Log.Info("sorting " + count + " records");
Arrays.Sort(data);
return(data);
}
private static RandomDatum[] Generate(int count)
{
if (Log.IsDebugEnabled())
{
Log.Debug("generating " + count + " records in debug");
}
RandomDatum[] data = new RandomDatum[count];
RandomDatum.Generator generator = new RandomDatum.Generator();
for (int i = 0; i < count; i++)
{
generator.Next();
data[i] = generator.GetValue();
}
return(data);
}
/// <exception cref="System.IO.IOException"/>
private void WriteMetadataTest(FileSystem fs, int count, int seed, Path file, SequenceFile.CompressionType
compressionType, CompressionCodec codec, SequenceFile.Metadata metadata)
{
fs.Delete(file, true);
Log.Info("creating " + count + " records with metadata and with " + compressionType
+ " compression");
SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(RandomDatum
), typeof(RandomDatum), compressionType, codec, null, metadata);
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
for (int i = 0; i < count; i++)
{
generator.Next();
RandomDatum key = generator.GetKey();
RandomDatum value = generator.GetValue();
writer.Append(key, value);
}
writer.Close();
}
/// <exception cref="System.IO.IOException"/>
private static void ReadTest(FileSystem fs, RandomDatum[] data, string file)
{
RandomDatum v = new RandomDatum();
int sample = (int)Math.Sqrt(data.Length);
Random random = new Random();
Log.Info("reading " + sample + " records");
SetFile.Reader reader = new SetFile.Reader(fs, file, conf);
for (int i = 0; i < sample; i++)
{
if (!reader.Seek(data[random.Next(data.Length)]))
{
throw new RuntimeException("wrong value at " + i);
}
}
reader.Close();
Log.Info("done reading " + data.Length);
}
/// <exception cref="System.IO.IOException"/>
private void CheckSort(FileSystem fs, int count, int seed, Path file)
{
Log.Info("sorting " + count + " records in memory for debug");
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
SortedDictionary <RandomDatum, RandomDatum> map = new SortedDictionary <RandomDatum
, RandomDatum>();
for (int i = 0; i < count; i++)
{
generator.Next();
RandomDatum key = generator.GetKey();
RandomDatum value = generator.GetValue();
map[key] = value;
}
Log.Debug("checking order of " + count + " records");
RandomDatum k = new RandomDatum();
RandomDatum v = new RandomDatum();
IEnumerator <KeyValuePair <RandomDatum, RandomDatum> > iterator = map.GetEnumerator(
);
SequenceFile.Reader reader = new SequenceFile.Reader(fs, file.Suffix(".sorted"),
conf);
for (int i_1 = 0; i_1 < count; i_1++)
{
KeyValuePair <RandomDatum, RandomDatum> entry = iterator.Next();
RandomDatum key = entry.Key;
RandomDatum value = entry.Value;
reader.Next(k, v);
if (!k.Equals(key))
{
throw new RuntimeException("wrong key at " + i_1);
}
if (!v.Equals(value))
{
throw new RuntimeException("wrong value at " + i_1);
}
}
reader.Close();
Log.Debug("sucessfully checked " + count + " records");
}
/// <exception cref="System.IO.IOException"/>
private static void ReadTest(FileSystem fs, RandomDatum[] data, string file, Configuration
conf)
{
RandomDatum v = new RandomDatum();
if (Log.IsDebugEnabled())
{
Log.Debug("reading " + data.Length + " debug");
}
ArrayFile.Reader reader = new ArrayFile.Reader(fs, file, conf);
try
{
for (int i = 0; i < data.Length; i++)
{
// try forwards
reader.Get(i, v);
if (!v.Equals(data[i]))
{
throw new RuntimeException("wrong value at " + i);
}
}
for (int i_1 = data.Length - 1; i_1 >= 0; i_1--)
{
// then backwards
reader.Get(i_1, v);
if (!v.Equals(data[i_1]))
{
throw new RuntimeException("wrong value at " + i_1);
}
}
if (Log.IsDebugEnabled())
{
Log.Debug("done reading " + data.Length + " debug");
}
}
finally
{
reader.Close();
}
}
/// <exception cref="System.IO.IOException"/>
private void MergeTest(FileSystem fs, int count, int seed, Path file, SequenceFile.CompressionType
compressionType, bool fast, int factor, int megabytes)
{
Log.Debug("creating " + factor + " files with " + count / factor + " records");
SequenceFile.Writer[] writers = new SequenceFile.Writer[factor];
Path[] names = new Path[factor];
Path[] sortedNames = new Path[factor];
for (int i = 0; i < factor; i++)
{
names[i] = file.Suffix("." + i);
sortedNames[i] = names[i].Suffix(".sorted");
fs.Delete(names[i], true);
fs.Delete(sortedNames[i], true);
writers[i] = SequenceFile.CreateWriter(fs, conf, names[i], typeof(RandomDatum), typeof(
RandomDatum), compressionType);
}
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
for (int i_1 = 0; i_1 < count; i_1++)
{
generator.Next();
RandomDatum key = generator.GetKey();
RandomDatum value = generator.GetValue();
writers[i_1 % factor].Append(key, value);
}
for (int i_2 = 0; i_2 < factor; i_2++)
{
writers[i_2].Close();
}
for (int i_3 = 0; i_3 < factor; i_3++)
{
Log.Debug("sorting file " + i_3 + " with " + count / factor + " records");
NewSorter(fs, fast, megabytes, factor).Sort(names[i_3], sortedNames[i_3]);
}
Log.Info("merging " + factor + " files with " + count / factor + " debug");
fs.Delete(new Path(file + ".sorted"), true);
NewSorter(fs, fast, megabytes, factor).Merge(sortedNames, file.Suffix(".sorted"));
}
/// <exception cref="System.IO.IOException"/>
private void ReadTest(FileSystem fs, int count, int seed, Path file)
{
Log.Debug("reading " + count + " records");
SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
RandomDatum k = new RandomDatum();
RandomDatum v = new RandomDatum();
DataOutputBuffer rawKey = new DataOutputBuffer();
SequenceFile.ValueBytes rawValue = reader.CreateValueBytes();
for (int i = 0; i < count; i++)
{
generator.Next();
RandomDatum key = generator.GetKey();
RandomDatum value = generator.GetValue();
try
{
if ((i % 5) == 0)
{
// Testing 'raw' apis
rawKey.Reset();
reader.NextRaw(rawKey, rawValue);
}
else
{
// Testing 'non-raw' apis
if ((i % 2) == 0)
{
reader.Next(k);
reader.GetCurrentValue(v);
}
else
{
reader.Next(k, v);
}
// Check
if (!k.Equals(key))
{
throw new RuntimeException("wrong key at " + i);
}
if (!v.Equals(value))
{
throw new RuntimeException("wrong value at " + i);
}
}
}
catch (IOException ioe)
{
Log.Info("Problem on row " + i);
Log.Info("Expected key = " + key);
Log.Info("Expected len = " + key.GetLength());
Log.Info("Actual key = " + k);
Log.Info("Actual len = " + k.GetLength());
Log.Info("Expected value = " + value);
Log.Info("Expected len = " + value.GetLength());
Log.Info("Actual value = " + v);
Log.Info("Actual len = " + v.GetLength());
Log.Info("Key equals: " + k.Equals(key));
Log.Info("value equals: " + v.Equals(value));
throw;
}
}
reader.Close();
}
public virtual void Next()
{
key = new RandomDatum(random);
value = new RandomDatum(random);
}
