/// <exception cref="System.IO.IOException"/>
private SequenceFile.Metadata ReadMetadata(FileSystem fs, Path file)
{
Log.Info("reading file: " + file.ToString());
SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
SequenceFile.Metadata meta = reader.GetMetadata();
reader.Close();
return(meta);
}
/// <exception cref="System.IO.IOException"/>
private void SortMetadataTest(FileSystem fs, Path unsortedFile, Path sortedFile,
SequenceFile.Metadata metadata)
{
fs.Delete(sortedFile, true);
Log.Info("sorting: " + unsortedFile + " to: " + sortedFile);
WritableComparator comparator = WritableComparator.Get(typeof(RandomDatum));
SequenceFile.Sorter sorter = new SequenceFile.Sorter(fs, comparator, typeof(RandomDatum
), typeof(RandomDatum), conf, metadata);
sorter.Sort(new Path[] { unsortedFile }, sortedFile, false);
}
/// <exception cref="System.IO.IOException"/>
private void WriteMetadataTest(FileSystem fs, int count, int seed, Path file, SequenceFile.CompressionType
compressionType, CompressionCodec codec, SequenceFile.Metadata metadata)
{
fs.Delete(file, true);
Log.Info("creating " + count + " records with metadata and with " + compressionType
+ " compression");
SequenceFile.Writer writer = SequenceFile.CreateWriter(fs, conf, file, typeof(RandomDatum
), typeof(RandomDatum), compressionType, codec, null, metadata);
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
for (int i = 0; i < count; i++)
{
generator.Next();
RandomDatum key = generator.GetKey();
RandomDatum value = generator.GetValue();
writer.Append(key, value);
}
writer.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestAppend()
{
Path file = new Path(RootPath, "testseqappend.seq");
fs.Delete(file, true);
Text key1 = new Text("Key1");
Text value1 = new Text("Value1");
Text value2 = new Text("Updated");
SequenceFile.Metadata metadata = new SequenceFile.Metadata();
metadata.Set(key1, value1);
SequenceFile.Writer.Option metadataOption = SequenceFile.Writer.Metadata(metadata
);
SequenceFile.Writer writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.
File(file), SequenceFile.Writer.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass
(typeof(string)), metadataOption);
writer.Append(1L, "one");
writer.Append(2L, "two");
writer.Close();
Verify2Values(file);
metadata.Set(key1, value2);
writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer
.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer
.AppendIfExists(true), metadataOption);
// Verify the Meta data is not changed
Assert.Equal(value1, writer.metadata.Get(key1));
writer.Append(3L, "three");
writer.Append(4L, "four");
writer.Close();
VerifyAll4Values(file);
// Verify the Meta data readable after append
SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.File
(file));
Assert.Equal(value1, reader.GetMetadata().Get(key1));
reader.Close();
// Verify failure if the compression details are different
try
{
SequenceFile.Writer.Option wrongCompressOption = SequenceFile.Writer.Compression(
SequenceFile.CompressionType.Record, new GzipCodec());
writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer
.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer
.AppendIfExists(true), wrongCompressOption);
writer.Close();
NUnit.Framework.Assert.Fail("Expected IllegalArgumentException for compression options"
);
}
catch (ArgumentException)
{
}
// Expected exception. Ignore it
try
{
SequenceFile.Writer.Option wrongCompressOption = SequenceFile.Writer.Compression(
SequenceFile.CompressionType.Block, new DefaultCodec());
writer = SequenceFile.CreateWriter(conf, SequenceFile.Writer.File(file), SequenceFile.Writer
.KeyClass(typeof(long)), SequenceFile.Writer.ValueClass(typeof(string)), SequenceFile.Writer
.AppendIfExists(true), wrongCompressOption);
writer.Close();
NUnit.Framework.Assert.Fail("Expected IllegalArgumentException for compression options"
);
}
catch (ArgumentException)
{
}
// Expected exception. Ignore it
fs.DeleteOnExit(file);
}
/// <summary>Unit tests for SequenceFile metadata.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestSequenceFileMetadata()
{
Log.Info("Testing SequenceFile with metadata");
int count = 1024 * 10;
CompressionCodec codec = new DefaultCodec();
Path file = new Path(Runtime.GetProperty("test.build.data", ".") + "/test.seq.metadata"
);
Path sortedFile = new Path(Runtime.GetProperty("test.build.data", ".") + "/test.sorted.seq.metadata"
);
Path recordCompressedFile = new Path(Runtime.GetProperty("test.build.data", ".")
+ "/test.rc.seq.metadata");
Path blockCompressedFile = new Path(Runtime.GetProperty("test.build.data", ".") +
"/test.bc.seq.metadata");
FileSystem fs = FileSystem.GetLocal(conf);
SequenceFile.Metadata theMetadata = new SequenceFile.Metadata();
theMetadata.Set(new Text("name_1"), new Text("value_1"));
theMetadata.Set(new Text("name_2"), new Text("value_2"));
theMetadata.Set(new Text("name_3"), new Text("value_3"));
theMetadata.Set(new Text("name_4"), new Text("value_4"));
int seed = new Random().Next();
try
{
// SequenceFile.Writer
WriteMetadataTest(fs, count, seed, file, SequenceFile.CompressionType.None, null,
theMetadata);
SequenceFile.Metadata aMetadata = ReadMetadata(fs, file);
if (!theMetadata.Equals(aMetadata))
{
Log.Info("The original metadata:\n" + theMetadata.ToString());
Log.Info("The retrieved metadata:\n" + aMetadata.ToString());
throw new RuntimeException("metadata not match: " + 1);
}
// SequenceFile.RecordCompressWriter
WriteMetadataTest(fs, count, seed, recordCompressedFile, SequenceFile.CompressionType
.Record, codec, theMetadata);
aMetadata = ReadMetadata(fs, recordCompressedFile);
if (!theMetadata.Equals(aMetadata))
{
Log.Info("The original metadata:\n" + theMetadata.ToString());
Log.Info("The retrieved metadata:\n" + aMetadata.ToString());
throw new RuntimeException("metadata not match: " + 2);
}
// SequenceFile.BlockCompressWriter
WriteMetadataTest(fs, count, seed, blockCompressedFile, SequenceFile.CompressionType
.Block, codec, theMetadata);
aMetadata = ReadMetadata(fs, blockCompressedFile);
if (!theMetadata.Equals(aMetadata))
{
Log.Info("The original metadata:\n" + theMetadata.ToString());
Log.Info("The retrieved metadata:\n" + aMetadata.ToString());
throw new RuntimeException("metadata not match: " + 3);
}
// SequenceFile.Sorter
SortMetadataTest(fs, file, sortedFile, theMetadata);
aMetadata = ReadMetadata(fs, recordCompressedFile);
if (!theMetadata.Equals(aMetadata))
{
Log.Info("The original metadata:\n" + theMetadata.ToString());
Log.Info("The retrieved metadata:\n" + aMetadata.ToString());
throw new RuntimeException("metadata not match: " + 4);
}
}
finally
{
fs.Close();
}
Log.Info("Successfully tested SequenceFile with metadata");
}
