/// <summary>Create an input stream with a codec taken from the global CodecPool.</summary>
/// <param name="codec">The codec to use to create the input stream.</param>
/// <param name="conf">The configuration to use if we need to create a new codec.</param>
/// <param name="in">The input stream to wrap.</param>
/// <returns>The new input stream</returns>
/// <exception cref="System.IO.IOException"/>
internal static CompressionInputStream CreateInputStreamWithCodecPool(CompressionCodec
codec, Configuration conf, InputStream @in)
{
Decompressor decompressor = CodecPool.GetDecompressor(codec);
CompressionInputStream stream = null;
try
{
stream = codec.CreateInputStream(@in, decompressor);
}
finally
{
if (stream == null)
{
CodecPool.ReturnDecompressor(decompressor);
}
else
{
stream.SetTrackedDecompressor(decompressor);
}
}
return(stream);
}
/// <exception cref="System.IO.IOException"/>
private static void CodecTest(Configuration conf, int seed, int count, string codecClass
)
{
// Create the codec
CompressionCodec codec = null;
try
{
codec = (CompressionCodec)ReflectionUtils.NewInstance(conf.GetClassByName(codecClass
), conf);
}
catch (TypeLoadException)
{
throw new IOException("Illegal codec!");
}
Log.Info("Created a Codec object of type: " + codecClass);
// Generate data
DataOutputBuffer data = new DataOutputBuffer();
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
for (int i = 0; i < count; ++i)
{
generator.Next();
RandomDatum key = generator.GetKey();
RandomDatum value = generator.GetValue();
key.Write(data);
value.Write(data);
}
Log.Info("Generated " + count + " records");
// Compress data
DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
CompressionOutputStream deflateFilter = codec.CreateOutputStream(compressedDataBuffer
);
DataOutputStream deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter
));
deflateOut.Write(data.GetData(), 0, data.GetLength());
deflateOut.Flush();
deflateFilter.Finish();
Log.Info("Finished compressing data");
// De-compress data
DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer
.GetLength());
CompressionInputStream inflateFilter = codec.CreateInputStream(deCompressedDataBuffer
);
DataInputStream inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter
));
// Check
DataInputBuffer originalData = new DataInputBuffer();
originalData.Reset(data.GetData(), 0, data.GetLength());
DataInputStream originalIn = new DataInputStream(new BufferedInputStream(originalData
));
for (int i_1 = 0; i_1 < count; ++i_1)
{
RandomDatum k1 = new RandomDatum();
RandomDatum v1 = new RandomDatum();
k1.ReadFields(originalIn);
v1.ReadFields(originalIn);
RandomDatum k2 = new RandomDatum();
RandomDatum v2 = new RandomDatum();
k2.ReadFields(inflateIn);
v2.ReadFields(inflateIn);
Assert.True("original and compressed-then-decompressed-output not equal"
, k1.Equals(k2) && v1.Equals(v2));
// original and compressed-then-decompressed-output have the same hashCode
IDictionary <RandomDatum, string> m = new Dictionary <RandomDatum, string>();
m[k1] = k1.ToString();
m[v1] = v1.ToString();
string result = m[k2];
Assert.Equal("k1 and k2 hashcode not equal", result, k1.ToString
());
result = m[v2];
Assert.Equal("v1 and v2 hashcode not equal", result, v1.ToString
());
}
// De-compress data byte-at-a-time
originalData.Reset(data.GetData(), 0, data.GetLength());
deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer
.GetLength());
inflateFilter = codec.CreateInputStream(deCompressedDataBuffer);
// Check
originalIn = new DataInputStream(new BufferedInputStream(originalData));
int expected;
do
{
expected = originalIn.Read();
Assert.Equal("Inflated stream read by byte does not match", expected
, inflateFilter.Read());
}while (expected != -1);
Log.Info("SUCCESS! Completed checking " + count + " records");
}
/// <summary>A little test program.</summary>
/// <param name="args"/>
/// <exception cref="System.Exception"/>
public static void Main(string[] args)
{
Configuration conf = new Configuration();
Org.Apache.Hadoop.IO.Compress.CompressionCodecFactory factory = new Org.Apache.Hadoop.IO.Compress.CompressionCodecFactory
(conf);
bool encode = false;
for (int i = 0; i < args.Length; ++i)
{
if ("-in".Equals(args[i]))
{
encode = true;
}
else
{
if ("-out".Equals(args[i]))
{
encode = false;
}
else
{
CompressionCodec codec = factory.GetCodec(new Path(args[i]));
if (codec == null)
{
System.Console.Out.WriteLine("Codec for " + args[i] + " not found.");
}
else
{
if (encode)
{
CompressionOutputStream @out = null;
InputStream @in = null;
try
{
@out = codec.CreateOutputStream(new FileOutputStream(args[i]));
byte[] buffer = new byte[100];
string inFilename = RemoveSuffix(args[i], codec.GetDefaultExtension());
@in = new FileInputStream(inFilename);
int len = @in.Read(buffer);
while (len > 0)
{
@out.Write(buffer, 0, len);
len = @in.Read(buffer);
}
}
finally
{
if (@out != null)
{
@out.Close();
}
if (@in != null)
{
@in.Close();
}
}
}
else
{
CompressionInputStream @in = null;
try
{
@in = codec.CreateInputStream(new FileInputStream(args[i]));
byte[] buffer = new byte[100];
int len = @in.Read(buffer);
while (len > 0)
{
System.Console.Out.Write(buffer, 0, len);
len = @in.Read(buffer);
}
}
finally
{
if (@in != null)
{
@in.Close();
}
}
}
}
}
}
}
}
