public virtual void TestSequenceFileSync() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); Path p = new Path("/testSequenceFileSync/foo"); int len = 1 << 16; FSDataOutputStream @out = fs.Create(p, FsPermission.GetDefault(), EnumSet.Of(CreateFlag .Create, CreateFlag.Overwrite, CreateFlag.SyncBlock), 4096, (short)1, len, null); SequenceFile.Writer w = SequenceFile.CreateWriter(new Configuration(), SequenceFile.Writer .Stream(@out), SequenceFile.Writer.KeyClass(typeof(RandomDatum)), SequenceFile.Writer .ValueClass(typeof(RandomDatum)), SequenceFile.Writer.Compression(SequenceFile.CompressionType .None, new DefaultCodec())); w.Hflush(); CheckSyncMetric(cluster, 0); w.Hsync(); CheckSyncMetric(cluster, 1); int seed = new Random().Next(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); generator.Next(); w.Append(generator.GetKey(), generator.GetValue()); w.Hsync(); CheckSyncMetric(cluster, 2); w.Close(); CheckSyncMetric(cluster, 2); @out.Close(); CheckSyncMetric(cluster, 3); cluster.Shutdown(); }
public virtual void SetUp() { // Generate data int seed = new Random().Next(); DataOutputBuffer dataBuf = new DataOutputBuffer(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); for (int i = 0; i < count; ++i) { generator.Next(); RandomDatum key = generator.GetKey(); RandomDatum value = generator.GetValue(); key.Write(dataBuf); value.Write(dataBuf); } Log.Info("Generated " + count + " records"); data = dataBuf.GetData(); dataLen = dataBuf.GetLength(); }
/// <exception cref="System.IO.IOException"/> private static void CodecTest(Configuration conf, int seed, int count, string codecClass ) { // Create the codec CompressionCodec codec = null; try { codec = (CompressionCodec)ReflectionUtils.NewInstance(conf.GetClassByName(codecClass ), conf); } catch (TypeLoadException) { throw new IOException("Illegal codec!"); } Log.Info("Created a Codec object of type: " + codecClass); // Generate data DataOutputBuffer data = new DataOutputBuffer(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); for (int i = 0; i < count; ++i) { generator.Next(); RandomDatum key = generator.GetKey(); RandomDatum value = generator.GetValue(); key.Write(data); value.Write(data); } Log.Info("Generated " + count + " records"); // Compress data DataOutputBuffer compressedDataBuffer = new DataOutputBuffer(); CompressionOutputStream deflateFilter = codec.CreateOutputStream(compressedDataBuffer ); DataOutputStream deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter )); deflateOut.Write(data.GetData(), 0, data.GetLength()); deflateOut.Flush(); deflateFilter.Finish(); Log.Info("Finished compressing data"); // De-compress data DataInputBuffer deCompressedDataBuffer = new DataInputBuffer(); deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer .GetLength()); CompressionInputStream inflateFilter = codec.CreateInputStream(deCompressedDataBuffer ); DataInputStream inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter )); // Check DataInputBuffer originalData = new DataInputBuffer(); originalData.Reset(data.GetData(), 0, data.GetLength()); DataInputStream originalIn = new DataInputStream(new BufferedInputStream(originalData )); for (int i_1 = 0; i_1 < count; ++i_1) { RandomDatum k1 = new RandomDatum(); RandomDatum v1 = new RandomDatum(); k1.ReadFields(originalIn); v1.ReadFields(originalIn); RandomDatum k2 = new RandomDatum(); RandomDatum v2 = new RandomDatum(); k2.ReadFields(inflateIn); v2.ReadFields(inflateIn); Assert.True("original and compressed-then-decompressed-output not equal" , k1.Equals(k2) && v1.Equals(v2)); // original and compressed-then-decompressed-output have the same hashCode IDictionary <RandomDatum, string> m = new Dictionary <RandomDatum, string>(); m[k1] = k1.ToString(); m[v1] = v1.ToString(); string result = m[k2]; Assert.Equal("k1 and k2 hashcode not equal", result, k1.ToString ()); result = m[v2]; Assert.Equal("v1 and v2 hashcode not equal", result, v1.ToString ()); } // De-compress data byte-at-a-time originalData.Reset(data.GetData(), 0, data.GetLength()); deCompressedDataBuffer.Reset(compressedDataBuffer.GetData(), 0, compressedDataBuffer .GetLength()); inflateFilter = codec.CreateInputStream(deCompressedDataBuffer); // Check originalIn = new DataInputStream(new BufferedInputStream(originalData)); int expected; do { expected = originalIn.Read(); Assert.Equal("Inflated stream read by byte does not match", expected , inflateFilter.Read()); }while (expected != -1); Log.Info("SUCCESS! Completed checking " + count + " records"); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="GeneralSecurityException"/> private void CryptoCodecTest(Configuration conf, int seed, int count, string encCodecClass , string decCodecClass, byte[] iv) { CryptoCodec encCodec = null; try { encCodec = (CryptoCodec)ReflectionUtils.NewInstance(conf.GetClassByName(encCodecClass ), conf); } catch (TypeLoadException) { throw new IOException("Illegal crypto codec!"); } Log.Info("Created a Codec object of type: " + encCodecClass); // Generate data DataOutputBuffer data = new DataOutputBuffer(); RandomDatum.Generator generator = new RandomDatum.Generator(seed); for (int i = 0; i < count; ++i) { generator.Next(); RandomDatum key = generator.GetKey(); RandomDatum value = generator.GetValue(); key.Write(data); value.Write(data); } Log.Info("Generated " + count + " records"); // Encrypt data DataOutputBuffer encryptedDataBuffer = new DataOutputBuffer(); CryptoOutputStream @out = new CryptoOutputStream(encryptedDataBuffer, encCodec, bufferSize , key, iv); @out.Write(data.GetData(), 0, data.GetLength()); @out.Flush(); @out.Close(); Log.Info("Finished encrypting data"); CryptoCodec decCodec = null; try { decCodec = (CryptoCodec)ReflectionUtils.NewInstance(conf.GetClassByName(decCodecClass ), conf); } catch (TypeLoadException) { throw new IOException("Illegal crypto codec!"); } Log.Info("Created a Codec object of type: " + decCodecClass); // Decrypt data DataInputBuffer decryptedDataBuffer = new DataInputBuffer(); decryptedDataBuffer.Reset(encryptedDataBuffer.GetData(), 0, encryptedDataBuffer.GetLength ()); CryptoInputStream @in = new CryptoInputStream(decryptedDataBuffer, decCodec, bufferSize , key, iv); DataInputStream dataIn = new DataInputStream(new BufferedInputStream(@in)); // Check DataInputBuffer originalData = new DataInputBuffer(); originalData.Reset(data.GetData(), 0, data.GetLength()); DataInputStream originalIn = new DataInputStream(new BufferedInputStream(originalData )); for (int i_1 = 0; i_1 < count; ++i_1) { RandomDatum k1 = new RandomDatum(); RandomDatum v1 = new RandomDatum(); k1.ReadFields(originalIn); v1.ReadFields(originalIn); RandomDatum k2 = new RandomDatum(); RandomDatum v2 = new RandomDatum(); k2.ReadFields(dataIn); v2.ReadFields(dataIn); Assert.True("original and encrypted-then-decrypted-output not equal" , k1.Equals(k2) && v1.Equals(v2)); // original and encrypted-then-decrypted-output have the same hashCode IDictionary <RandomDatum, string> m = new Dictionary <RandomDatum, string>(); m[k1] = k1.ToString(); m[v1] = v1.ToString(); string result = m[k2]; Assert.Equal("k1 and k2 hashcode not equal", result, k1.ToString ()); result = m[v2]; Assert.Equal("v1 and v2 hashcode not equal", result, v1.ToString ()); } // Decrypt data byte-at-a-time originalData.Reset(data.GetData(), 0, data.GetLength()); decryptedDataBuffer.Reset(encryptedDataBuffer.GetData(), 0, encryptedDataBuffer.GetLength ()); @in = new CryptoInputStream(decryptedDataBuffer, decCodec, bufferSize, key, iv); // Check originalIn = new DataInputStream(new BufferedInputStream(originalData)); int expected; do { expected = originalIn.Read(); Assert.Equal("Decrypted stream read by byte does not match", expected , @in.Read()); }while (expected != -1); // Seek to a certain position and decrypt originalData.Reset(data.GetData(), 0, data.GetLength()); decryptedDataBuffer.Reset(encryptedDataBuffer.GetData(), 0, encryptedDataBuffer.GetLength ()); @in = new CryptoInputStream(new TestCryptoStreams.FakeInputStream(decryptedDataBuffer ), decCodec, bufferSize, key, iv); int seekPos = data.GetLength() / 3; @in.Seek(seekPos); // Check TestCryptoStreams.FakeInputStream originalInput = new TestCryptoStreams.FakeInputStream (originalData); originalInput.Seek(seekPos); do { expected = originalInput.Read(); Assert.Equal("Decrypted stream read by byte does not match", expected , @in.Read()); }while (expected != -1); Log.Info("SUCCESS! Completed checking " + count + " records"); // Check secure random generator TestSecureRandom(encCodec); }