public virtual void TestGzipCompatibility() { Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); Log.Info("seed: " + seed); DataOutputBuffer dflbuf = new DataOutputBuffer(); GZIPOutputStream gzout = new GZIPOutputStream(dflbuf); byte[] b = new byte[r.Next(128 * 1024 + 1)]; r.NextBytes(b); gzout.Write(b); gzout.Close(); DataInputBuffer gzbuf = new DataInputBuffer(); gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength()); Configuration conf = new Configuration(); conf.SetBoolean(CommonConfigurationKeys.IoNativeLibAvailableKey, false); CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf); Decompressor decom = codec.CreateDecompressor(); NUnit.Framework.Assert.IsNotNull(decom); Assert.Equal(typeof(BuiltInGzipDecompressor), decom.GetType()); InputStream gzin = codec.CreateInputStream(gzbuf, decom); dflbuf.Reset(); IOUtils.CopyBytes(gzin, dflbuf, 4096); byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength()); Assert.AssertArrayEquals(b, dflchk); }
/// <exception cref="System.IO.IOException"/> protected internal override void ProcessArguments(List <PathData> items) { base.ProcessArguments(items); if (exitCode != 0) { // check for error collecting paths return; } FSDataOutputStream @out = dst.fs.Create(dst.path); try { foreach (PathData src in srcs) { FSDataInputStream @in = src.fs.Open(src.path); try { IOUtils.CopyBytes(@in, @out, GetConf(), false); if (delimiter != null) { @out.Write(Runtime.GetBytesForString(delimiter, "UTF-8")); } } finally { @in.Close(); } } } finally { @out.Close(); } }
/// <exception cref="System.IO.IOException"/> private long DumpFromOffset(PathData item, long offset) { long fileSize = item.RefreshStatus().GetLen(); if (offset > fileSize) { return(fileSize); } // treat a negative offset as relative to end of the file, floor of 0 if (offset < 0) { offset = Math.Max(fileSize + offset, 0); } FSDataInputStream @in = item.fs.Open(item.path); try { @in.Seek(offset); // use conf so the system configured io block size is used IOUtils.CopyBytes(@in, System.Console.Out, GetConf(), false); offset = @in.GetPos(); } finally { @in.Close(); } return(offset); }
public virtual void TestMRWebAppSSLDisabled() { MRApp app = new _MRApp_175(2, 2, true, this.GetType().FullName, true); Configuration conf = new Configuration(); // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY conf.Set(YarnConfiguration.YarnHttpPolicyKey, HttpConfig.Policy.HttpsOnly.ToString ()); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); string hostPort = NetUtils.GetHostPortString(((MRClientService)app.GetClientService ()).GetWebApp().GetListenerAddress()); // http:// should be accessible Uri httpUrl = new Uri("http://" + hostPort); HttpURLConnection conn = (HttpURLConnection)httpUrl.OpenConnection(); InputStream @in = conn.GetInputStream(); ByteArrayOutputStream @out = new ByteArrayOutputStream(); IOUtils.CopyBytes(@in, @out, 1024); NUnit.Framework.Assert.IsTrue(@out.ToString().Contains("MapReduce Application")); // https:// is not accessible. Uri httpsUrl = new Uri("https://" + hostPort); try { HttpURLConnection httpsConn = (HttpURLConnection)httpsUrl.OpenConnection(); httpsConn.GetInputStream(); NUnit.Framework.Assert.Fail("https:// is not accessible, expected to fail"); } catch (Exception e) { NUnit.Framework.Assert.IsTrue(e is SSLException); } app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); }
/// <summary>Executes the filesystem operation.</summary> /// <param name="fs">filesystem instance to use.</param> /// <returns>void.</returns> /// <exception cref="System.IO.IOException">thrown if an IO error occured.</exception> public virtual Void Execute(FileSystem fs) { int bufferSize = fs.GetConf().GetInt("httpfs.buffer.size", 4096); OutputStream os = fs.Append(path, bufferSize); IOUtils.CopyBytes(@is, os, bufferSize, true); os.Close(); return(null); }
public virtual void TestCopyBytesWithCountShouldNotCloseStreamsWhenCloseIsFalse() { InputStream inputStream = Org.Mockito.Mockito.Mock <InputStream>(); OutputStream outputStream = Org.Mockito.Mockito.Mock <OutputStream>(); Org.Mockito.Mockito.DoReturn(-1).When(inputStream).Read(new byte[4096], 0, 1); IOUtils.CopyBytes(inputStream, outputStream, (long)1, false); Org.Mockito.Mockito.Verify(inputStream, Org.Mockito.Mockito.AtMost(0)).Close(); Org.Mockito.Mockito.Verify(outputStream, Org.Mockito.Mockito.AtMost(0)).Close(); }
/// <exception cref="System.Exception"/> private static string ReadOut(Uri url) { HttpsURLConnection conn = (HttpsURLConnection)url.OpenConnection(); conn.SetSSLSocketFactory(clientSslFactory.CreateSSLSocketFactory()); InputStream @in = conn.GetInputStream(); ByteArrayOutputStream @out = new ByteArrayOutputStream(); IOUtils.CopyBytes(@in, @out, 1024); return(@out.ToString()); }
/// <exception cref="System.IO.IOException"/> private void PrintToStdout(InputStream @in) { try { IOUtils.CopyBytes(@in, @out, GetConf(), false); } finally { @in.Close(); } }
/// <exception cref="System.IO.IOException"/> public virtual void Write(OutputStream os) { IOUtils.SkipFully(@is, offset); if (len == -1) { IOUtils.CopyBytes(@is, os, 4096, true); } else { IOUtils.CopyBytes(@is, os, len, true); } }
public virtual void TestCopyBytesShouldCloseStreamsWhenCloseIsTrue() { InputStream inputStream = Org.Mockito.Mockito.Mock <InputStream>(); OutputStream outputStream = Org.Mockito.Mockito.Mock <OutputStream>(); Org.Mockito.Mockito.DoReturn(-1).When(inputStream).Read(new byte[1]); IOUtils.CopyBytes(inputStream, outputStream, 1, true); Org.Mockito.Mockito.Verify(inputStream, Org.Mockito.Mockito.AtLeastOnce()).Close( ); Org.Mockito.Mockito.Verify(outputStream, Org.Mockito.Mockito.AtLeastOnce()).Close (); }
public virtual void Log4jFile() { InputStream @is = Org.Apache.Hadoop.Lib.Server.Server.GetResource("default-log4j.properties" ); OutputStream os = new FileOutputStream(new FilePath(TestDirHelper.GetTestDir(), "server-log4j.properties" )); IOUtils.CopyBytes(@is, os, 1024, true); Configuration conf = new Configuration(false); Org.Apache.Hadoop.Lib.Server.Server server = CreateServer(conf); server.Init(); }
/// <exception cref="System.IO.IOException"/> public virtual void TestFileSink() { outFile = GetTestTempFile("test-file-sink-", ".out"); string outPath = outFile.GetAbsolutePath(); // NB: specify large period to avoid multiple metrics snapshotting: new ConfigBuilder().Add("*.period", 10000).Add("test.sink.mysink0.class", typeof( FileSink).FullName).Add("test.sink.mysink0.filename", outPath).Add("test.sink.mysink0.context" , "test1").Save(TestMetricsConfig.GetTestFilename("hadoop-metrics2-test")); // NB: we filter by context to exclude "metricssystem" context metrics: MetricsSystemImpl ms = new MetricsSystemImpl("test"); ms.Start(); TestFileSink.MyMetrics1 mm1 = new TestFileSink.MyMetrics1().RegisterWith(ms); new TestFileSink.MyMetrics2().RegisterWith(ms); mm1.testMetric1.Incr(); mm1.testMetric2.Incr(2); ms.PublishMetricsNow(); // publish the metrics ms.Stop(); ms.Shutdown(); InputStream @is = null; ByteArrayOutputStream baos = null; string outFileContent = null; try { @is = new FileInputStream(outFile); baos = new ByteArrayOutputStream((int)outFile.Length()); IOUtils.CopyBytes(@is, baos, 1024, true); outFileContent = Runtime.GetStringForBytes(baos.ToByteArray(), "UTF-8"); } finally { IOUtils.Cleanup(null, baos, @is); } // Check the out file content. Should be something like the following: //1360244820087 test1.testRecord1: Context=test1, testTag1=testTagValue1, testTag2=testTagValue2, Hostname=myhost, testMetric1=1, testMetric2=2 //1360244820089 test1.testRecord2: Context=test1, testTag22=testTagValue22, Hostname=myhost // Note that in the below expression we allow tags and metrics to go in arbitrary order. Pattern expectedContentPattern = Pattern.Compile("^\\d+\\s+test1.testRecord1:\\s+Context=test1,\\s+" + "(testTag1=testTagValue1,\\s+testTag2=testTagValue2|testTag2=testTagValue2,\\s+testTag1=testTagValue1)," + "\\s+Hostname=.*,\\s+(testMetric1=1,\\s+testMetric2=2|testMetric2=2,\\s+testMetric1=1)" + "$[\\n\\r]*^\\d+\\s+test1.testRecord2:\\s+Context=test1," + "\\s+testTag22=testTagValue22,\\s+Hostname=.*$[\\n\\r]*" , Pattern.Multiline); // line #1: // line #2: Assert.True(expectedContentPattern.Matcher(outFileContent).Matches ()); }
// Helper filter filesystem that registers created files as temp files to // be deleted on exit unless successfully renamed /// <exception cref="System.IO.IOException"/> internal virtual void WriteStreamToFile(InputStream @in, PathData target, bool lazyPersist ) { FSDataOutputStream @out = null; try { @out = Create(target, lazyPersist); IOUtils.CopyBytes(@in, @out, GetConf(), true); } finally { IOUtils.CloseStream(@out); } }
/// <summary>Read dataFile and compute its MD5 checksum.</summary> /// <exception cref="System.IO.IOException"/> public static MD5Hash ComputeMd5ForFile(FilePath dataFile) { InputStream @in = new FileInputStream(dataFile); try { MessageDigest digester = MD5Hash.GetDigester(); DigestInputStream dis = new DigestInputStream(@in, digester); IOUtils.CopyBytes(dis, new IOUtils.NullOutputStream(), 128 * 1024); return(new MD5Hash(digester.Digest())); } finally { IOUtils.CloseStream(@in); } }
/// <summary>Executes the filesystem operation.</summary> /// <param name="fs">filesystem instance to use.</param> /// <returns>The URI of the created file.</returns> /// <exception cref="System.IO.IOException">thrown if an IO error occured.</exception> public virtual Void Execute(FileSystem fs) { if (replication == -1) { replication = fs.GetDefaultReplication(path); } if (blockSize == -1) { blockSize = fs.GetDefaultBlockSize(path); } FsPermission fsPermission = new FsPermission(permission); int bufferSize = fs.GetConf().GetInt("httpfs.buffer.size", 4096); OutputStream os = fs.Create(path, fsPermission, @override, bufferSize, replication , blockSize, null); IOUtils.CopyBytes(@is, os, bufferSize, true); os.Close(); return(null); }
/// <exception cref="System.IO.IOException"/> protected internal override void ProcessArguments(List <PathData> args) { if (!dst.exists) { dst.fs.Create(dst.path, false).Close(); } InputStream @is = null; FSDataOutputStream fos = dst.fs.Append(dst.path); try { if (readStdin) { if (args.Count == 0) { IOUtils.CopyBytes(Runtime.@in, fos, DefaultIoLength); } else { throw new IOException("stdin (-) must be the sole input argument when present"); } } // Read in each input file and write to the target. foreach (PathData source in args) { @is = new FileInputStream(source.ToFile()); IOUtils.CopyBytes(@is, fos, DefaultIoLength); IOUtils.CloseStream(@is); @is = null; } } finally { if (@is != null) { IOUtils.CloseStream(@is); } if (fos != null) { IOUtils.CloseStream(fos); } } }
public virtual void TestCopyBytesShouldCloseInputSteamWhenOutputStreamCloseThrowsException () { InputStream inputStream = Org.Mockito.Mockito.Mock <InputStream>(); OutputStream outputStream = Org.Mockito.Mockito.Mock <OutputStream>(); Org.Mockito.Mockito.DoReturn(-1).When(inputStream).Read(new byte[1]); Org.Mockito.Mockito.DoThrow(new IOException()).When(outputStream).Close(); try { IOUtils.CopyBytes(inputStream, outputStream, 1, true); } catch (IOException) { } Org.Mockito.Mockito.Verify(inputStream, Org.Mockito.Mockito.AtLeastOnce()).Close( ); Org.Mockito.Mockito.Verify(outputStream, Org.Mockito.Mockito.AtLeastOnce()).Close (); }
/// <summary>Copy specified file into a temporary file.</summary> /// <remarks> /// Copy specified file into a temporary file. Then rename the /// temporary file to the original name. This will cause any /// hardlinks to the original file to be removed. The temporary /// files are created in the same directory. The temporary files will /// be recovered (especially on Windows) on datanode restart. /// </remarks> /// <exception cref="System.IO.IOException"/> private void UnlinkFile(FilePath file, Block b) { FilePath tmpFile = DatanodeUtil.CreateTmpFile(b, DatanodeUtil.GetUnlinkTmpFile(file )); try { FileInputStream @in = new FileInputStream(file); try { FileOutputStream @out = new FileOutputStream(tmpFile); try { IOUtils.CopyBytes(@in, @out, 16 * 1024); } finally { @out.Close(); } } finally { @in.Close(); } if (file.Length() != tmpFile.Length()) { throw new IOException("Copy of file " + file + " size " + file.Length() + " into file " + tmpFile + " resulted in a size of " + tmpFile.Length()); } FileUtil.ReplaceFile(tmpFile, file); } catch (IOException e) { bool done = tmpFile.Delete(); if (!done) { DataNode.Log.Info("detachFile failed to delete temporary file " + tmpFile); } throw; } }
/// <summary>Unpack matching files from a jar.</summary> /// <remarks> /// Unpack matching files from a jar. Entries inside the jar that do /// not match the given pattern will be skipped. /// </remarks> /// <param name="jarFile">the .jar file to unpack</param> /// <param name="toDir">the destination directory into which to unpack the jar</param> /// <param name="unpackRegex">the pattern to match jar entries against</param> /// <exception cref="System.IO.IOException"/> public static void UnJar(FilePath jarFile, FilePath toDir, Pattern unpackRegex ) { JarFile jar = new JarFile(jarFile); try { Enumeration <JarEntry> entries = ((Enumeration <JarEntry>)jar.Entries()); while (entries.MoveNext()) { JarEntry entry = entries.Current; if (!entry.IsDirectory() && unpackRegex.Matcher(entry.GetName()).Matches()) { InputStream @in = jar.GetInputStream(entry); try { FilePath file = new FilePath(toDir, entry.GetName()); EnsureDirectory(file.GetParentFile()); OutputStream @out = new FileOutputStream(file); try { IOUtils.CopyBytes(@in, @out, 8192); } finally { @out.Close(); } } finally { @in.Close(); } } } } finally { jar.Close(); } }
/// <exception cref="System.IO.IOException"/> private static void CreateUnlinkTmpFile(ReplicaInfo replicaInfo, bool changeBlockFile , bool isRename) { FilePath src; if (changeBlockFile) { src = replicaInfo.GetBlockFile(); } else { src = replicaInfo.GetMetaFile(); } FilePath dst = DatanodeUtil.GetUnlinkTmpFile(src); if (isRename) { src.RenameTo(dst); } else { FileInputStream @in = new FileInputStream(src); try { FileOutputStream @out = new FileOutputStream(dst); try { IOUtils.CopyBytes(@in, @out, 1); } finally { @out.Close(); } } finally { @in.Close(); } } }
/// <exception cref="System.Exception"/> private void DoTestEntirelyCorruptFile(int numDataNodes) { long fileSize = 4096; Path file = new Path("/testFile"); short replFactor = (short)numDataNodes; Configuration conf = new Configuration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, numDataNodes); // Set short retry timeouts so this test runs faster conf.SetInt(DFSConfigKeys.DfsClientRetryWindowBase, 10); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes ).Build(); try { cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, file, fileSize, replFactor, 12345L); /*seed*/ DFSTestUtil.WaitReplication(fs, file, replFactor); ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, file); int blockFilesCorrupted = cluster.CorruptBlockOnDataNodes(block); NUnit.Framework.Assert.AreEqual("All replicas not corrupted", replFactor, blockFilesCorrupted ); try { IOUtils.CopyBytes(fs.Open(file), new IOUtils.NullOutputStream(), conf, true); NUnit.Framework.Assert.Fail("Didn't get exception"); } catch (IOException ioe) { DFSClient.Log.Info("Got expected exception", ioe); } } finally { cluster.Shutdown(); } }
/// <exception cref="System.Exception"/> public virtual void TestManyClosedSocketsInCache() { // Make a small file Configuration clientConf = new Configuration(conf); clientConf.Set(DFSConfigKeys.DfsClientContext, "testManyClosedSocketsInCache"); DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI() , clientConf); PeerCache peerCache = ClientContext.GetFromConf(clientConf).GetPeerCache(); DFSTestUtil.CreateFile(fs, TestFile, 1L, (short)1, 0L); // Insert a bunch of dead sockets in the cache, by opening // many streams concurrently, reading all of the data, // and then closing them. InputStream[] stms = new InputStream[5]; try { for (int i = 0; i < stms.Length; i++) { stms[i] = fs.Open(TestFile); } foreach (InputStream stm in stms) { IOUtils.CopyBytes(stm, new IOUtils.NullOutputStream(), 1024); } } finally { IOUtils.Cleanup(null, stms); } NUnit.Framework.Assert.AreEqual(5, peerCache.Size()); // Let all the xceivers timeout Sharpen.Thread.Sleep(1500); AssertXceiverCount(0); // Client side still has the sockets cached NUnit.Framework.Assert.AreEqual(5, peerCache.Size()); // Reading should not throw an exception. DFSTestUtil.ReadFile(fs, TestFile); }
/// <exception cref="System.IO.IOException"/> internal virtual void GzipConcatTest(Configuration conf, Type decomClass) { Random r = new Random(); long seed = r.NextLong(); r.SetSeed(seed); Log.Info(decomClass + " seed: " + seed); int Concat = r.Next(4) + 3; int Buflen = 128 * 1024; DataOutputBuffer dflbuf = new DataOutputBuffer(); DataOutputBuffer chkbuf = new DataOutputBuffer(); byte[] b = new byte[Buflen]; for (int i = 0; i < Concat; ++i) { GZIPOutputStream gzout = new GZIPOutputStream(dflbuf); r.NextBytes(b); int len = r.Next(Buflen); int off = r.Next(Buflen - len); chkbuf.Write(b, off, len); gzout.Write(b, off, len); gzout.Close(); } byte[] chk = Arrays.CopyOf(chkbuf.GetData(), chkbuf.GetLength()); CompressionCodec codec = ReflectionUtils.NewInstance <GzipCodec>(conf); Decompressor decom = codec.CreateDecompressor(); NUnit.Framework.Assert.IsNotNull(decom); Assert.Equal(decomClass, decom.GetType()); DataInputBuffer gzbuf = new DataInputBuffer(); gzbuf.Reset(dflbuf.GetData(), dflbuf.GetLength()); InputStream gzin = codec.CreateInputStream(gzbuf, decom); dflbuf.Reset(); IOUtils.CopyBytes(gzin, dflbuf, 4096); byte[] dflchk = Arrays.CopyOf(dflbuf.GetData(), dflbuf.GetLength()); Assert.AssertArrayEquals(chk, dflchk); }
public virtual void TestCopyBytesWithCountShouldThrowOutTheStreamClosureExceptions () { InputStream inputStream = Org.Mockito.Mockito.Mock <InputStream>(); OutputStream outputStream = Org.Mockito.Mockito.Mock <OutputStream>(); Org.Mockito.Mockito.DoReturn(-1).When(inputStream).Read(new byte[4096], 0, 1); Org.Mockito.Mockito.DoThrow(new IOException("Exception in closing the stream")).When (outputStream).Close(); try { IOUtils.CopyBytes(inputStream, outputStream, (long)1, true); NUnit.Framework.Assert.Fail("Should throw out the exception"); } catch (IOException e) { Assert.Equal("Not throwing the expected exception.", "Exception in closing the stream" , e.Message); } Org.Mockito.Mockito.Verify(inputStream, Org.Mockito.Mockito.AtLeastOnce()).Close( ); Org.Mockito.Mockito.Verify(outputStream, Org.Mockito.Mockito.AtLeastOnce()).Close (); }
/// <summary>Download link and have it be the response.</summary> /// <param name="req">the http request</param> /// <param name="resp">the http response</param> /// <param name="link">the link to download</param> /// <param name="c">the cookie to set if any</param> /// <exception cref="System.IO.IOException">on any error.</exception> private static void ProxyLink(HttpServletRequest req, HttpServletResponse resp, URI link, Cookie c, string proxyHost) { DefaultHttpClient client = new DefaultHttpClient(); client.GetParams().SetParameter(ClientPNames.CookiePolicy, CookiePolicy.BrowserCompatibility ).SetBooleanParameter(ClientPNames.AllowCircularRedirects, true); // Make sure we send the request from the proxy address in the config // since that is what the AM filter checks against. IP aliasing or // similar could cause issues otherwise. IPAddress localAddress = Sharpen.Extensions.GetAddressByName(proxyHost); if (Log.IsDebugEnabled()) { Log.Debug("local InetAddress for proxy host: {}", localAddress); } client.GetParams().SetParameter(ConnRoutePNames.LocalAddress, localAddress); HttpGet httpGet = new HttpGet(link); Enumeration <string> names = req.GetHeaderNames(); while (names.MoveNext()) { string name = names.Current; if (passThroughHeaders.Contains(name)) { string value = req.GetHeader(name); if (Log.IsDebugEnabled()) { Log.Debug("REQ HEADER: {} : {}", name, value); } httpGet.SetHeader(name, value); } } string user = req.GetRemoteUser(); if (user != null && !user.IsEmpty()) { httpGet.SetHeader("Cookie", ProxyUserCookieName + "=" + URLEncoder.Encode(user, "ASCII" )); } OutputStream @out = resp.GetOutputStream(); try { HttpResponse httpResp = client.Execute(httpGet); resp.SetStatus(httpResp.GetStatusLine().GetStatusCode()); foreach (Header header in httpResp.GetAllHeaders()) { resp.SetHeader(header.GetName(), header.GetValue()); } if (c != null) { resp.AddCookie(c); } InputStream @in = httpResp.GetEntity().GetContent(); if (@in != null) { IOUtils.CopyBytes(@in, @out, 4096, true); } } finally { httpGet.ReleaseConnection(); } }
/* Copy count bytes at the given offset from one stream to another */ /// <exception cref="System.IO.IOException"/> internal static void CopyFromOffset(FSDataInputStream @in, OutputStream @out, long offset, long count) { @in.Seek(offset); IOUtils.CopyBytes(@in, @out, count, false); }