/// <exception cref="System.IO.IOException"/> private void AnalyzeResult(FileSystem fs, TestDFSIO.TestType testType, long execTime ) { string dir = Runtime.GetProperty("test.build.dir", "target/test-dir"); AnalyzeResult(fs, testType, execTime, dir + "/" + DefaultResFileName); }
/// <exception cref="System.IO.IOException"/> private void SequentialTest(FileSystem fs, TestDFSIO.TestType testType, long fileSize , int nrFiles) { // in bytes TestDFSIO.IOStatMapper ioer = null; switch (testType) { case TestDFSIO.TestType.TestTypeRead: { ioer = new TestDFSIO.ReadMapper(); break; } case TestDFSIO.TestType.TestTypeWrite: { ioer = new TestDFSIO.WriteMapper(); break; } case TestDFSIO.TestType.TestTypeAppend: { ioer = new TestDFSIO.AppendMapper(); break; } case TestDFSIO.TestType.TestTypeReadRandom: case TestDFSIO.TestType.TestTypeReadBackward: case TestDFSIO.TestType.TestTypeReadSkip: { ioer = new TestDFSIO.RandomReadMapper(); break; } case TestDFSIO.TestType.TestTypeTruncate: { ioer = new TestDFSIO.TruncateMapper(); break; } default: { return; } } for (int i = 0; i < nrFiles; i++) { ioer.DoIO(Reporter.Null, BaseFileName + Sharpen.Extensions.ToString(i), fileSize); } }
private Path GetReduceFilePath(TestDFSIO.TestType testType) { switch (testType) { case TestDFSIO.TestType.TestTypeWrite: { return(new Path(GetWriteDir(config), "part-00000")); } case TestDFSIO.TestType.TestTypeAppend: { return(new Path(GetAppendDir(config), "part-00000")); } case TestDFSIO.TestType.TestTypeRead: { return(new Path(GetReadDir(config), "part-00000")); } case TestDFSIO.TestType.TestTypeReadRandom: case TestDFSIO.TestType.TestTypeReadBackward: case TestDFSIO.TestType.TestTypeReadSkip: { return(new Path(GetRandomReadDir(config), "part-00000")); } case TestDFSIO.TestType.TestTypeTruncate: { return(new Path(GetTruncateDir(config), "part-00000")); } default: { break; } } return(null); }
/// <exception cref="System.IO.IOException"/> public virtual int Run(string[] args) { // Tool TestDFSIO.TestType testType = null; int bufferSize = DefaultBufferSize; long nrBytes = 1 * Mega; int nrFiles = 1; long skipSize = 0; string resFileName = DefaultResFileName; string compressionClass = null; bool isSequential = false; string version = typeof(TestDFSIO).Name + ".1.8"; Log.Info(version); if (args.Length == 0) { System.Console.Error.WriteLine("Missing arguments."); return(-1); } for (int i = 0; i < args.Length; i++) { // parse command line if (args[i].StartsWith("-read")) { testType = TestDFSIO.TestType.TestTypeRead; } else { if (args[i].Equals("-write")) { testType = TestDFSIO.TestType.TestTypeWrite; } else { if (args[i].Equals("-append")) { testType = TestDFSIO.TestType.TestTypeAppend; } else { if (args[i].Equals("-random")) { if (testType != TestDFSIO.TestType.TestTypeRead) { return(-1); } testType = TestDFSIO.TestType.TestTypeReadRandom; } else { if (args[i].Equals("-backward")) { if (testType != TestDFSIO.TestType.TestTypeRead) { return(-1); } testType = TestDFSIO.TestType.TestTypeReadBackward; } else { if (args[i].Equals("-skip")) { if (testType != TestDFSIO.TestType.TestTypeRead) { return(-1); } testType = TestDFSIO.TestType.TestTypeReadSkip; } else { if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-truncate")) { testType = TestDFSIO.TestType.TestTypeTruncate; } else { if (args[i].Equals("-clean")) { testType = TestDFSIO.TestType.TestTypeCleanup; } else { if (args[i].StartsWith("-seq")) { isSequential = true; } else { if (args[i].StartsWith("-compression")) { compressionClass = args[++i]; } else { if (args[i].Equals("-nrFiles")) { nrFiles = System.Convert.ToInt32(args[++i]); } else { if (args[i].Equals("-fileSize") || args[i].Equals("-size")) { nrBytes = ParseSize(args[++i]); } else { if (args[i].Equals("-skipSize")) { skipSize = ParseSize(args[++i]); } else { if (args[i].Equals("-bufferSize")) { bufferSize = System.Convert.ToInt32(args[++i]); } else { if (args[i].Equals("-resFile")) { resFileName = args[++i]; } else { System.Console.Error.WriteLine("Illegal argument: " + args[i]); return(-1); } } } } } } } } } } } } } } } } if (testType == null) { return(-1); } if (testType == TestDFSIO.TestType.TestTypeReadBackward) { skipSize = -bufferSize; } else { if (testType == TestDFSIO.TestType.TestTypeReadSkip && skipSize == 0) { skipSize = bufferSize; } } Log.Info("nrFiles = " + nrFiles); Log.Info("nrBytes (MB) = " + ToMB(nrBytes)); Log.Info("bufferSize = " + bufferSize); if (skipSize > 0) { Log.Info("skipSize = " + skipSize); } Log.Info("baseDir = " + GetBaseDir(config)); if (compressionClass != null) { config.Set("test.io.compression.class", compressionClass); Log.Info("compressionClass = " + compressionClass); } config.SetInt("test.io.file.buffer.size", bufferSize); config.SetLong("test.io.skip.size", skipSize); config.SetBoolean(DFSConfigKeys.DfsSupportAppendKey, true); FileSystem fs = FileSystem.Get(config); if (isSequential) { long tStart = Runtime.CurrentTimeMillis(); SequentialTest(fs, testType, nrBytes, nrFiles); long execTime = Runtime.CurrentTimeMillis() - tStart; string resultLine = "Seq Test exec time sec: " + (float)execTime / 1000; Log.Info(resultLine); return(0); } if (testType == TestDFSIO.TestType.TestTypeCleanup) { Cleanup(fs); return(0); } CreateControlFile(fs, nrBytes, nrFiles); long tStart_1 = Runtime.CurrentTimeMillis(); switch (testType) { case TestDFSIO.TestType.TestTypeWrite: { WriteTest(fs); break; } case TestDFSIO.TestType.TestTypeRead: { ReadTest(fs); break; } case TestDFSIO.TestType.TestTypeAppend: { AppendTest(fs); break; } case TestDFSIO.TestType.TestTypeReadRandom: case TestDFSIO.TestType.TestTypeReadBackward: case TestDFSIO.TestType.TestTypeReadSkip: { RandomReadTest(fs); break; } case TestDFSIO.TestType.TestTypeTruncate: { TruncateTest(fs); break; } default: { break; } } long execTime_1 = Runtime.CurrentTimeMillis() - tStart_1; AnalyzeResult(fs, testType, execTime_1, resFileName); return(0); }
/// <exception cref="System.IO.IOException"/> private void AnalyzeResult(FileSystem fs, TestDFSIO.TestType testType, long execTime , string resFileName) { Path reduceFile = GetReduceFilePath(testType); long tasks = 0; long size = 0; long time = 0; float rate = 0; float sqrate = 0; DataInputStream @in = null; BufferedReader lines = null; try { @in = new DataInputStream(fs.Open(reduceFile)); lines = new BufferedReader(new InputStreamReader(@in)); string line; while ((line = lines.ReadLine()) != null) { StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%"); string attr = tokens.NextToken(); if (attr.EndsWith(":tasks")) { tasks = long.Parse(tokens.NextToken()); } else { if (attr.EndsWith(":size")) { size = long.Parse(tokens.NextToken()); } else { if (attr.EndsWith(":time")) { time = long.Parse(tokens.NextToken()); } else { if (attr.EndsWith(":rate")) { rate = float.ParseFloat(tokens.NextToken()); } else { if (attr.EndsWith(":sqrate")) { sqrate = float.ParseFloat(tokens.NextToken()); } } } } } } } finally { if (@in != null) { @in.Close(); } if (lines != null) { lines.Close(); } } double med = rate / 1000 / tasks; double stdDev = Math.Sqrt(Math.Abs(sqrate / 1000 / tasks - med * med)); string[] resultLines = new string[] { "----- TestDFSIO ----- : " + testType, " Date & time: " + Sharpen.Extensions.CreateDate(Runtime.CurrentTimeMillis()), " Number of files: " + tasks, "Total MBytes processed: " + ToMB(size), " Throughput mb/sec: " + size * 1000.0 / (time * Mega), "Average IO rate mb/sec: " + med, " IO rate std deviation: " + stdDev, " Test exec time sec: " + (float)execTime / 1000, string.Empty }; TextWriter res = null; try { res = new TextWriter(new FileOutputStream(new FilePath(resFileName), true)); for (int i = 0; i < resultLines.Length; i++) { Log.Info(resultLines[i]); res.WriteLine(resultLines[i]); } } finally { if (res != null) { res.Close(); } } }