/// <exception cref="System.IO.IOException"/>
private void AnalyzeResult(FileSystem fs, TestDFSIO.TestType testType, long execTime
)
{
string dir = Runtime.GetProperty("test.build.dir", "target/test-dir");
AnalyzeResult(fs, testType, execTime, dir + "/" + DefaultResFileName);
}
/// <exception cref="System.IO.IOException"/>
private void SequentialTest(FileSystem fs, TestDFSIO.TestType testType, long fileSize
, int nrFiles)
{
// in bytes
TestDFSIO.IOStatMapper ioer = null;
switch (testType)
{
case TestDFSIO.TestType.TestTypeRead:
{
ioer = new TestDFSIO.ReadMapper();
break;
}
case TestDFSIO.TestType.TestTypeWrite:
{
ioer = new TestDFSIO.WriteMapper();
break;
}
case TestDFSIO.TestType.TestTypeAppend:
{
ioer = new TestDFSIO.AppendMapper();
break;
}
case TestDFSIO.TestType.TestTypeReadRandom:
case TestDFSIO.TestType.TestTypeReadBackward:
case TestDFSIO.TestType.TestTypeReadSkip:
{
ioer = new TestDFSIO.RandomReadMapper();
break;
}
case TestDFSIO.TestType.TestTypeTruncate:
{
ioer = new TestDFSIO.TruncateMapper();
break;
}
default:
{
return;
}
}
for (int i = 0; i < nrFiles; i++)
{
ioer.DoIO(Reporter.Null, BaseFileName + Sharpen.Extensions.ToString(i), fileSize);
}
}
private Path GetReduceFilePath(TestDFSIO.TestType testType)
{
switch (testType)
{
case TestDFSIO.TestType.TestTypeWrite:
{
return(new Path(GetWriteDir(config), "part-00000"));
}
case TestDFSIO.TestType.TestTypeAppend:
{
return(new Path(GetAppendDir(config), "part-00000"));
}
case TestDFSIO.TestType.TestTypeRead:
{
return(new Path(GetReadDir(config), "part-00000"));
}
case TestDFSIO.TestType.TestTypeReadRandom:
case TestDFSIO.TestType.TestTypeReadBackward:
case TestDFSIO.TestType.TestTypeReadSkip:
{
return(new Path(GetRandomReadDir(config), "part-00000"));
}
case TestDFSIO.TestType.TestTypeTruncate:
{
return(new Path(GetTruncateDir(config), "part-00000"));
}
default:
{
break;
}
}
return(null);
}
/// <exception cref="System.IO.IOException"/>
public virtual int Run(string[] args)
{
// Tool
TestDFSIO.TestType testType = null;
int bufferSize = DefaultBufferSize;
long nrBytes = 1 * Mega;
int nrFiles = 1;
long skipSize = 0;
string resFileName = DefaultResFileName;
string compressionClass = null;
bool isSequential = false;
string version = typeof(TestDFSIO).Name + ".1.8";
Log.Info(version);
if (args.Length == 0)
{
System.Console.Error.WriteLine("Missing arguments.");
return(-1);
}
for (int i = 0; i < args.Length; i++)
{
// parse command line
if (args[i].StartsWith("-read"))
{
testType = TestDFSIO.TestType.TestTypeRead;
}
else
{
if (args[i].Equals("-write"))
{
testType = TestDFSIO.TestType.TestTypeWrite;
}
else
{
if (args[i].Equals("-append"))
{
testType = TestDFSIO.TestType.TestTypeAppend;
}
else
{
if (args[i].Equals("-random"))
{
if (testType != TestDFSIO.TestType.TestTypeRead)
{
return(-1);
}
testType = TestDFSIO.TestType.TestTypeReadRandom;
}
else
{
if (args[i].Equals("-backward"))
{
if (testType != TestDFSIO.TestType.TestTypeRead)
{
return(-1);
}
testType = TestDFSIO.TestType.TestTypeReadBackward;
}
else
{
if (args[i].Equals("-skip"))
{
if (testType != TestDFSIO.TestType.TestTypeRead)
{
return(-1);
}
testType = TestDFSIO.TestType.TestTypeReadSkip;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-truncate"))
{
testType = TestDFSIO.TestType.TestTypeTruncate;
}
else
{
if (args[i].Equals("-clean"))
{
testType = TestDFSIO.TestType.TestTypeCleanup;
}
else
{
if (args[i].StartsWith("-seq"))
{
isSequential = true;
}
else
{
if (args[i].StartsWith("-compression"))
{
compressionClass = args[++i];
}
else
{
if (args[i].Equals("-nrFiles"))
{
nrFiles = System.Convert.ToInt32(args[++i]);
}
else
{
if (args[i].Equals("-fileSize") || args[i].Equals("-size"))
{
nrBytes = ParseSize(args[++i]);
}
else
{
if (args[i].Equals("-skipSize"))
{
skipSize = ParseSize(args[++i]);
}
else
{
if (args[i].Equals("-bufferSize"))
{
bufferSize = System.Convert.ToInt32(args[++i]);
}
else
{
if (args[i].Equals("-resFile"))
{
resFileName = args[++i];
}
else
{
System.Console.Error.WriteLine("Illegal argument: " + args[i]);
return(-1);
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
}
if (testType == null)
{
return(-1);
}
if (testType == TestDFSIO.TestType.TestTypeReadBackward)
{
skipSize = -bufferSize;
}
else
{
if (testType == TestDFSIO.TestType.TestTypeReadSkip && skipSize == 0)
{
skipSize = bufferSize;
}
}
Log.Info("nrFiles = " + nrFiles);
Log.Info("nrBytes (MB) = " + ToMB(nrBytes));
Log.Info("bufferSize = " + bufferSize);
if (skipSize > 0)
{
Log.Info("skipSize = " + skipSize);
}
Log.Info("baseDir = " + GetBaseDir(config));
if (compressionClass != null)
{
config.Set("test.io.compression.class", compressionClass);
Log.Info("compressionClass = " + compressionClass);
}
config.SetInt("test.io.file.buffer.size", bufferSize);
config.SetLong("test.io.skip.size", skipSize);
config.SetBoolean(DFSConfigKeys.DfsSupportAppendKey, true);
FileSystem fs = FileSystem.Get(config);
if (isSequential)
{
long tStart = Runtime.CurrentTimeMillis();
SequentialTest(fs, testType, nrBytes, nrFiles);
long execTime = Runtime.CurrentTimeMillis() - tStart;
string resultLine = "Seq Test exec time sec: " + (float)execTime / 1000;
Log.Info(resultLine);
return(0);
}
if (testType == TestDFSIO.TestType.TestTypeCleanup)
{
Cleanup(fs);
return(0);
}
CreateControlFile(fs, nrBytes, nrFiles);
long tStart_1 = Runtime.CurrentTimeMillis();
switch (testType)
{
case TestDFSIO.TestType.TestTypeWrite:
{
WriteTest(fs);
break;
}
case TestDFSIO.TestType.TestTypeRead:
{
ReadTest(fs);
break;
}
case TestDFSIO.TestType.TestTypeAppend:
{
AppendTest(fs);
break;
}
case TestDFSIO.TestType.TestTypeReadRandom:
case TestDFSIO.TestType.TestTypeReadBackward:
case TestDFSIO.TestType.TestTypeReadSkip:
{
RandomReadTest(fs);
break;
}
case TestDFSIO.TestType.TestTypeTruncate:
{
TruncateTest(fs);
break;
}
default:
{
break;
}
}
long execTime_1 = Runtime.CurrentTimeMillis() - tStart_1;
AnalyzeResult(fs, testType, execTime_1, resFileName);
return(0);
}
/// <exception cref="System.IO.IOException"/>
private void AnalyzeResult(FileSystem fs, TestDFSIO.TestType testType, long execTime
, string resFileName)
{
Path reduceFile = GetReduceFilePath(testType);
long tasks = 0;
long size = 0;
long time = 0;
float rate = 0;
float sqrate = 0;
DataInputStream @in = null;
BufferedReader lines = null;
try
{
@in = new DataInputStream(fs.Open(reduceFile));
lines = new BufferedReader(new InputStreamReader(@in));
string line;
while ((line = lines.ReadLine()) != null)
{
StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%");
string attr = tokens.NextToken();
if (attr.EndsWith(":tasks"))
{
tasks = long.Parse(tokens.NextToken());
}
else
{
if (attr.EndsWith(":size"))
{
size = long.Parse(tokens.NextToken());
}
else
{
if (attr.EndsWith(":time"))
{
time = long.Parse(tokens.NextToken());
}
else
{
if (attr.EndsWith(":rate"))
{
rate = float.ParseFloat(tokens.NextToken());
}
else
{
if (attr.EndsWith(":sqrate"))
{
sqrate = float.ParseFloat(tokens.NextToken());
}
}
}
}
}
}
}
finally
{
if (@in != null)
{
@in.Close();
}
if (lines != null)
{
lines.Close();
}
}
double med = rate / 1000 / tasks;
double stdDev = Math.Sqrt(Math.Abs(sqrate / 1000 / tasks - med * med));
string[] resultLines = new string[] { "----- TestDFSIO ----- : " + testType, " Date & time: "
+ Sharpen.Extensions.CreateDate(Runtime.CurrentTimeMillis()), " Number of files: "
+ tasks, "Total MBytes processed: " + ToMB(size), " Throughput mb/sec: " +
size * 1000.0 / (time * Mega), "Average IO rate mb/sec: " + med, " IO rate std deviation: "
+ stdDev, " Test exec time sec: " + (float)execTime / 1000, string.Empty };
TextWriter res = null;
try
{
res = new TextWriter(new FileOutputStream(new FilePath(resFileName), true));
for (int i = 0; i < resultLines.Length; i++)
{
Log.Info(resultLines[i]);
res.WriteLine(resultLines[i]);
}
}
finally
{
if (res != null)
{
res.Close();
}
}
}
