/// <exception cref="System.Exception"/>
private static IList <string> ReadSplit(FixedLengthInputFormat format, InputSplit
split, Job job)
{
IList <string> result = new AList <string>();
TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job
.GetConfiguration());
RecordReader <LongWritable, BytesWritable> reader = format.CreateRecordReader(split
, context);
MapContext <LongWritable, BytesWritable, LongWritable, BytesWritable> mcontext = new
MapContextImpl <LongWritable, BytesWritable, LongWritable, BytesWritable>(job.GetConfiguration
(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil.CreateDummyReporter
(), split);
LongWritable key;
BytesWritable value;
try
{
reader.Initialize(split, mcontext);
while (reader.NextKeyValue())
{
key = reader.GetCurrentKey();
value = reader.GetCurrentValue();
result.AddItem(Sharpen.Runtime.GetStringForBytes(value.GetBytes(), 0, value.GetLength
()));
}
}
finally
{
reader.Close();
}
return(result);
}
/// <summary>Test with record length set to a negative value</summary>
/// <exception cref="System.Exception"/>
public virtual void TestNegativeRecordLength()
{
localFs.Delete(workDir, true);
Path file = new Path(workDir, new string("testFormat.txt"));
CreateFile(file, null, 10, 10);
// Set the fixed length record length config property
Job job = Job.GetInstance(defaultConf);
FixedLengthInputFormat format = new FixedLengthInputFormat();
FixedLengthInputFormat.SetRecordLength(job.GetConfiguration(), -10);
FileInputFormat.SetInputPaths(job, workDir);
IList <InputSplit> splits = format.GetSplits(job);
bool exceptionThrown = false;
foreach (InputSplit split in splits)
{
try
{
TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job
.GetConfiguration());
RecordReader <LongWritable, BytesWritable> reader = format.CreateRecordReader(split
, context);
MapContext <LongWritable, BytesWritable, LongWritable, BytesWritable> mcontext = new
MapContextImpl <LongWritable, BytesWritable, LongWritable, BytesWritable>(job.GetConfiguration
(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil.CreateDummyReporter
(), split);
reader.Initialize(split, mcontext);
}
catch (IOException ioe)
{
exceptionThrown = true;
Log.Info("Exception message:" + ioe.Message);
}
}
NUnit.Framework.Assert.IsTrue("Exception for negative record length:", exceptionThrown
);
}
/// <exception cref="System.Exception"/>
private void RunRandomTests(CompressionCodec codec)
{
StringBuilder fileName = new StringBuilder("testFormat.txt");
if (codec != null)
{
fileName.Append(".gz");
}
localFs.Delete(workDir, true);
Path file = new Path(workDir, fileName.ToString());
int seed = new Random().Next();
Log.Info("Seed = " + seed);
Random random = new Random(seed);
int MaxTests = 20;
LongWritable key;
BytesWritable value;
for (int i = 0; i < MaxTests; i++)
{
Log.Info("----------------------------------------------------------");
// Maximum total records of 999
int totalRecords = random.Next(999) + 1;
// Test an empty file
if (i == 8)
{
totalRecords = 0;
}
// Maximum bytes in a record of 100K
int recordLength = random.Next(1024 * 100) + 1;
// For the 11th test, force a record length of 1
if (i == 10)
{
recordLength = 1;
}
// The total bytes in the test file
int fileSize = (totalRecords * recordLength);
Log.Info("totalRecords=" + totalRecords + " recordLength=" + recordLength);
// Create the job
Job job = Job.GetInstance(defaultConf);
if (codec != null)
{
ReflectionUtils.SetConf(codec, job.GetConfiguration());
}
// Create the test file
AList <string> recordList = CreateFile(file, codec, recordLength, totalRecords);
NUnit.Framework.Assert.IsTrue(localFs.Exists(file));
//set the fixed length record length config property for the job
FixedLengthInputFormat.SetRecordLength(job.GetConfiguration(), recordLength);
int numSplits = 1;
// Arbitrarily set number of splits.
if (i > 0)
{
if (i == (MaxTests - 1))
{
// Test a split size that is less than record len
numSplits = (int)(fileSize / Math.Floor(recordLength / 2));
}
else
{
if (MaxTests % i == 0)
{
// Let us create a split size that is forced to be
// smaller than the end file itself, (ensures 1+ splits)
numSplits = fileSize / (fileSize - random.Next(fileSize));
}
else
{
// Just pick a random split size with no upper bound
numSplits = Math.Max(1, fileSize / random.Next(int.MaxValue));
}
}
Log.Info("Number of splits set to: " + numSplits);
}
job.GetConfiguration().SetLong("mapreduce.input.fileinputformat.split.maxsize", (
long)(fileSize / numSplits));
// setup the input path
FileInputFormat.SetInputPaths(job, workDir);
// Try splitting the file in a variety of sizes
FixedLengthInputFormat format = new FixedLengthInputFormat();
IList <InputSplit> splits = format.GetSplits(job);
Log.Info("Actual number of splits = " + splits.Count);
// Test combined split lengths = total file size
long recordOffset = 0;
int recordNumber = 0;
foreach (InputSplit split in splits)
{
TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job
.GetConfiguration());
RecordReader <LongWritable, BytesWritable> reader = format.CreateRecordReader(split
, context);
MapContext <LongWritable, BytesWritable, LongWritable, BytesWritable> mcontext = new
MapContextImpl <LongWritable, BytesWritable, LongWritable, BytesWritable>(job.GetConfiguration
(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil.CreateDummyReporter
(), split);
reader.Initialize(split, mcontext);
Type clazz = reader.GetType();
NUnit.Framework.Assert.AreEqual("RecordReader class should be FixedLengthRecordReader:"
, typeof(FixedLengthRecordReader), clazz);
// Plow through the records in this split
while (reader.NextKeyValue())
{
key = reader.GetCurrentKey();
value = reader.GetCurrentValue();
NUnit.Framework.Assert.AreEqual("Checking key", (long)(recordNumber * recordLength
), key.Get());
string valueString = Sharpen.Runtime.GetStringForBytes(value.GetBytes(), 0, value
.GetLength());
NUnit.Framework.Assert.AreEqual("Checking record length:", recordLength, value.GetLength
());
NUnit.Framework.Assert.IsTrue("Checking for more records than expected:", recordNumber
< totalRecords);
string origRecord = recordList[recordNumber];
NUnit.Framework.Assert.AreEqual("Checking record content:", origRecord, valueString
);
recordNumber++;
}
reader.Close();
}
NUnit.Framework.Assert.AreEqual("Total original records should be total read records:"
, recordList.Count, recordNumber);
}
}
