public void Write(RecordWriter xout)
{
xout.Open("DataRecord");
xout.NamedIndex("pl", IBM.ICU.Impl.Duration.Impl.DataRecord.EPluralization.names, pl);
xout.StringTable("pluralName", pluralNames);
xout.NamedIndexArray("gender", IBM.ICU.Impl.Duration.Impl.DataRecord.EGender.names, genders);
xout.StringArray("singularName", singularNames);
xout.StringArray("halfName", halfNames);
xout.StringArray("numberName", numberNames);
xout.StringArray("mediumName", mediumNames);
xout.StringArray("shortName", shortNames);
xout.StringArray("measure", measures);
xout.StringArray("rqdSuffix", rqdSuffixes);
xout.StringArray("optSuffix", optSuffixes);
xout.StringArray("halves", halves);
xout.NamedIndexArray("halfPlacement", IBM.ICU.Impl.Duration.Impl.DataRecord.EHalfPlacement.names,
halfPlacements);
xout.NamedIndexArray("halfSupport", IBM.ICU.Impl.Duration.Impl.DataRecord.EHalfSupport.names, halfSupport);
xout.String("fifteenMinutes", fifteenMinutes);
xout.String("fiveMinutes", fiveMinutes);
xout.Bool("requiresDigitSeparator", requiresDigitSeparator);
xout.String("digitPrefix", digitPrefix);
xout.String("countSep", countSep);
xout.String("shortUnitSep", shortUnitSep);
xout.StringArray("unitSep", unitSep);
xout.BoolArray("unitSepRequiresDP", unitSepRequiresDP);
xout.BoolArray("requiresSkipMarker", requiresSkipMarker);
xout.NamedIndex("numberSystem", IBM.ICU.Impl.Duration.Impl.DataRecord.ENumberSystem.names, numberSystem);
xout.Character("zero", zero);
xout.Character("decimalSep", decimalSep);
xout.Bool("omitSingularCount", omitSingularCount);
xout.Bool("omitDualCount", omitDualCount);
xout.NamedIndex("zeroHandling", IBM.ICU.Impl.Duration.Impl.DataRecord.EZeroHandling.names, zeroHandling);
xout.NamedIndex("decimalHandling", IBM.ICU.Impl.Duration.Impl.DataRecord.EDecimalHandling.names,
decimalHandling);
xout.NamedIndex("fractionHandling", IBM.ICU.Impl.Duration.Impl.DataRecord.EFractionHandling.names,
fractionHandling);
xout.String("skippedUnitMarker", skippedUnitMarker);
xout.Bool("allowZero", allowZero);
xout.Bool("weeksAloneOnly", weeksAloneOnly);
xout.NamedIndex("useMilliseconds", IBM.ICU.Impl.Duration.Impl.DataRecord.EMilliSupport.names, useMilliseconds);
if (scopeData != null)
{
xout.Open("ScopeDataList");
for (int i = 0; i < scopeData.Length; ++i)
{
scopeData[i].Write(xout);
}
xout.Close();
}
xout.Close();
}
public void Write(RecordWriter xout)
{
xout.Open("ScopeData");
xout.String("prefix", prefix);
xout.Bool("requiresDigitPrefix", requiresDigitPrefix);
xout.String("suffix", suffix);
xout.Close();
}
public virtual void TestFormatWithCustomSeparator()
{
JobConf job = new JobConf();
string separator = "\u0001";
job.Set("mapreduce.output.textoutputformat.separator", separator);
job.Set(JobContext.TaskAttemptId, attempt);
FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent());
FileOutputFormat.SetWorkOutputPath(job, workDir);
FileSystem fs = workDir.GetFileSystem(job);
if (!fs.Mkdirs(workDir))
{
NUnit.Framework.Assert.Fail("Failed to create output directory");
}
string file = "test_custom.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat <object, object
>();
RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, reporter);
Org.Apache.Hadoop.IO.Text key1 = new Org.Apache.Hadoop.IO.Text("key1");
Org.Apache.Hadoop.IO.Text key2 = new Org.Apache.Hadoop.IO.Text("key2");
Org.Apache.Hadoop.IO.Text val1 = new Org.Apache.Hadoop.IO.Text("val1");
Org.Apache.Hadoop.IO.Text val2 = new Org.Apache.Hadoop.IO.Text("val2");
NullWritable nullWritable = NullWritable.Get();
try
{
theRecordWriter.Write(key1, val1);
theRecordWriter.Write(null, nullWritable);
theRecordWriter.Write(null, val1);
theRecordWriter.Write(nullWritable, val2);
theRecordWriter.Write(key2, nullWritable);
theRecordWriter.Write(key1, null);
theRecordWriter.Write(null, null);
theRecordWriter.Write(key2, val2);
}
finally
{
theRecordWriter.Close(reporter);
}
FilePath expectedFile = new FilePath(new Path(workDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append(separator).Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append(separator).Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), output);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Close(TaskAttemptContext context)
{
long bytesOutPrev = GetOutputBytes(fsStats);
real.Close(context);
long bytesOutCurr = GetOutputBytes(fsStats);
fileOutputByteCounter.Increment(bytesOutCurr - bytesOutPrev);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Close(Reporter reporter)
{
long bytesOutPrev = GetOutputBytes(fsStats);
real.Close(reporter);
long bytesOutCurr = GetOutputBytes(fsStats);
fileOutputByteCounter.Increment(bytesOutCurr - bytesOutPrev);
}
/// <exception cref="System.IO.IOException"/>
public void Close(Reporter reporter)
{
IEnumerator <string> keys = this.recordWriters.Keys.GetEnumerator();
while (keys.HasNext())
{
RecordWriter <K, V> rw = this.recordWriters[keys.Next()];
rw.Close(reporter);
}
this.recordWriters.Clear();
}
/// <exception cref="System.IO.IOException"/>
private static void Test1(JobConf job)
{
FileSystem fs = FileSystem.GetLocal(job);
string name = "part-00000";
TestMultipleTextOutputFormat.KeyBasedMultipleTextOutputFormat theOutputFormat = new
TestMultipleTextOutputFormat.KeyBasedMultipleTextOutputFormat();
RecordWriter <Text, Text> rw = theOutputFormat.GetRecordWriter(fs, job, name, null
);
WriteData(rw);
rw.Close(null);
}
private void CloseQuietly <Outkey, Outvalue>(RecordWriter <OUTKEY, OUTVALUE> c, Reporter
r)
{
if (c != null)
{
try
{
c.Close(r);
}
catch (Exception e)
{
Log.Info("Exception in closing " + c, e);
}
}
}
/// <exception cref="System.IO.IOException"/>
private static void Test2(JobConf job)
{
FileSystem fs = FileSystem.GetLocal(job);
string name = "part-00000";
//pretend that we have input file with 1/2/3 as the suffix
job.Set(JobContext.MapInputFile, "1/2/3");
// we use the last two legs of the input file as the output file
job.Set("mapred.outputformat.numOfTrailingLegs", "2");
MultipleTextOutputFormat <Text, Text> theOutputFormat = new MultipleTextOutputFormat
<Text, Text>();
RecordWriter <Text, Text> rw = theOutputFormat.GetRecordWriter(fs, job, name, null
);
WriteData(rw);
rw.Close(null);
}
private void Dispose()
{
if (RecordWriter != null)
{
RecordWriter.Close();
RecordWriter.Dispose();
}
try
{
File.Move(FilePath, GetNewFilePath(FilePath));
}
catch (IOException)
{
//
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void WriteMapFileOutput(RecordWriter theRecordWriter, TaskAttemptContext
context)
{
try
{
int key = 0;
for (int i = 0; i < 10; ++i)
{
key = i;
Text val = (i % 2 == 1) ? val1 : val2;
theRecordWriter.Write(new LongWritable(key), val);
}
}
finally
{
theRecordWriter.Close(context);
}
}
/// <summary>
/// A test that mimics a failed task to ensure that it does
/// not get into the COMMIT_PENDING state, by using a fake
/// UmbilicalProtocol's implementation that fails if the commit.
/// </summary>
/// <remarks>
/// A test that mimics a failed task to ensure that it does
/// not get into the COMMIT_PENDING state, by using a fake
/// UmbilicalProtocol's implementation that fails if the commit.
/// protocol is played.
/// The test mocks the various steps in a failed task's
/// life-cycle using a special OutputCommitter and UmbilicalProtocol
/// implementation.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestTaskCleanupDoesNotCommit()
{
// Mimic a job with a special committer that does not cleanup
// files when a task fails.
JobConf job = new JobConf();
job.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithoutCleanup));
Path outDir = new Path(rootDir, "output");
FileOutputFormat.SetOutputPath(job, outDir);
// Mimic job setup
string dummyAttemptID = "attempt_200707121733_0001_m_000000_0";
TaskAttemptID attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID));
OutputCommitter committer = new TestTaskCommit.CommitterWithoutCleanup();
JobContext jContext = new JobContextImpl(job, ((JobID)attemptID.GetJobID()));
committer.SetupJob(jContext);
// Mimic a map task
dummyAttemptID = "attempt_200707121733_0001_m_000001_0";
attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID));
Task task = new MapTask(null, attemptID, 0, null, 1);
task.SetConf(job);
task.LocalizeConfiguration(job);
task.Initialize(job, ((JobID)attemptID.GetJobID()), Reporter.Null, false);
// Mimic the map task writing some output.
string file = "test.txt";
FileSystem localFs = FileSystem.GetLocal(job);
TextOutputFormat <Text, Text> theOutputFormat = new TextOutputFormat <Text, Text>();
RecordWriter <Text, Text> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, Reporter.Null);
theRecordWriter.Write(new Text("key"), new Text("value"));
theRecordWriter.Close(Reporter.Null);
// Mimic a task failure; setting up the task for cleanup simulates
// the abort protocol to be played.
// Without checks in the framework, this will fail
// as the committer will cause a COMMIT to happen for
// the cleanup task.
task.SetTaskCleanupTask();
TestTaskCommit.MyUmbilical umbilical = new TestTaskCommit.MyUmbilical(this);
task.Run(job, umbilical);
NUnit.Framework.Assert.IsTrue("Task did not succeed", umbilical.taskDone);
}
// A random task attempt id for testing.
/// <exception cref="System.IO.IOException"/>
private void WriteOutput(RecordWriter theRecordWriter, Reporter reporter)
{
NullWritable nullWritable = NullWritable.Get();
try
{
theRecordWriter.Write(key1, val1);
theRecordWriter.Write(null, nullWritable);
theRecordWriter.Write(null, val1);
theRecordWriter.Write(nullWritable, val2);
theRecordWriter.Write(key2, nullWritable);
theRecordWriter.Write(key1, null);
theRecordWriter.Write(null, null);
theRecordWriter.Write(key2, val2);
}
finally
{
theRecordWriter.Close(reporter);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private void WriteOutput(RecordWriter theRecordWriter, TaskAttemptContext context
)
{
NullWritable nullWritable = NullWritable.Get();
try
{
theRecordWriter.Write(key1, val1);
theRecordWriter.Write(null, nullWritable);
theRecordWriter.Write(null, val1);
theRecordWriter.Write(nullWritable, val2);
theRecordWriter.Write(key2, nullWritable);
theRecordWriter.Write(key1, null);
theRecordWriter.Write(null, null);
theRecordWriter.Write(key2, val2);
}
finally
{
theRecordWriter.Close(context);
}
}
/// <exception cref="System.IO.IOException"/>
internal static long WriteBench(JobConf conf)
{
// OutputFormat instantiation
long filelen = conf.GetLong("filebench.file.bytes", 5 * 1024 * 1024 * 1024);
Org.Apache.Hadoop.IO.Text key = new Org.Apache.Hadoop.IO.Text();
Org.Apache.Hadoop.IO.Text val = new Org.Apache.Hadoop.IO.Text();
string fn = conf.Get("test.filebench.name", string.Empty);
Path outd = FileOutputFormat.GetOutputPath(conf);
conf.Set("mapred.work.output.dir", outd.ToString());
OutputFormat outf = conf.GetOutputFormat();
RecordWriter <Org.Apache.Hadoop.IO.Text, Org.Apache.Hadoop.IO.Text> rw = outf.GetRecordWriter
(outd.GetFileSystem(conf), conf, fn, Reporter.Null);
try
{
long acc = 0L;
DateTime start = new DateTime();
for (int i = 0; acc < filelen; ++i)
{
i %= keys.Length;
key.Set(keys[i]);
val.Set(values[i]);
rw.Write(key, val);
acc += keys[i].Length;
acc += values[i].Length;
}
DateTime end = new DateTime();
return(end.GetTime() - start.GetTime());
}
finally
{
rw.Close(Reporter.Null);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void Close(Reporter reporter)
{
writer.Close(reporter);
}
public void Close() => _recordWriter.Close();
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Close(TaskAttemptContext context)
{
writer.Close(context);
}
public virtual void TestCompress()
{
JobConf job = new JobConf();
job.Set(JobContext.TaskAttemptId, attempt);
job.Set(FileOutputFormat.Compress, "true");
FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent());
FileOutputFormat.SetWorkOutputPath(job, workDir);
FileSystem fs = workDir.GetFileSystem(job);
if (!fs.Mkdirs(workDir))
{
NUnit.Framework.Assert.Fail("Failed to create output directory");
}
string file = "test_compress.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat <object, object
>();
RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, reporter);
Org.Apache.Hadoop.IO.Text key1 = new Org.Apache.Hadoop.IO.Text("key1");
Org.Apache.Hadoop.IO.Text key2 = new Org.Apache.Hadoop.IO.Text("key2");
Org.Apache.Hadoop.IO.Text val1 = new Org.Apache.Hadoop.IO.Text("val1");
Org.Apache.Hadoop.IO.Text val2 = new Org.Apache.Hadoop.IO.Text("val2");
NullWritable nullWritable = NullWritable.Get();
try
{
theRecordWriter.Write(key1, val1);
theRecordWriter.Write(null, nullWritable);
theRecordWriter.Write(null, val1);
theRecordWriter.Write(nullWritable, val2);
theRecordWriter.Write(key2, nullWritable);
theRecordWriter.Write(key1, null);
theRecordWriter.Write(null, null);
theRecordWriter.Write(key2, val2);
}
finally
{
theRecordWriter.Close(reporter);
}
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append("\t").Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append("\t").Append(val2).Append("\n");
DefaultCodec codec = new DefaultCodec();
codec.SetConf(job);
Path expectedFile = new Path(workDir, file + codec.GetDefaultExtension());
FileInputStream istream = new FileInputStream(expectedFile.ToString());
CompressionInputStream cistream = codec.CreateInputStream(istream);
LineReader reader = new LineReader(cistream);
string output = string.Empty;
Org.Apache.Hadoop.IO.Text @out = new Org.Apache.Hadoop.IO.Text();
while (reader.ReadLine(@out) > 0)
{
output += @out;
output += "\n";
}
reader.Close();
NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), output);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void TestBinary()
{
Configuration conf = new Configuration();
Job job = Job.GetInstance(conf);
Path outdir = new Path(Runtime.GetProperty("test.build.data", "/tmp"), "outseq");
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
FileOutputFormat.SetOutputPath(job, outdir);
SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputKeyClass(job, typeof(IntWritable
));
SequenceFileAsBinaryOutputFormat.SetSequenceFileOutputValueClass(job, typeof(DoubleWritable
));
SequenceFileAsBinaryOutputFormat.SetCompressOutput(job, true);
SequenceFileAsBinaryOutputFormat.SetOutputCompressionType(job, SequenceFile.CompressionType
.Block);
BytesWritable bkey = new BytesWritable();
BytesWritable bval = new BytesWritable();
TaskAttemptContext context = MapReduceTestUtil.CreateDummyMapTaskAttemptContext(job
.GetConfiguration());
OutputFormat <BytesWritable, BytesWritable> outputFormat = new SequenceFileAsBinaryOutputFormat
();
OutputCommitter committer = outputFormat.GetOutputCommitter(context);
committer.SetupJob(job);
RecordWriter <BytesWritable, BytesWritable> writer = outputFormat.GetRecordWriter(
context);
IntWritable iwritable = new IntWritable();
DoubleWritable dwritable = new DoubleWritable();
DataOutputBuffer outbuf = new DataOutputBuffer();
Log.Info("Creating data by SequenceFileAsBinaryOutputFormat");
try
{
for (int i = 0; i < Records; ++i)
{
iwritable = new IntWritable(r.Next());
iwritable.Write(outbuf);
bkey.Set(outbuf.GetData(), 0, outbuf.GetLength());
outbuf.Reset();
dwritable = new DoubleWritable(r.NextDouble());
dwritable.Write(outbuf);
bval.Set(outbuf.GetData(), 0, outbuf.GetLength());
outbuf.Reset();
writer.Write(bkey, bval);
}
}
finally
{
writer.Close(context);
}
committer.CommitTask(context);
committer.CommitJob(job);
InputFormat <IntWritable, DoubleWritable> iformat = new SequenceFileInputFormat <IntWritable
, DoubleWritable>();
int count = 0;
r.SetSeed(seed);
SequenceFileInputFormat.SetInputPaths(job, outdir);
Log.Info("Reading data by SequenceFileInputFormat");
foreach (InputSplit split in iformat.GetSplits(job))
{
RecordReader <IntWritable, DoubleWritable> reader = iformat.CreateRecordReader(split
, context);
MapContext <IntWritable, DoubleWritable, BytesWritable, BytesWritable> mcontext =
new MapContextImpl <IntWritable, DoubleWritable, BytesWritable, BytesWritable>(job
.GetConfiguration(), context.GetTaskAttemptID(), reader, null, null, MapReduceTestUtil
.CreateDummyReporter(), split);
reader.Initialize(split, mcontext);
try
{
int sourceInt;
double sourceDouble;
while (reader.NextKeyValue())
{
sourceInt = r.Next();
sourceDouble = r.NextDouble();
iwritable = reader.GetCurrentKey();
dwritable = reader.GetCurrentValue();
NUnit.Framework.Assert.AreEqual("Keys don't match: " + "*" + iwritable.Get() + ":"
+ sourceInt + "*", sourceInt, iwritable.Get());
NUnit.Framework.Assert.IsTrue("Vals don't match: " + "*" + dwritable.Get() + ":"
+ sourceDouble + "*", double.Compare(dwritable.Get(), sourceDouble) == 0);
++count;
}
}
finally
{
reader.Close();
}
}
NUnit.Framework.Assert.AreEqual("Some records not found", Records, count);
}