public Writer(Configuration conf, FileSystem fs, string dirName, WritableComparator
comparator, Type valClass, SequenceFile.CompressionType compress, CompressionCodec
codec, Progressable progress)
: this(conf, new Path(dirName), Comparator(comparator), ValueClass(valClass), Compression
(compress, codec), Progressable(progress))
{
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
// call to primitiveCreate
CheckPath(f);
// Default impl assumes that permissions do not matter
// calling the regular create is good enough.
// FSs that implement permissions should override this.
if (!createParent)
{
// parent must exist.
// since this.create makes parent dirs automatically
// we must throw exception if parent does not exist.
FileStatus stat = GetFileStatus(f.GetParent());
if (stat == null)
{
throw new FileNotFoundException("Missing parent:" + f);
}
if (!stat.IsDirectory())
{
throw new ParentNotDirectoryException("parent is not a dir:" + f);
}
}
// parent does exist - go ahead with create of file.
return(fsImpl.PrimitiveCreate(f, absolutePermission, flag, bufferSize, replication
, blockSize, progress, checksumOpt));
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
// deliberately empty
return(null);
}
public MergeQueue(Configuration conf, FileSystem fs, IList <Merger.Segment <K, V> >
segments, RawComparator <K> comparator, Progressable reporter, bool sortSegments,
CompressionCodec codec, TaskType taskType)
: this(conf, fs, segments, comparator, reporter, sortSegments, taskType)
{
this.codec = codec;
}
public override FSDataOutputStream CreateNonRecursive(Path f, FsPermission permission
, EnumSet <CreateFlag> flags, int bufferSize, short replication, long blockSize,
Progressable progress)
{
return(fs.CreateNonRecursive(f, permission, flags, bufferSize, replication, blockSize
, progress));
}
public CombineOutputCollector(TestPipeApplication _enclosing, Counters.Counter outCounter
, Progressable progressable)
{
this._enclosing = _enclosing;
this.outCounter = outCounter;
this.progressable = progressable;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> flag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
return(myFs.CreateInternal(FullPath(f), flag, absolutePermission, bufferSize, replication
, blockSize, progress, checksumOpt, createParent));
}
/// <exception cref="System.IO.IOException"/>
public override RecordWriter <K, V> GetRecordWriter(FileSystem ignored, JobConf job
, string name, Progressable progress)
{
bool isCompressed = GetCompressOutput(job);
string keyValueSeparator = job.Get("mapreduce.output.textoutputformat.separator",
"\t");
if (!isCompressed)
{
Path file = FileOutputFormat.GetTaskOutputPath(job, name);
FileSystem fs = file.GetFileSystem(job);
FSDataOutputStream fileOut = fs.Create(file, progress);
return(new TextOutputFormat.LineRecordWriter <K, V>(fileOut, keyValueSeparator));
}
else
{
Type codecClass = GetOutputCompressorClass(job, typeof(GzipCodec));
// create the named codec
CompressionCodec codec = ReflectionUtils.NewInstance(codecClass, job);
// build the filename including the extension
Path file = FileOutputFormat.GetTaskOutputPath(job, name + codec.GetDefaultExtension
());
FileSystem fs = file.GetFileSystem(job);
FSDataOutputStream fileOut = fs.Create(file, progress);
return(new TextOutputFormat.LineRecordWriter <K, V>(new DataOutputStream(codec.CreateOutputStream
(fileOut)), keyValueSeparator));
}
}
protected internal Progress(Progressable prog)
{
if (prog == null)
{
throw new ArgumentException("Progress must not be null");
}
progress = prog;
}
/// <exception cref="System.IO.IOException"/>
public LazyRecordWriter(JobConf job, OutputFormat of, string name, Progressable progress
)
{
this.of = of;
this.job = job;
this.name = name;
this.progress = progress;
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
DFSOutputStream dfsos = dfs.PrimitiveCreate(GetUriPath(f), absolutePermission, createFlag
, createParent, replication, blockSize, progress, bufferSize, checksumOpt);
return(dfs.CreateWrappedOutputStream(dfsos, statistics, dfsos.GetInitialLen()));
}
/// <exception cref="System.IO.IOException"/>
public override RecordWriter <K, V> GetRecordWriter(FileSystem ignored, JobConf job
, string name, Progressable progress)
{
if (baseOut == null)
{
GetBaseOutputFormat(job);
}
return(new LazyOutputFormat.LazyRecordWriter <K, V>(job, baseOut, name, progress));
}
/// <exception cref="System.IO.IOException"/>
public static RawKeyValueIterator Merge <K, V>(Configuration conf, FileSystem fs,
IList <Merger.Segment <K, V> > segments, int mergeFactor, Path tmpDir, RawComparator
<K> comparator, Progressable reporter, Counters.Counter readsCounter, Counters.Counter
writesCounter, Progress mergePhase)
{
System.Type keyClass = typeof(K);
System.Type valueClass = typeof(V);
return(Merge(conf, fs, keyClass, valueClass, segments, mergeFactor, tmpDir, comparator
, reporter, false, readsCounter, writesCounter, mergePhase));
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateInternal(Path f, EnumSet <CreateFlag> createFlag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
{
FSDataOutputStream @out = new FSDataOutputStream(new ChecksumFs.ChecksumFSOutputSummer
(this, f, createFlag, absolutePermission, bufferSize, replication, blockSize, progress
, checksumOpt, createParent), null);
return(@out);
}
public _RecordWriter_82(MultipleOutputFormat <K, V> _enclosing, string myName, JobConf
myJob, FileSystem myFS, Progressable myProgressable)
{
this._enclosing = _enclosing;
this.myName = myName;
this.myJob = myJob;
this.myFS = myFS;
this.myProgressable = myProgressable;
this.recordWriters = new SortedDictionary <string, RecordWriter <K, V> >();
}
/// <summary>
/// Create a composite record writer that can write key/value data to different
/// output files
/// </summary>
/// <param name="fs">the file system to use</param>
/// <param name="job">the job conf for the job</param>
/// <param name="name">the leaf file name for the output file (such as part-00000")</param>
/// <param name="arg3">a progressable for reporting progress.</param>
/// <returns>a composite record writer</returns>
/// <exception cref="System.IO.IOException"/>
public override RecordWriter <K, V> GetRecordWriter(FileSystem fs, JobConf job, string
name, Progressable arg3)
{
FileSystem myFS = fs;
string myName = GenerateLeafFileName(name);
JobConf myJob = job;
Progressable myProgressable = arg3;
return(new _RecordWriter_82(this, myName, myJob, myFS, myProgressable));
}
public override FSDataOutputStream CreateNonRecursive(Path f, FsPermission permission
, EnumSet <CreateFlag> flags, int bufferSize, short replication, long blockSize,
Progressable progress)
{
if (Exists(f) && !flags.Contains(CreateFlag.Overwrite))
{
throw new FileAlreadyExistsException("File already exists: " + f);
}
return(new FSDataOutputStream(new BufferedOutputStream(CreateOutputStreamWithMode
(f, false, permission), bufferSize), statistics));
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream Append(Path f, int bufferSize, Progressable progress
)
{
if (!Exists(f))
{
throw new FileNotFoundException("File " + f + " not found");
}
if (GetFileStatus(f).IsDirectory())
{
throw new IOException("Cannot append to a diretory (=" + f + " )");
}
return(new FSDataOutputStream(new BufferedOutputStream(CreateOutputStreamWithMode
(f, true, null), bufferSize), statistics));
}
/// <summary>
/// <inheritDoc/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual RecordWriter <K, V> GetRecordWriter(FileSystem filesystem, JobConf
job, string name, Progressable progress)
{
RecordWriter <K, V> w = base.GetRecordWriter(new TaskAttemptContextImpl(job, TaskAttemptID
.ForName(job.Get(MRJobConfig.TaskAttemptId))));
DBOutputFormat.DBRecordWriter writer = (DBOutputFormat.DBRecordWriter)w;
try
{
return(new DBOutputFormat.DBRecordWriter(this, writer.GetConnection(), writer.GetStatement
()));
}
catch (SQLException se)
{
throw new IOException(se);
}
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream CreateNonRecursive(Path f, FsPermission permission
, EnumSet <CreateFlag> flags, int bufferSize, short replication, long blockSize,
Progressable progress)
{
InodeTree.ResolveResult <FileSystem> res;
try
{
res = fsState.Resolve(GetUriPath(f), false);
}
catch (FileNotFoundException)
{
throw ReadOnlyMountTable("create", f);
}
System.Diagnostics.Debug.Assert((res.remainingPath != null));
return(res.targetFileSystem.CreateNonRecursive(res.remainingPath, permission, flags
, bufferSize, replication, blockSize, progress));
}
public MergeQueue(Configuration conf, FileSystem fs, IList <Merger.Segment <K, V> >
segments, RawComparator <K> comparator, Progressable reporter, bool sortSegments,
TaskType taskType)
{
this.conf = conf;
this.fs = fs;
this.comparator = comparator;
this.segments = segments;
this.reporter = reporter;
if (taskType == TaskType.Map)
{
ConsiderFinalMergeForProgress();
}
if (sortSegments)
{
segments.Sort(segmentComparator);
}
}
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
/// <exception cref="Org.Apache.Hadoop.FS.FileAlreadyExistsException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Org.Apache.Hadoop.FS.ParentNotDirectoryException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnsupportedFileSystemException"/>
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream Create(Path f, EnumSet <CreateFlag> createFlag,
params Options.CreateOpts[] opts)
{
// Need to translate the FileContext-style options into FileSystem-style
// Permissions with umask
Options.CreateOpts.Perms permOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.Perms
>(opts);
FsPermission umask = FsPermission.GetUMask(fs.GetConf());
FsPermission permission = (permOpt != null) ? permOpt.GetValue() : FsPermission.GetFileDefault
().ApplyUMask(umask);
permission = permission.ApplyUMask(umask);
// Overwrite
bool overwrite = createFlag.Contains(CreateFlag.Overwrite);
// bufferSize
int bufferSize = fs.GetConf().GetInt(CommonConfigurationKeysPublic.IoFileBufferSizeKey
, CommonConfigurationKeysPublic.IoFileBufferSizeDefault);
Options.CreateOpts.BufferSize bufOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.BufferSize
>(opts);
bufferSize = (bufOpt != null) ? bufOpt.GetValue() : bufferSize;
// replication
short replication = fs.GetDefaultReplication(f);
Options.CreateOpts.ReplicationFactor repOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.ReplicationFactor
>(opts);
replication = (repOpt != null) ? repOpt.GetValue() : replication;
// blockSize
long blockSize = fs.GetDefaultBlockSize(f);
Options.CreateOpts.BlockSize blockOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.BlockSize
>(opts);
blockSize = (blockOpt != null) ? blockOpt.GetValue() : blockSize;
// Progressable
Progressable progress = null;
Options.CreateOpts.Progress progressOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.Progress
>(opts);
progress = (progressOpt != null) ? progressOpt.GetValue() : progress;
return(fs.Create(f, permission, overwrite, bufferSize, replication, blockSize, progress
));
}
/// <exception cref="System.IO.IOException"/>
public override RecordWriter <K, V> GetRecordWriter(FileSystem ignored, JobConf job
, string name, Progressable progress)
{
// get the path of the temporary output file
Path file = FileOutputFormat.GetTaskOutputPath(job, name);
FileSystem fs = file.GetFileSystem(job);
CompressionCodec codec = null;
SequenceFile.CompressionType compressionType = SequenceFile.CompressionType.None;
if (GetCompressOutput(job))
{
// find the kind of compression to do
compressionType = GetOutputCompressionType(job);
// find the right codec
Type codecClass = GetOutputCompressorClass(job, typeof(DefaultCodec));
codec = ReflectionUtils.NewInstance(codecClass, job);
}
SequenceFile.Writer @out = SequenceFile.CreateWriter(fs, job, file, job.GetOutputKeyClass
(), job.GetOutputValueClass(), compressionType, codec, progress);
return(new _RecordWriter_71(@out));
}
/// <exception cref="System.IO.IOException"/>
public ChecksumFSOutputSummer(ChecksumFs fs, Path file, EnumSet <CreateFlag> createFlag
, FsPermission absolutePermission, int bufferSize, short replication, long blockSize
, Progressable progress, Options.ChecksumOpt checksumOpt, bool createParent)
: base(DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, fs.GetBytesPerSum())
)
{
// checksumOpt is passed down to the raw fs. Unless it implements
// checksum impelemts internally, checksumOpt will be ignored.
// If the raw fs does checksum internally, we will end up with
// two layers of checksumming. i.e. checksumming checksum file.
this.datas = fs.GetRawFs().CreateInternal(file, createFlag, absolutePermission, bufferSize
, replication, blockSize, progress, checksumOpt, createParent);
// Now create the chekcsumfile; adjust the buffsize
int bytesPerSum = fs.GetBytesPerSum();
int sumBufferSize = fs.GetSumBufferSize(bytesPerSum, bufferSize);
this.sums = fs.GetRawFs().CreateInternal(fs.GetChecksumFile(file), EnumSet.Of(CreateFlag
.Create, CreateFlag.Overwrite), absolutePermission, sumBufferSize, replication,
blockSize, progress, checksumOpt, createParent);
sums.Write(ChecksumVersion, 0, ChecksumVersion.Length);
sums.WriteInt(bytesPerSum);
}
public void Sort(IndexedSortable s, int p, int r, Progressable rep)
{
int N = r - p;
// build heap w/ reverse comparator, then write in-place from end
int t = int.HighestOneBit(N);
for (int i = t; i > 1; i = (int)(((uint)i) >> 1))
{
for (int j = (int)(((uint)i) >> 1); j < i; ++j)
{
DownHeap(s, p - 1, j, N + 1);
}
if (null != rep)
{
rep.Progress();
}
}
for (int i_1 = r - 1; i_1 > p; --i_1)
{
s.Swap(p, i_1);
DownHeap(s, p - 1, 1, i_1 - p + 1);
}
}
/// <exception cref="System.IO.IOException"/>
public MergeQueue(Configuration conf, FileSystem fs, Path[] inputs, bool deleteInputs
, CompressionCodec codec, RawComparator <K> comparator, Progressable reporter, Counters.Counter
mergedMapOutputsCounter, TaskType taskType)
{
this.conf = conf;
this.fs = fs;
this.codec = codec;
this.comparator = comparator;
this.reporter = reporter;
if (taskType == TaskType.Map)
{
ConsiderFinalMergeForProgress();
}
foreach (Path file in inputs)
{
Log.Debug("MergeQ: adding: " + file);
segments.AddItem(new Merger.Segment <K, V>(conf, fs, file, codec, !deleteInputs, (
file.ToString().EndsWith(Task.MergedOutputPrefix) ? null : mergedMapOutputsCounter
)));
}
// Sort segments on file-lengths
segments.Sort(segmentComparator);
}
public FSDataOutputStream create(Path path, FsPermission fsPermission,
bool overwrite, int bufferSize,
short replication, long blockSize,
Progressable progressable
) {
MockFile file = null;
for(MockFile currentFile: files) {
if (currentFile.path.equals(path)) {
file = currentFile;
break;
}
}
if (file == null) {
file = new MockFile(path.toString(), (int) blockSize, new byte[0]);
files.add(file);
}
return new MockOutputStream(file);
}
public FakeCollector(TestPipeApplication _enclosing, Counters.Counter outCounter,
Progressable progressable)
: base(_enclosing)
{
this._enclosing = _enclosing;
}
/// <exception cref="System.IO.IOException"/>
protected internal override FSDataOutputStream PrimitiveCreate(Path f, FsPermission
absolutePermission, EnumSet <CreateFlag> flag, int bufferSize, short replication
, long blockSize, Progressable progress, Options.ChecksumOpt checksumOpt)
{
return(fs.PrimitiveCreate(f, absolutePermission, flag, bufferSize, replication, blockSize
, progress, checksumOpt));
}
public FSDataOutputStream append(Path path, int bufferSize,
Progressable progressable
) {
return create(path, FsPermission.getDefault(), true, bufferSize,
(short) 3, 256 * 1024, progressable);
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream Create(Path f, FsPermission permission, EnumSet
<CreateFlag> flags, int bufferSize, short replication, long blockSize, Progressable
progress, Options.ChecksumOpt checksumOpt)
{
return(fs.Create(f, permission, flags, bufferSize, replication, blockSize, progress
, checksumOpt));
}
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream Create(Path f, FsPermission permission, bool overwrite
, int bufferSize, short replication, long blockSize, Progressable progress)
{
return(fs.Create(f, permission, overwrite, bufferSize, replication, blockSize, progress
));
}
