protected internal static void SetReducerConf(Configuration jobConf, Type inputKeyClass
, Type inputValueClass, Type outputKeyClass, Type outputValueClass, Configuration
reducerConf, string prefix)
{
// if the Reducer does not have a Configuration, create an empty one
if (reducerConf == null)
{
// using a Configuration without defaults to make it lightweight.
// still the chain's conf may have all defaults and this conf is
// overlapped to the chain's Configuration one.
reducerConf = new Configuration(false);
}
// store the input/output classes of the reducer in
// the reducer configuration
reducerConf.SetClass(ReducerInputKeyClass, inputKeyClass, typeof(object));
reducerConf.SetClass(ReducerInputValueClass, inputValueClass, typeof(object));
reducerConf.SetClass(ReducerOutputKeyClass, outputKeyClass, typeof(object));
reducerConf.SetClass(ReducerOutputValueClass, outputValueClass, typeof(object));
// serialize the reducer configuration in the chain's configuration.
Stringifier <Configuration> stringifier = new DefaultStringifier <Configuration>(jobConf
, typeof(Configuration));
try
{
jobConf.Set(prefix + ChainReducerConfig, stringifier.ToString(new Configuration(reducerConf
)));
}
catch (IOException ioEx)
{
throw new RuntimeException(ioEx);
}
}
protected internal static void SetMapperConf(bool isMap, Configuration jobConf, Type
inputKeyClass, Type inputValueClass, Type outputKeyClass, Type outputValueClass
, Configuration mapperConf, int index, string prefix)
{
// if the Mapper does not have a configuration, create an empty one
if (mapperConf == null)
{
// using a Configuration without defaults to make it lightweight.
// still the chain's conf may have all defaults and this conf is
// overlapped to the chain configuration one.
mapperConf = new Configuration(true);
}
// store the input/output classes of the mapper in the mapper conf
mapperConf.SetClass(MapperInputKeyClass, inputKeyClass, typeof(object));
mapperConf.SetClass(MapperInputValueClass, inputValueClass, typeof(object));
mapperConf.SetClass(MapperOutputKeyClass, outputKeyClass, typeof(object));
mapperConf.SetClass(MapperOutputValueClass, outputValueClass, typeof(object));
// serialize the mapper configuration in the chain configuration.
Stringifier <Configuration> stringifier = new DefaultStringifier <Configuration>(jobConf
, typeof(Configuration));
try
{
jobConf.Set(prefix + ChainMapperConfig + index, stringifier.ToString(new Configuration
(mapperConf)));
}
catch (IOException ioEx)
{
throw new RuntimeException(ioEx);
}
// increment the chain counter
jobConf.SetInt(prefix + ChainMapperSize, index + 1);
}
/// <summary>
/// Creates a
/// <see cref="Org.Apache.Hadoop.Conf.Configuration"/>
/// for the Map or Reduce in the chain.
/// <p>
/// It creates a new Configuration using the chain job's Configuration as base
/// and adds to it the configuration properties for the chain element. The keys
/// of the chain element Configuration have precedence over the given
/// Configuration.
/// </p>
/// </summary>
/// <param name="jobConf">the chain job's Configuration.</param>
/// <param name="confKey">
/// the key for chain element configuration serialized in the chain
/// job's Configuration.
/// </param>
/// <returns>
/// a new Configuration aggregating the chain job's Configuration with
/// the chain element configuration properties.
/// </returns>
protected internal static Configuration GetChainElementConf(Configuration jobConf
, string confKey)
{
Configuration conf = null;
try
{
using (Stringifier <Configuration> stringifier = new DefaultStringifier <Configuration
>(jobConf, typeof(Configuration)))
{
string confString = jobConf.Get(confKey, null);
if (confString != null)
{
conf = stringifier.FromString(jobConf.Get(confKey, null));
}
}
}
catch (IOException ioex)
{
throw new RuntimeException(ioex);
}
// we have to do this because the Writable desearialization clears all
// values set in the conf making not possible do a
// new Configuration(jobConf) in the creation of the conf above
jobConf = new Configuration(jobConf);
if (conf != null)
{
foreach (KeyValuePair <string, string> entry in conf)
{
jobConf.Set(entry.Key, entry.Value);
}
}
return(jobConf);
}
public override string ToString() => DefaultStringifier.ToString("Function", Name, Id);
public override string ToString() => DefaultStringifier.ToString("Replaceable", Name, Id);
public override string ToString() => DefaultStringifier.ToString("LimitedList", Name, Id);
public override string ToString() => DefaultStringifier.ToString("Regex", Name, Id);
public override string ToString() => DefaultStringifier.ToString("ContinueWith", Name, Id);
public override string ToString() => DefaultStringifier.ToString(this);
public override string ToString() => DefaultStringifier.ToString("Create", Name, Id);
public override string ToString() => DefaultStringifier.ToString("Sequential", Name, Id);
public override string ToString() => DefaultStringifier.ToString("Examine", Name, Id);
public override string ToString() => DefaultStringifier.ToString("Earley", Name, Id);
