private static DBConfiguration SetOutput(JobConf job, string tableName)
{
job.SetOutputFormat(typeof(DBOutputFormat));
job.SetReduceSpeculativeExecution(false);
DBConfiguration dbConf = new DBConfiguration(job);
dbConf.SetOutputTableName(tableName);
return(dbConf);
}
/// <summary>Initializes the map-part of the job with the appropriate input settings.
/// </summary>
/// <param name="job">The job</param>
/// <param name="inputClass">
/// the class object implementing DBWritable, which is the
/// Java object holding tuple fields.
/// </param>
/// <param name="inputQuery">
/// the input query to select fields. Example :
/// "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
/// </param>
/// <param name="inputCountQuery">
/// the input query that returns the number of records in
/// the table.
/// Example : "SELECT COUNT(f1) FROM Mytable"
/// </param>
/// <seealso cref="DBInputFormat{T}.SetInput(Org.Apache.Hadoop.Mapred.JobConf, System.Type{T}, string, string, string, string[])
/// "/>
public static void SetInput(JobConf job, Type inputClass, string inputQuery, string
inputCountQuery)
{
job.SetInputFormat(typeof(DBInputFormat));
DBConfiguration dbConf = new DBConfiguration(job);
dbConf.SetInputClass(inputClass);
dbConf.SetInputQuery(inputQuery);
dbConf.SetInputCountQuery(inputCountQuery);
}
/// <summary>Initializes the map-part of the job with the appropriate input settings.
/// </summary>
/// <param name="job">The job</param>
/// <param name="inputClass">
/// the class object implementing DBWritable, which is the
/// Java object holding tuple fields.
/// </param>
/// <param name="tableName">The table to read data from</param>
/// <param name="conditions">
/// The condition which to select data with, eg. '(updated >
/// 20070101 AND length > 0)'
/// </param>
/// <param name="orderBy">the fieldNames in the orderBy clause.</param>
/// <param name="fieldNames">The field names in the table</param>
/// <seealso cref="DBInputFormat{T}.SetInput(Org.Apache.Hadoop.Mapred.JobConf, System.Type{T}, string, string)
/// "/>
public static void SetInput(JobConf job, Type inputClass, string tableName, string
conditions, string orderBy, params string[] fieldNames)
{
job.SetInputFormat(typeof(DBInputFormat));
DBConfiguration dbConf = new DBConfiguration(job);
dbConf.SetInputClass(inputClass);
dbConf.SetInputTableName(tableName);
dbConf.SetInputFieldNames(fieldNames);
dbConf.SetInputConditions(conditions);
dbConf.SetInputOrderBy(orderBy);
}
/// <summary>test configuration for db.</summary>
/// <remarks>test configuration for db. should works DBConfiguration.* parameters.</remarks>
public virtual void TestSetInput()
{
JobConf configuration = new JobConf();
string[] fieldNames = new string[] { "field1", "field2" };
DBInputFormat.SetInput(configuration, typeof(DBInputFormat.NullDBWritable), "table"
, "conditions", "orderBy", fieldNames);
NUnit.Framework.Assert.AreEqual("org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable"
, configuration.GetClass(DBConfiguration.InputClassProperty, null).FullName);
NUnit.Framework.Assert.AreEqual("table", configuration.Get(DBConfiguration.InputTableNameProperty
, null));
string[] fields = configuration.GetStrings(DBConfiguration.InputFieldNamesProperty
);
NUnit.Framework.Assert.AreEqual("field1", fields[0]);
NUnit.Framework.Assert.AreEqual("field2", fields[1]);
NUnit.Framework.Assert.AreEqual("conditions", configuration.Get(DBConfiguration.InputConditionsProperty
, null));
NUnit.Framework.Assert.AreEqual("orderBy", configuration.Get(DBConfiguration.InputOrderByProperty
, null));
configuration = new JobConf();
DBInputFormat.SetInput(configuration, typeof(DBInputFormat.NullDBWritable), "query"
, "countQuery");
NUnit.Framework.Assert.AreEqual("query", configuration.Get(DBConfiguration.InputQuery
, null));
NUnit.Framework.Assert.AreEqual("countQuery", configuration.Get(DBConfiguration.InputCountQuery
, null));
JobConf jConfiguration = new JobConf();
DBConfiguration.ConfigureDB(jConfiguration, "driverClass", "dbUrl", "user", "password"
);
NUnit.Framework.Assert.AreEqual("driverClass", jConfiguration.Get(DBConfiguration
.DriverClassProperty));
NUnit.Framework.Assert.AreEqual("dbUrl", jConfiguration.Get(DBConfiguration.UrlProperty
));
NUnit.Framework.Assert.AreEqual("user", jConfiguration.Get(DBConfiguration.UsernameProperty
));
NUnit.Framework.Assert.AreEqual("password", jConfiguration.Get(DBConfiguration.PasswordProperty
));
jConfiguration = new JobConf();
DBConfiguration.ConfigureDB(jConfiguration, "driverClass", "dbUrl");
NUnit.Framework.Assert.AreEqual("driverClass", jConfiguration.Get(DBConfiguration
.DriverClassProperty));
NUnit.Framework.Assert.AreEqual("dbUrl", jConfiguration.Get(DBConfiguration.UrlProperty
));
NUnit.Framework.Assert.IsNull(jConfiguration.Get(DBConfiguration.UsernameProperty
));
NUnit.Framework.Assert.IsNull(jConfiguration.Get(DBConfiguration.PasswordProperty
));
}
/// <summary>test DBRecordReader.</summary>
/// <remarks>test DBRecordReader. This reader should creates keys, values, know about position..
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestDBRecordReader()
{
JobConf job = Org.Mockito.Mockito.Mock <JobConf>();
DBConfiguration dbConfig = Org.Mockito.Mockito.Mock <DBConfiguration>();
string[] fields = new string[] { "field1", "filed2" };
DBInputFormat.DBRecordReader reader = new DBInputFormat.DBRecordReader(this, new
DBInputFormat.DBInputSplit(), typeof(DBInputFormat.NullDBWritable), job, DriverForTest
.GetConnection(), dbConfig, "condition", fields, "table");
LongWritable key = reader.CreateKey();
NUnit.Framework.Assert.AreEqual(0, key.Get());
DBWritable value = ((DBWritable)reader.CreateValue());
NUnit.Framework.Assert.AreEqual("org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable"
, value.GetType().FullName);
NUnit.Framework.Assert.AreEqual(0, reader.GetPos());
NUnit.Framework.Assert.IsFalse(reader.Next(key, value));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetOutput()
{
JobConf job = new JobConf();
DBOutputFormat.SetOutput(job, "hadoop_output", fieldNames);
DBConfiguration dbConf = new DBConfiguration(job);
string actual = format.ConstructQuery(dbConf.GetOutputTableName(), dbConf.GetOutputFieldNames
());
NUnit.Framework.Assert.AreEqual(expected, actual);
job = new JobConf();
dbConf = new DBConfiguration(job);
DBOutputFormat.SetOutput(job, "hadoop_output", nullFieldNames.Length);
NUnit.Framework.Assert.IsNull(dbConf.GetOutputFieldNames());
NUnit.Framework.Assert.AreEqual(nullFieldNames.Length, dbConf.GetOutputFieldCount
());
actual = format.ConstructQuery(dbConf.GetOutputTableName(), new string[dbConf.GetOutputFieldCount
()]);
NUnit.Framework.Assert.AreEqual(nullExpected, actual);
}
/// <summary>Initializes the reduce-part of the job with the appropriate output settings
/// </summary>
/// <param name="job">The job</param>
/// <param name="tableName">The table to insert data into</param>
/// <param name="fieldNames">The field names in the table.</param>
public static void SetOutput(JobConf job, string tableName, params string[] fieldNames
)
{
if (fieldNames.Length > 0 && fieldNames[0] != null)
{
DBConfiguration dbConf = SetOutput(job, tableName);
dbConf.SetOutputFieldNames(fieldNames);
}
else
{
if (fieldNames.Length > 0)
{
SetOutput(job, tableName, fieldNames.Length);
}
else
{
throw new ArgumentException("Field names must be greater than 0");
}
}
}
/// <param name="split">The InputSplit to read data for</param>
/// <exception cref="Java.Sql.SQLException"></exception>
protected internal DBRecordReader(DBInputFormat <T> _enclosing, DBInputFormat.DBInputSplit
split, Type inputClass, JobConf job, Connection conn, DBConfiguration dbConfig,
string cond, string[] fields, string table)
: base(split, inputClass, job, conn, dbConfig, cond, fields, table)
{
this._enclosing = _enclosing;
}
/// <summary>Initializes the reduce-part of the job with the appropriate output settings
/// </summary>
/// <param name="job">The job</param>
/// <param name="tableName">The table to insert data into</param>
/// <param name="fieldCount">the number of fields in the table.</param>
public static void SetOutput(JobConf job, string tableName, int fieldCount)
{
DBConfiguration dbConf = SetOutput(job, tableName);
dbConf.SetOutputFieldCount(fieldCount);
}