private void TestCommonSplitterTypes(DataDrivenDBInputFormat <DBInputFormat.NullDBWritable
> format)
{
NUnit.Framework.Assert.AreEqual(typeof(BigDecimalSplitter), format.GetSplitter(Types
.Decimal).GetType());
NUnit.Framework.Assert.AreEqual(typeof(BigDecimalSplitter), format.GetSplitter(Types
.Numeric).GetType());
NUnit.Framework.Assert.AreEqual(typeof(BooleanSplitter), format.GetSplitter(Types
.Boolean).GetType());
NUnit.Framework.Assert.AreEqual(typeof(BooleanSplitter), format.GetSplitter(Types
.Bit).GetType());
NUnit.Framework.Assert.AreEqual(typeof(IntegerSplitter), format.GetSplitter(Types
.Bigint).GetType());
NUnit.Framework.Assert.AreEqual(typeof(IntegerSplitter), format.GetSplitter(Types
.Tinyint).GetType());
NUnit.Framework.Assert.AreEqual(typeof(IntegerSplitter), format.GetSplitter(Types
.Smallint).GetType());
NUnit.Framework.Assert.AreEqual(typeof(IntegerSplitter), format.GetSplitter(Types
.Integer).GetType());
NUnit.Framework.Assert.AreEqual(typeof(FloatSplitter), format.GetSplitter(Types.Double
).GetType());
NUnit.Framework.Assert.AreEqual(typeof(FloatSplitter), format.GetSplitter(Types.Real
).GetType());
NUnit.Framework.Assert.AreEqual(typeof(FloatSplitter), format.GetSplitter(Types.Float
).GetType());
NUnit.Framework.Assert.AreEqual(typeof(TextSplitter), format.GetSplitter(Types.Longvarchar
).GetType());
NUnit.Framework.Assert.AreEqual(typeof(TextSplitter), format.GetSplitter(Types.Char
).GetType());
NUnit.Framework.Assert.AreEqual(typeof(TextSplitter), format.GetSplitter(Types.Varchar
).GetType());
// if unknown data type splitter is null
NUnit.Framework.Assert.IsNull(format.GetSplitter(Types.Binary));
}
/// <exception cref="System.Exception"/>
public virtual void TestDataDrivenDBInputFormat()
{
JobContext jobContext = Org.Mockito.Mockito.Mock <JobContext>();
Configuration configuration = new Configuration();
configuration.SetInt(MRJobConfig.NumMaps, 1);
Org.Mockito.Mockito.When(jobContext.GetConfiguration()).ThenReturn(configuration);
DataDrivenDBInputFormat <DBInputFormat.NullDBWritable> format = new DataDrivenDBInputFormat
<DBInputFormat.NullDBWritable>();
IList <InputSplit> splits = format.GetSplits(jobContext);
NUnit.Framework.Assert.AreEqual(1, splits.Count);
DataDrivenDBInputFormat.DataDrivenDBInputSplit split = (DataDrivenDBInputFormat.DataDrivenDBInputSplit
)splits[0];
NUnit.Framework.Assert.AreEqual("1=1", split.GetLowerClause());
NUnit.Framework.Assert.AreEqual("1=1", split.GetUpperClause());
// 2
configuration.SetInt(MRJobConfig.NumMaps, 2);
DataDrivenDBInputFormat.SetBoundingQuery(configuration, "query");
NUnit.Framework.Assert.AreEqual("query", configuration.Get(DBConfiguration.InputBoundingQuery
));
Job job = Org.Mockito.Mockito.Mock <Job>();
Org.Mockito.Mockito.When(job.GetConfiguration()).ThenReturn(configuration);
DataDrivenDBInputFormat.SetInput(job, typeof(DBInputFormat.NullDBWritable), "query"
, "Bounding Query");
NUnit.Framework.Assert.AreEqual("Bounding Query", configuration.Get(DBConfiguration
.InputBoundingQuery));
}
/// <summary>test splitters from DataDrivenDBInputFormat.</summary>
/// <remarks>
/// test splitters from DataDrivenDBInputFormat. For different data types may
/// be different splitter
/// </remarks>
public virtual void TestDataDrivenDBInputFormatSplitter()
{
DataDrivenDBInputFormat <DBInputFormat.NullDBWritable> format = new DataDrivenDBInputFormat
<DBInputFormat.NullDBWritable>();
TestCommonSplitterTypes(format);
NUnit.Framework.Assert.AreEqual(typeof(DateSplitter), format.GetSplitter(Types.Timestamp
).GetType());
NUnit.Framework.Assert.AreEqual(typeof(DateSplitter), format.GetSplitter(Types.Date
).GetType());
NUnit.Framework.Assert.AreEqual(typeof(DateSplitter), format.GetSplitter(Types.Time
).GetType());
}
/// <exception cref="System.Exception"/>
public virtual void TestDateSplits()
{
Statement s = connection.CreateStatement();
string DateTable = "datetable";
string Col = "foo";
try
{
// delete the table if it already exists.
s.ExecuteUpdate("DROP TABLE " + DateTable);
}
catch (SQLException)
{
}
// Create the table.
s.ExecuteUpdate("CREATE TABLE " + DateTable + "(" + Col + " DATE)");
s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-04-01')");
s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-04-02')");
s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-05-01')");
s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2011-04-01')");
// commit this tx.
connection.Commit();
Configuration conf = new Configuration();
conf.Set("fs.defaultFS", "file:///");
FileSystem fs = FileSystem.GetLocal(conf);
fs.Delete(new Path(OutDir), true);
// now do a dd import
Job job = Job.GetInstance(conf);
job.SetMapperClass(typeof(TestDataDrivenDBInputFormat.ValMapper));
job.SetReducerClass(typeof(Reducer));
job.SetMapOutputKeyClass(typeof(TestDataDrivenDBInputFormat.DateCol));
job.SetMapOutputValueClass(typeof(NullWritable));
job.SetOutputKeyClass(typeof(TestDataDrivenDBInputFormat.DateCol));
job.SetOutputValueClass(typeof(NullWritable));
job.SetNumReduceTasks(1);
job.GetConfiguration().SetInt("mapreduce.map.tasks", 2);
FileOutputFormat.SetOutputPath(job, new Path(OutDir));
DBConfiguration.ConfigureDB(job.GetConfiguration(), DriverClass, DbUrl, null, null
);
DataDrivenDBInputFormat.SetInput(job, typeof(TestDataDrivenDBInputFormat.DateCol)
, DateTable, null, Col, Col);
bool ret = job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("job failed", ret);
// Check to see that we imported as much as we thought we did.
NUnit.Framework.Assert.AreEqual("Did not get all the records", 4, job.GetCounters
().FindCounter(TaskCounter.ReduceOutputRecords).GetValue());
}