/// <exception cref="System.Exception"/> public virtual void TestDataDrivenDBInputFormat() { JobContext jobContext = Org.Mockito.Mockito.Mock <JobContext>(); Configuration configuration = new Configuration(); configuration.SetInt(MRJobConfig.NumMaps, 1); Org.Mockito.Mockito.When(jobContext.GetConfiguration()).ThenReturn(configuration); DataDrivenDBInputFormat <DBInputFormat.NullDBWritable> format = new DataDrivenDBInputFormat <DBInputFormat.NullDBWritable>(); IList <InputSplit> splits = format.GetSplits(jobContext); NUnit.Framework.Assert.AreEqual(1, splits.Count); DataDrivenDBInputFormat.DataDrivenDBInputSplit split = (DataDrivenDBInputFormat.DataDrivenDBInputSplit )splits[0]; NUnit.Framework.Assert.AreEqual("1=1", split.GetLowerClause()); NUnit.Framework.Assert.AreEqual("1=1", split.GetUpperClause()); // 2 configuration.SetInt(MRJobConfig.NumMaps, 2); DataDrivenDBInputFormat.SetBoundingQuery(configuration, "query"); NUnit.Framework.Assert.AreEqual("query", configuration.Get(DBConfiguration.InputBoundingQuery )); Job job = Org.Mockito.Mockito.Mock <Job>(); Org.Mockito.Mockito.When(job.GetConfiguration()).ThenReturn(configuration); DataDrivenDBInputFormat.SetInput(job, typeof(DBInputFormat.NullDBWritable), "query" , "Bounding Query"); NUnit.Framework.Assert.AreEqual("Bounding Query", configuration.Get(DBConfiguration .InputBoundingQuery)); }
/// <exception cref="System.Exception"/> public virtual void TestDateSplits() { Statement s = connection.CreateStatement(); string DateTable = "datetable"; string Col = "foo"; try { // delete the table if it already exists. s.ExecuteUpdate("DROP TABLE " + DateTable); } catch (SQLException) { } // Create the table. s.ExecuteUpdate("CREATE TABLE " + DateTable + "(" + Col + " DATE)"); s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-04-01')"); s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-04-02')"); s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-05-01')"); s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2011-04-01')"); // commit this tx. connection.Commit(); Configuration conf = new Configuration(); conf.Set("fs.defaultFS", "file:///"); FileSystem fs = FileSystem.GetLocal(conf); fs.Delete(new Path(OutDir), true); // now do a dd import Job job = Job.GetInstance(conf); job.SetMapperClass(typeof(TestDataDrivenDBInputFormat.ValMapper)); job.SetReducerClass(typeof(Reducer)); job.SetMapOutputKeyClass(typeof(TestDataDrivenDBInputFormat.DateCol)); job.SetMapOutputValueClass(typeof(NullWritable)); job.SetOutputKeyClass(typeof(TestDataDrivenDBInputFormat.DateCol)); job.SetOutputValueClass(typeof(NullWritable)); job.SetNumReduceTasks(1); job.GetConfiguration().SetInt("mapreduce.map.tasks", 2); FileOutputFormat.SetOutputPath(job, new Path(OutDir)); DBConfiguration.ConfigureDB(job.GetConfiguration(), DriverClass, DbUrl, null, null ); DataDrivenDBInputFormat.SetInput(job, typeof(TestDataDrivenDBInputFormat.DateCol) , DateTable, null, Col, Col); bool ret = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("job failed", ret); // Check to see that we imported as much as we thought we did. NUnit.Framework.Assert.AreEqual("Did not get all the records", 4, job.GetCounters ().FindCounter(TaskCounter.ReduceOutputRecords).GetValue()); }