private void TestCommonSplitterTypes(DataDrivenDBInputFormat <DBInputFormat.NullDBWritable > format) { NUnit.Framework.Assert.AreEqual(typeof(BigDecimalSplitter), format.GetSplitter(Types .Decimal).GetType()); NUnit.Framework.Assert.AreEqual(typeof(BigDecimalSplitter), format.GetSplitter(Types .Numeric).GetType()); NUnit.Framework.Assert.AreEqual(typeof(BooleanSplitter), format.GetSplitter(Types .Boolean).GetType()); NUnit.Framework.Assert.AreEqual(typeof(BooleanSplitter), format.GetSplitter(Types .Bit).GetType()); NUnit.Framework.Assert.AreEqual(typeof(IntegerSplitter), format.GetSplitter(Types .Bigint).GetType()); NUnit.Framework.Assert.AreEqual(typeof(IntegerSplitter), format.GetSplitter(Types .Tinyint).GetType()); NUnit.Framework.Assert.AreEqual(typeof(IntegerSplitter), format.GetSplitter(Types .Smallint).GetType()); NUnit.Framework.Assert.AreEqual(typeof(IntegerSplitter), format.GetSplitter(Types .Integer).GetType()); NUnit.Framework.Assert.AreEqual(typeof(FloatSplitter), format.GetSplitter(Types.Double ).GetType()); NUnit.Framework.Assert.AreEqual(typeof(FloatSplitter), format.GetSplitter(Types.Real ).GetType()); NUnit.Framework.Assert.AreEqual(typeof(FloatSplitter), format.GetSplitter(Types.Float ).GetType()); NUnit.Framework.Assert.AreEqual(typeof(TextSplitter), format.GetSplitter(Types.Longvarchar ).GetType()); NUnit.Framework.Assert.AreEqual(typeof(TextSplitter), format.GetSplitter(Types.Char ).GetType()); NUnit.Framework.Assert.AreEqual(typeof(TextSplitter), format.GetSplitter(Types.Varchar ).GetType()); // if unknown data type splitter is null NUnit.Framework.Assert.IsNull(format.GetSplitter(Types.Binary)); }
/// <exception cref="System.Exception"/> public virtual void TestDataDrivenDBInputFormat() { JobContext jobContext = Org.Mockito.Mockito.Mock <JobContext>(); Configuration configuration = new Configuration(); configuration.SetInt(MRJobConfig.NumMaps, 1); Org.Mockito.Mockito.When(jobContext.GetConfiguration()).ThenReturn(configuration); DataDrivenDBInputFormat <DBInputFormat.NullDBWritable> format = new DataDrivenDBInputFormat <DBInputFormat.NullDBWritable>(); IList <InputSplit> splits = format.GetSplits(jobContext); NUnit.Framework.Assert.AreEqual(1, splits.Count); DataDrivenDBInputFormat.DataDrivenDBInputSplit split = (DataDrivenDBInputFormat.DataDrivenDBInputSplit )splits[0]; NUnit.Framework.Assert.AreEqual("1=1", split.GetLowerClause()); NUnit.Framework.Assert.AreEqual("1=1", split.GetUpperClause()); // 2 configuration.SetInt(MRJobConfig.NumMaps, 2); DataDrivenDBInputFormat.SetBoundingQuery(configuration, "query"); NUnit.Framework.Assert.AreEqual("query", configuration.Get(DBConfiguration.InputBoundingQuery )); Job job = Org.Mockito.Mockito.Mock <Job>(); Org.Mockito.Mockito.When(job.GetConfiguration()).ThenReturn(configuration); DataDrivenDBInputFormat.SetInput(job, typeof(DBInputFormat.NullDBWritable), "query" , "Bounding Query"); NUnit.Framework.Assert.AreEqual("Bounding Query", configuration.Get(DBConfiguration .InputBoundingQuery)); }
/// <summary>test splitters from DataDrivenDBInputFormat.</summary> /// <remarks> /// test splitters from DataDrivenDBInputFormat. For different data types may /// be different splitter /// </remarks> public virtual void TestDataDrivenDBInputFormatSplitter() { DataDrivenDBInputFormat <DBInputFormat.NullDBWritable> format = new DataDrivenDBInputFormat <DBInputFormat.NullDBWritable>(); TestCommonSplitterTypes(format); NUnit.Framework.Assert.AreEqual(typeof(DateSplitter), format.GetSplitter(Types.Timestamp ).GetType()); NUnit.Framework.Assert.AreEqual(typeof(DateSplitter), format.GetSplitter(Types.Date ).GetType()); NUnit.Framework.Assert.AreEqual(typeof(DateSplitter), format.GetSplitter(Types.Time ).GetType()); }
/// <exception cref="System.Exception"/> public virtual void TestDateSplits() { Statement s = connection.CreateStatement(); string DateTable = "datetable"; string Col = "foo"; try { // delete the table if it already exists. s.ExecuteUpdate("DROP TABLE " + DateTable); } catch (SQLException) { } // Create the table. s.ExecuteUpdate("CREATE TABLE " + DateTable + "(" + Col + " DATE)"); s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-04-01')"); s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-04-02')"); s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2010-05-01')"); s.ExecuteUpdate("INSERT INTO " + DateTable + " VALUES('2011-04-01')"); // commit this tx. connection.Commit(); Configuration conf = new Configuration(); conf.Set("fs.defaultFS", "file:///"); FileSystem fs = FileSystem.GetLocal(conf); fs.Delete(new Path(OutDir), true); // now do a dd import Job job = Job.GetInstance(conf); job.SetMapperClass(typeof(TestDataDrivenDBInputFormat.ValMapper)); job.SetReducerClass(typeof(Reducer)); job.SetMapOutputKeyClass(typeof(TestDataDrivenDBInputFormat.DateCol)); job.SetMapOutputValueClass(typeof(NullWritable)); job.SetOutputKeyClass(typeof(TestDataDrivenDBInputFormat.DateCol)); job.SetOutputValueClass(typeof(NullWritable)); job.SetNumReduceTasks(1); job.GetConfiguration().SetInt("mapreduce.map.tasks", 2); FileOutputFormat.SetOutputPath(job, new Path(OutDir)); DBConfiguration.ConfigureDB(job.GetConfiguration(), DriverClass, DbUrl, null, null ); DataDrivenDBInputFormat.SetInput(job, typeof(TestDataDrivenDBInputFormat.DateCol) , DateTable, null, Col, Col); bool ret = job.WaitForCompletion(true); NUnit.Framework.Assert.IsTrue("job failed", ret); // Check to see that we imported as much as we thought we did. NUnit.Framework.Assert.AreEqual("Did not get all the records", 4, job.GetCounters ().FindCounter(TaskCounter.ReduceOutputRecords).GetValue()); }