private static DBConfiguration SetOutput(JobConf job, string tableName) { job.SetOutputFormat(typeof(DBOutputFormat)); job.SetReduceSpeculativeExecution(false); DBConfiguration dbConf = new DBConfiguration(job); dbConf.SetOutputTableName(tableName); return(dbConf); }
/// <summary>Initializes the map-part of the job with the appropriate input settings. /// </summary> /// <param name="job">The job</param> /// <param name="inputClass"> /// the class object implementing DBWritable, which is the /// Java object holding tuple fields. /// </param> /// <param name="inputQuery"> /// the input query to select fields. Example : /// "SELECT f1, f2, f3 FROM Mytable ORDER BY f1" /// </param> /// <param name="inputCountQuery"> /// the input query that returns the number of records in /// the table. /// Example : "SELECT COUNT(f1) FROM Mytable" /// </param> /// <seealso cref="DBInputFormat{T}.SetInput(Org.Apache.Hadoop.Mapred.JobConf, System.Type{T}, string, string, string, string[]) /// "/> public static void SetInput(JobConf job, Type inputClass, string inputQuery, string inputCountQuery) { job.SetInputFormat(typeof(DBInputFormat)); DBConfiguration dbConf = new DBConfiguration(job); dbConf.SetInputClass(inputClass); dbConf.SetInputQuery(inputQuery); dbConf.SetInputCountQuery(inputCountQuery); }
/// <summary>Initializes the map-part of the job with the appropriate input settings. /// </summary> /// <param name="job">The job</param> /// <param name="inputClass"> /// the class object implementing DBWritable, which is the /// Java object holding tuple fields. /// </param> /// <param name="tableName">The table to read data from</param> /// <param name="conditions"> /// The condition which to select data with, eg. '(updated > /// 20070101 AND length > 0)' /// </param> /// <param name="orderBy">the fieldNames in the orderBy clause.</param> /// <param name="fieldNames">The field names in the table</param> /// <seealso cref="DBInputFormat{T}.SetInput(Org.Apache.Hadoop.Mapred.JobConf, System.Type{T}, string, string) /// "/> public static void SetInput(JobConf job, Type inputClass, string tableName, string conditions, string orderBy, params string[] fieldNames) { job.SetInputFormat(typeof(DBInputFormat)); DBConfiguration dbConf = new DBConfiguration(job); dbConf.SetInputClass(inputClass); dbConf.SetInputTableName(tableName); dbConf.SetInputFieldNames(fieldNames); dbConf.SetInputConditions(conditions); dbConf.SetInputOrderBy(orderBy); }
/// <summary>test configuration for db.</summary> /// <remarks>test configuration for db. should works DBConfiguration.* parameters.</remarks> public virtual void TestSetInput() { JobConf configuration = new JobConf(); string[] fieldNames = new string[] { "field1", "field2" }; DBInputFormat.SetInput(configuration, typeof(DBInputFormat.NullDBWritable), "table" , "conditions", "orderBy", fieldNames); NUnit.Framework.Assert.AreEqual("org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable" , configuration.GetClass(DBConfiguration.InputClassProperty, null).FullName); NUnit.Framework.Assert.AreEqual("table", configuration.Get(DBConfiguration.InputTableNameProperty , null)); string[] fields = configuration.GetStrings(DBConfiguration.InputFieldNamesProperty ); NUnit.Framework.Assert.AreEqual("field1", fields[0]); NUnit.Framework.Assert.AreEqual("field2", fields[1]); NUnit.Framework.Assert.AreEqual("conditions", configuration.Get(DBConfiguration.InputConditionsProperty , null)); NUnit.Framework.Assert.AreEqual("orderBy", configuration.Get(DBConfiguration.InputOrderByProperty , null)); configuration = new JobConf(); DBInputFormat.SetInput(configuration, typeof(DBInputFormat.NullDBWritable), "query" , "countQuery"); NUnit.Framework.Assert.AreEqual("query", configuration.Get(DBConfiguration.InputQuery , null)); NUnit.Framework.Assert.AreEqual("countQuery", configuration.Get(DBConfiguration.InputCountQuery , null)); JobConf jConfiguration = new JobConf(); DBConfiguration.ConfigureDB(jConfiguration, "driverClass", "dbUrl", "user", "password" ); NUnit.Framework.Assert.AreEqual("driverClass", jConfiguration.Get(DBConfiguration .DriverClassProperty)); NUnit.Framework.Assert.AreEqual("dbUrl", jConfiguration.Get(DBConfiguration.UrlProperty )); NUnit.Framework.Assert.AreEqual("user", jConfiguration.Get(DBConfiguration.UsernameProperty )); NUnit.Framework.Assert.AreEqual("password", jConfiguration.Get(DBConfiguration.PasswordProperty )); jConfiguration = new JobConf(); DBConfiguration.ConfigureDB(jConfiguration, "driverClass", "dbUrl"); NUnit.Framework.Assert.AreEqual("driverClass", jConfiguration.Get(DBConfiguration .DriverClassProperty)); NUnit.Framework.Assert.AreEqual("dbUrl", jConfiguration.Get(DBConfiguration.UrlProperty )); NUnit.Framework.Assert.IsNull(jConfiguration.Get(DBConfiguration.UsernameProperty )); NUnit.Framework.Assert.IsNull(jConfiguration.Get(DBConfiguration.PasswordProperty )); }
/// <summary>test DBRecordReader.</summary> /// <remarks>test DBRecordReader. This reader should creates keys, values, know about position.. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDBRecordReader() { JobConf job = Org.Mockito.Mockito.Mock <JobConf>(); DBConfiguration dbConfig = Org.Mockito.Mockito.Mock <DBConfiguration>(); string[] fields = new string[] { "field1", "filed2" }; DBInputFormat.DBRecordReader reader = new DBInputFormat.DBRecordReader(this, new DBInputFormat.DBInputSplit(), typeof(DBInputFormat.NullDBWritable), job, DriverForTest .GetConnection(), dbConfig, "condition", fields, "table"); LongWritable key = reader.CreateKey(); NUnit.Framework.Assert.AreEqual(0, key.Get()); DBWritable value = ((DBWritable)reader.CreateValue()); NUnit.Framework.Assert.AreEqual("org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable" , value.GetType().FullName); NUnit.Framework.Assert.AreEqual(0, reader.GetPos()); NUnit.Framework.Assert.IsFalse(reader.Next(key, value)); }
/// <exception cref="System.IO.IOException"/> public virtual void TestSetOutput() { JobConf job = new JobConf(); DBOutputFormat.SetOutput(job, "hadoop_output", fieldNames); DBConfiguration dbConf = new DBConfiguration(job); string actual = format.ConstructQuery(dbConf.GetOutputTableName(), dbConf.GetOutputFieldNames ()); NUnit.Framework.Assert.AreEqual(expected, actual); job = new JobConf(); dbConf = new DBConfiguration(job); DBOutputFormat.SetOutput(job, "hadoop_output", nullFieldNames.Length); NUnit.Framework.Assert.IsNull(dbConf.GetOutputFieldNames()); NUnit.Framework.Assert.AreEqual(nullFieldNames.Length, dbConf.GetOutputFieldCount ()); actual = format.ConstructQuery(dbConf.GetOutputTableName(), new string[dbConf.GetOutputFieldCount ()]); NUnit.Framework.Assert.AreEqual(nullExpected, actual); }
/// <summary>Initializes the reduce-part of the job with the appropriate output settings /// </summary> /// <param name="job">The job</param> /// <param name="tableName">The table to insert data into</param> /// <param name="fieldNames">The field names in the table.</param> public static void SetOutput(JobConf job, string tableName, params string[] fieldNames ) { if (fieldNames.Length > 0 && fieldNames[0] != null) { DBConfiguration dbConf = SetOutput(job, tableName); dbConf.SetOutputFieldNames(fieldNames); } else { if (fieldNames.Length > 0) { SetOutput(job, tableName, fieldNames.Length); } else { throw new ArgumentException("Field names must be greater than 0"); } } }
/// <param name="split">The InputSplit to read data for</param> /// <exception cref="Java.Sql.SQLException"></exception> protected internal DBRecordReader(DBInputFormat <T> _enclosing, DBInputFormat.DBInputSplit split, Type inputClass, JobConf job, Connection conn, DBConfiguration dbConfig, string cond, string[] fields, string table) : base(split, inputClass, job, conn, dbConfig, cond, fields, table) { this._enclosing = _enclosing; }
/// <summary>Initializes the reduce-part of the job with the appropriate output settings /// </summary> /// <param name="job">The job</param> /// <param name="tableName">The table to insert data into</param> /// <param name="fieldCount">the number of fields in the table.</param> public static void SetOutput(JobConf job, string tableName, int fieldCount) { DBConfiguration dbConf = SetOutput(job, tableName); dbConf.SetOutputFieldCount(fieldCount); }