/// <exception cref="Java.Sql.SQLException"/> public OracleDBRecordReader(DBInputFormat.DBInputSplit split, Type inputClass, Configuration conf, Connection conn, DBConfiguration dbConfig, string cond, string[] fields, string table) : base(split, inputClass, conf, conn, dbConfig, cond, fields, table) { SetSessionTimeZone(conf, conn); }
/// <exception cref="Java.Sql.SQLException"/> public OracleDataDrivenDBRecordReader(DBInputFormat.DBInputSplit split, Type inputClass , Configuration conf, Connection conn, DBConfiguration dbConfig, string cond, string [] fields, string table) : base(split, inputClass, conf, conn, dbConfig, cond, fields, table, "ORACLE") { // Must initialize the tz used by the connection for Oracle. OracleDBRecordReader.SetSessionTimeZone(conf, conn); }
/// <param name="split">The InputSplit to read data for</param> /// <exception cref="Java.Sql.SQLException"></exception> public DataDrivenDBRecordReader(DBInputFormat.DBInputSplit split, Type inputClass , Configuration conf, Connection conn, DBConfiguration dbConfig, string cond, string [] fields, string table, string dbProduct) : base(split, inputClass, conf, conn, dbConfig, cond, fields, table) { // database manufacturer string. this.dbProductName = dbProduct; }
/// <summary>Returns the query for selecting the records from an Oracle DB.</summary> protected internal override string GetSelectQuery() { StringBuilder query = new StringBuilder(); DBConfiguration dbConf = GetDBConf(); string conditions = GetConditions(); string tableName = GetTableName(); string[] fieldNames = GetFieldNames(); // Oracle-specific codepath to use rownum instead of LIMIT/OFFSET. if (dbConf.GetInputQuery() == null) { query.Append("SELECT "); for (int i = 0; i < fieldNames.Length; i++) { query.Append(fieldNames[i]); if (i != fieldNames.Length - 1) { query.Append(", "); } } query.Append(" FROM ").Append(tableName); if (conditions != null && conditions.Length > 0) { query.Append(" WHERE ").Append(conditions); } string orderBy = dbConf.GetInputOrderBy(); if (orderBy != null && orderBy.Length > 0) { query.Append(" ORDER BY ").Append(orderBy); } } else { //PREBUILT QUERY query.Append(dbConf.GetInputQuery()); } try { DBInputFormat.DBInputSplit split = GetSplit(); if (split.GetLength() > 0) { string querystring = query.ToString(); query = new StringBuilder(); query.Append("SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( "); query.Append(querystring); query.Append(" ) a WHERE rownum <= ").Append(split.GetEnd()); query.Append(" ) WHERE dbif_rno > ").Append(split.GetStart()); } } catch (IOException) { } // ignore, will not throw. return(query.ToString()); }
/// <param name="split">The InputSplit to read data for</param> /// <exception cref="Java.Sql.SQLException"></exception> public DBRecordReader(DBInputFormat.DBInputSplit split, Type inputClass, Configuration conf, Connection conn, DBConfiguration dbConfig, string cond, string[] fields, string table) { this.inputClass = inputClass; this.split = split; this.conf = conf; this.connection = conn; this.dbConf = dbConfig; this.conditions = cond; this.fieldNames = fields; this.tableName = table; }
/// <summary>test generate sql script for OracleDBRecordReader.</summary> /// <exception cref="System.Exception"/> public virtual void TestOracleDBRecordReader() { DBInputFormat.DBInputSplit splitter = new DBInputFormat.DBInputSplit(1, 10); Configuration configuration = new Configuration(); Connection connect = DriverForTest.GetConnection(); DBConfiguration dbConfiguration = new DBConfiguration(configuration); dbConfiguration.SetInputOrderBy("Order"); string[] fields = new string[] { "f1", "f2" }; OracleDBRecordReader <DBInputFormat.NullDBWritable> recorder = new OracleDBRecordReader <DBInputFormat.NullDBWritable>(splitter, typeof(DBInputFormat.NullDBWritable), configuration , connect, dbConfiguration, "condition", fields, "table"); NUnit.Framework.Assert.AreEqual("SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 10 ) WHERE dbif_rno > 1" , recorder.GetSelectQuery()); }
/// <exception cref="Java.Sql.SQLException"/> public MySQLDBRecordReader(DBInputFormat.DBInputSplit split, Type inputClass, Configuration conf, Connection conn, DBConfiguration dbConfig, string cond, string[] fields, string table) : base(split, inputClass, conf, conn, dbConfig, cond, fields, table) { }
/// <summary> /// <inheritDoc/> /// /// </summary> /// <exception cref="System.IO.IOException"/> public override IList <InputSplit> GetSplits(JobContext job) { ResultSet results = null; Statement statement = null; try { statement = connection.CreateStatement(); results = statement.ExecuteQuery(GetCountQuery()); results.Next(); long count = results.GetLong(1); int chunks = job.GetConfiguration().GetInt(MRJobConfig.NumMaps, 1); long chunkSize = (count / chunks); results.Close(); statement.Close(); IList <InputSplit> splits = new AList <InputSplit>(); // Split the rows into n-number of chunks and adjust the last chunk // accordingly for (int i = 0; i < chunks; i++) { DBInputFormat.DBInputSplit split; if ((i + 1) == chunks) { split = new DBInputFormat.DBInputSplit(i * chunkSize, count); } else { split = new DBInputFormat.DBInputSplit(i * chunkSize, (i * chunkSize) + chunkSize ); } splits.AddItem(split); } connection.Commit(); return(splits); } catch (SQLException e) { throw new IOException("Got SQLException", e); } finally { try { if (results != null) { results.Close(); } } catch (SQLException) { } try { if (statement != null) { statement.Close(); } } catch (SQLException) { } CloseConnection(); } }
/// <exception cref="System.IO.IOException"/> protected internal virtual RecordReader <LongWritable, T> CreateDBRecordReader(DBInputFormat.DBInputSplit split, Configuration conf) { Type inputClass = (Type)(dbConf.GetInputClass()); try { // use database product name to determine appropriate record reader. if (dbProductName.StartsWith("ORACLE")) { // use Oracle-specific db reader. return(new OracleDBRecordReader <T>(split, inputClass, conf, CreateConnection(), GetDBConf (), conditions, fieldNames, tableName)); } else { if (dbProductName.StartsWith("MYSQL")) { // use MySQL-specific db reader. return(new MySQLDBRecordReader <T>(split, inputClass, conf, CreateConnection(), GetDBConf (), conditions, fieldNames, tableName)); } else { // Generic reader. return(new DBRecordReader <T>(split, inputClass, conf, CreateConnection(), GetDBConf (), conditions, fieldNames, tableName)); } } } catch (SQLException ex) { throw new IOException(ex.Message); } }
/// <exception cref="System.IO.IOException"/> protected internal override RecordReader <LongWritable, T> CreateDBRecordReader(DBInputFormat.DBInputSplit split, Configuration conf) { DBConfiguration dbConf = GetDBConf(); Type inputClass = (Type)(dbConf.GetInputClass()); try { // Use Oracle-specific db reader return(new OracleDataDrivenDBRecordReader <T>(split, inputClass, conf, CreateConnection (), dbConf, dbConf.GetInputConditions(), dbConf.GetInputFieldNames(), dbConf.GetInputTableName ())); } catch (SQLException ex) { throw new IOException(ex.Message); } }
/// <exception cref="System.IO.IOException"/> protected internal override RecordReader <LongWritable, T> CreateDBRecordReader(DBInputFormat.DBInputSplit split, Configuration conf) { DBConfiguration dbConf = GetDBConf(); Type inputClass = (Type)(dbConf.GetInputClass()); string dbProductName = GetDBProductName(); Log.Debug("Creating db record reader for db product: " + dbProductName); try { // use database product name to determine appropriate record reader. if (dbProductName.StartsWith("MYSQL")) { // use MySQL-specific db reader. return(new MySQLDataDrivenDBRecordReader <T>(split, inputClass, conf, CreateConnection (), dbConf, dbConf.GetInputConditions(), dbConf.GetInputFieldNames(), dbConf.GetInputTableName ())); } else { // Generic reader. return(new DataDrivenDBRecordReader <T>(split, inputClass, conf, CreateConnection( ), dbConf, dbConf.GetInputConditions(), dbConf.GetInputFieldNames(), dbConf.GetInputTableName (), dbProductName)); } } catch (SQLException ex) { throw new IOException(ex.Message); } }