/// <summary>
///
/// </summary>
/// <param name="kdb"></param>
/// <param name="table_name"></param>
/// <param name="batch_size"></param>
/// <param name="ktype"></param>
/// <param name="options"></param>
/// <param name="workers"></param>
public KineticaIngestor(Kinetica kdb, string table_name,
int batch_size, KineticaType ktype,
Dictionary <string, string> options = null,
Utils.WorkerList workers = null)
{
this.kineticaDB = kdb;
this.table_name = table_name;
this.ktype = ktype;
// Validate and save the batch size
if (batch_size < 1)
{
throw new KineticaException($"Batch size must be greater than one; given {batch_size}.");
}
this.batch_size = batch_size;
// Save the options (make it read-only if it exists)
if (options != null)
{
this.options = options;
//this.options = options.ToImmutableDictionary<string, string>();
}
else
{
this.options = null;
}
// Set up the primary and shard key builders
// -----------------------------------------
this.primary_key_builder = new Utils.RecordKeyBuilder <T>(true, this.ktype);
this.shard_key_builder = new Utils.RecordKeyBuilder <T>(false, this.ktype);
// Based on the Java implementation
if (this.primary_key_builder.hasKey())
{ // There is a primary key for the given T
// Now check if there is a distinct shard key
if (!this.shard_key_builder.hasKey() ||
this.shard_key_builder.hasSameKey(this.primary_key_builder))
{
this.shard_key_builder = this.primary_key_builder; // no distinct shard key
}
}
else // there is no primary key for the given T
{
this.primary_key_builder = null;
// Check if there is shard key for T
if (!this.shard_key_builder.hasKey())
{
this.shard_key_builder = null;
}
} // done setting up the key builders
// Set up the worker queues
// -------------------------
// Do we update records if there are matching primary keys in the
// database already?
bool update_on_existing_pk = ((options != null) &&
options.ContainsKey(InsertRecordsRequest <T> .Options.UPDATE_ON_EXISTING_PK) &&
options[InsertRecordsRequest <T> .Options.UPDATE_ON_EXISTING_PK].Equals(InsertRecordsRequest <T> .Options.TRUE));
// Do T type records have a primary key?
bool has_primary_key = (this.primary_key_builder != null);
this.worker_queues = new List <Utils.WorkerQueue <T> >();
try
{
// If no workers are given, try to get them from Kinetica
if ((workers == null) || (workers.Count == 0))
{
workers = new Utils.WorkerList(kdb);
}
// If we end up with multiple workers, either given by the
// user or obtained from Kinetica, then use those
if ((workers != null) && (workers.Count > 0))
{
// Add worker queues per worker
foreach (System.Uri worker_url in workers)
{
string insert_records_worker_url_str = (worker_url.ToString() + "insert/records");
System.Uri url = new System.Uri(insert_records_worker_url_str);
Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url, batch_size,
has_primary_key,
update_on_existing_pk);
this.worker_queues.Add(worker_queue);
}
// Get the worker rank information from Kinetica
this.routing_table = kdb.adminShowShards().rank;
// Check that enough worker URLs are specified
for (int i = 0; i < routing_table.Count; ++i)
{
if (this.routing_table[i] > this.worker_queues.Count)
{
throw new KineticaException("Not enough worker URLs specified.");
}
}
}
else // multihead-ingest is NOT turned on; use the regular Kinetica IP address
{
string insert_records_url_str = (kdb.URL.ToString() + "insert/records");
System.Uri url = new System.Uri(insert_records_url_str);
Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url, batch_size, has_primary_key, update_on_existing_pk);
this.worker_queues.Add(worker_queue);
this.routing_table = null;
}
}
catch (Exception ex)
{
throw new KineticaException(ex.ToString());
}
// Create the random number generator
this.random = new Random((int)DateTime.Now.Ticks);
} // end constructor KineticaIngestor
/// <summary>
/// Create a RecordRetriever object with the given parameters.
/// </summary>
/// <param name="kdb"></param>
/// <param name="table_name"></param>
/// <param name="ktype"></param>
/// <param name="workers"></param>
public RecordRetriever(Kinetica kdb, string table_name,
KineticaType ktype,
Utils.WorkerList workers = null)
{
this.kineticaDB = kdb;
this.table_name = table_name;
this.ktype = ktype;
// Set up the shard key builder
// ----------------------------
this.shard_key_builder = new Utils.RecordKeyBuilder <T>(false, this.ktype);
// Check if there is shard key for T
if (!this.shard_key_builder.hasKey())
{
this.shard_key_builder = null;
}
// Set up the worker queues
// -------------------------
this.worker_queues = new List <Utils.WorkerQueue <T> >();
try
{
// If no workers are given, try to get them from Kinetica
if ((workers == null) || (workers.Count == 0))
{
workers = new Utils.WorkerList(kdb);
}
// If we end up with multiple workers, either given by the
// user or obtained from Kinetica, then use those
if ((workers != null) && (workers.Count > 0))
{
// Add worker queues per worker
foreach (System.Uri worker_url in workers)
{
string get_records_worker_url_str = (worker_url.ToString() + "get/records");
System.Uri url = new System.Uri(get_records_worker_url_str);
Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url);
this.worker_queues.Add(worker_queue);
}
// Get the worker rank information from Kinetica
this.routing_table = kdb.adminShowShards().rank;
// Check that enough worker URLs are specified
for (int i = 0; i < routing_table.Count; ++i)
{
if (this.routing_table[i] > this.worker_queues.Count)
{
throw new KineticaException("Not enough worker URLs specified.");
}
}
}
else // multihead-ingest is NOT turned on; use the regular Kinetica IP address
{
string get_records_url_str = (kdb.URL.ToString() + "get/records");
System.Uri url = new System.Uri(get_records_url_str);
Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url);
this.worker_queues.Add(worker_queue);
this.routing_table = null;
}
}
catch (Exception ex)
{
throw new KineticaException(ex.ToString());
}
// Create the random number generator
this.random = new Random((int)DateTime.Now.Ticks);
} // end constructor RecordRetriever
