/// <summary>
/// Create a KineticaType object based on an existing table in the database.
/// </summary>
/// <param name="kinetica"></param>
/// <param name="tableName"></param>
/// <returns></returns>
public static KineticaType fromTable(Kinetica kinetica, string tableName)
{
var response = kinetica.showTable(tableName);
var typeIdCount = response.type_ids.Count;
if (typeIdCount == 0)
{
throw new KineticaException("Table " + tableName + " does not exist.");
}
string typeID = response.type_ids[0];
if (typeIdCount > 1)
{
for (int i = 1; i < typeIdCount; ++i)
{
if (response.type_ids[i] != typeID)
{
throw new KineticaException("Table " + tableName + " is not homogeneous.");
}
}
}
return(new KineticaType(response.type_labels[0], response.type_schemas[0], response.properties[0], typeID));
}
/// <summary>
/// Create a KineticaType object based on an existing type in the database.
/// </summary>
/// <param name="kinetica"></param>
/// <param name="typeId"></param>
/// <returns></returns>
public static KineticaType fromTypeID(Kinetica kinetica, string typeId)
{
var response = kinetica.showTypes(typeId, "");
if (response.type_ids.Count < 1)
{
throw new KineticaException("Type " + typeId + " does not exist.");
}
return(new KineticaType(response.labels[0], response.type_schemas[0], response.properties[0]));
}
} // end saveSourceType
/// <summary>
/// Given a handle to the server, creates a type in the database based
/// on this data type.
/// </summary>
/// <param name="kinetica">The handle to the database server.</param>
/// <returns>The ID with which the database identifies this type.</returns>
public string create(Kinetica kinetica)
{
// Save the association between this KineticaType's source and itself in the Kinetica object
// for future reference (it helps with encoding and decoding records)
if (this.m_data.sourceType != null)
{
kinetica.SetKineticaSourceClassToTypeMapping(this.m_data.sourceType, this);
}
// Register the type with Kinetica
CreateTypeResponse response = kinetica.createType(m_data.schemaString, m_data.label, m_properties);
return(response.type_id);
} // end create()
/// <summary>
///
/// </summary>
/// <param name="kdb"></param>
/// <param name="table_name"></param>
/// <param name="batch_size"></param>
/// <param name="ktype"></param>
/// <param name="options"></param>
/// <param name="workers"></param>
public KineticaIngestor(Kinetica kdb, string table_name,
int batch_size, KineticaType ktype,
Dictionary <string, string> options = null,
Utils.WorkerList workers = null)
{
this.kineticaDB = kdb;
this.table_name = table_name;
this.ktype = ktype;
// Validate and save the batch size
if (batch_size < 1)
{
throw new KineticaException($"Batch size must be greater than one; given {batch_size}.");
}
this.batch_size = batch_size;
// Save the options (make it read-only if it exists)
if (options != null)
{
this.options = options;
//this.options = options.ToImmutableDictionary<string, string>();
}
else
{
this.options = null;
}
// Set up the primary and shard key builders
// -----------------------------------------
this.primary_key_builder = new Utils.RecordKeyBuilder <T>(true, this.ktype);
this.shard_key_builder = new Utils.RecordKeyBuilder <T>(false, this.ktype);
// Based on the Java implementation
if (this.primary_key_builder.hasKey())
{ // There is a primary key for the given T
// Now check if there is a distinct shard key
if (!this.shard_key_builder.hasKey() ||
this.shard_key_builder.hasSameKey(this.primary_key_builder))
{
this.shard_key_builder = this.primary_key_builder; // no distinct shard key
}
}
else // there is no primary key for the given T
{
this.primary_key_builder = null;
// Check if there is shard key for T
if (!this.shard_key_builder.hasKey())
{
this.shard_key_builder = null;
}
} // done setting up the key builders
// Set up the worker queues
// -------------------------
// Do we update records if there are matching primary keys in the
// database already?
bool update_on_existing_pk = ((options != null) &&
options.ContainsKey(InsertRecordsRequest <T> .Options.UPDATE_ON_EXISTING_PK) &&
options[InsertRecordsRequest <T> .Options.UPDATE_ON_EXISTING_PK].Equals(InsertRecordsRequest <T> .Options.TRUE));
// Do T type records have a primary key?
bool has_primary_key = (this.primary_key_builder != null);
this.worker_queues = new List <Utils.WorkerQueue <T> >();
try
{
// If no workers are given, try to get them from Kinetica
if ((workers == null) || (workers.Count == 0))
{
workers = new Utils.WorkerList(kdb);
}
// If we end up with multiple workers, either given by the
// user or obtained from Kinetica, then use those
if ((workers != null) && (workers.Count > 0))
{
// Add worker queues per worker
foreach (System.Uri worker_url in workers)
{
string insert_records_worker_url_str = (worker_url.ToString() + "insert/records");
System.Uri url = new System.Uri(insert_records_worker_url_str);
Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url, batch_size,
has_primary_key,
update_on_existing_pk);
this.worker_queues.Add(worker_queue);
}
// Get the worker rank information from Kinetica
this.routing_table = kdb.adminShowShards().rank;
// Check that enough worker URLs are specified
for (int i = 0; i < routing_table.Count; ++i)
{
if (this.routing_table[i] > this.worker_queues.Count)
{
throw new KineticaException("Not enough worker URLs specified.");
}
}
}
else // multihead-ingest is NOT turned on; use the regular Kinetica IP address
{
string insert_records_url_str = (kdb.URL.ToString() + "insert/records");
System.Uri url = new System.Uri(insert_records_url_str);
Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url, batch_size, has_primary_key, update_on_existing_pk);
this.worker_queues.Add(worker_queue);
this.routing_table = null;
}
}
catch (Exception ex)
{
throw new KineticaException(ex.ToString());
}
// Create the random number generator
this.random = new Random((int)DateTime.Now.Ticks);
} // end constructor KineticaIngestor
/// <summary>
/// Create a RecordRetriever object with the given parameters.
/// </summary>
/// <param name="kdb"></param>
/// <param name="table_name"></param>
/// <param name="ktype"></param>
/// <param name="workers"></param>
public RecordRetriever(Kinetica kdb, string table_name,
KineticaType ktype,
Utils.WorkerList workers = null)
{
this.kineticaDB = kdb;
this.table_name = table_name;
this.ktype = ktype;
// Set up the shard key builder
// ----------------------------
this.shard_key_builder = new Utils.RecordKeyBuilder <T>(false, this.ktype);
// Check if there is shard key for T
if (!this.shard_key_builder.hasKey())
{
this.shard_key_builder = null;
}
// Set up the worker queues
// -------------------------
this.worker_queues = new List <Utils.WorkerQueue <T> >();
try
{
// If no workers are given, try to get them from Kinetica
if ((workers == null) || (workers.Count == 0))
{
workers = new Utils.WorkerList(kdb);
}
// If we end up with multiple workers, either given by the
// user or obtained from Kinetica, then use those
if ((workers != null) && (workers.Count > 0))
{
// Add worker queues per worker
foreach (System.Uri worker_url in workers)
{
string get_records_worker_url_str = (worker_url.ToString() + "get/records");
System.Uri url = new System.Uri(get_records_worker_url_str);
Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url);
this.worker_queues.Add(worker_queue);
}
// Get the worker rank information from Kinetica
this.routing_table = kdb.adminShowShards().rank;
// Check that enough worker URLs are specified
for (int i = 0; i < routing_table.Count; ++i)
{
if (this.routing_table[i] > this.worker_queues.Count)
{
throw new KineticaException("Not enough worker URLs specified.");
}
}
}
else // multihead-ingest is NOT turned on; use the regular Kinetica IP address
{
string get_records_url_str = (kdb.URL.ToString() + "get/records");
System.Uri url = new System.Uri(get_records_url_str);
Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url);
this.worker_queues.Add(worker_queue);
this.routing_table = null;
}
}
catch (Exception ex)
{
throw new KineticaException(ex.ToString());
}
// Create the random number generator
this.random = new Random((int)DateTime.Now.Ticks);
} // end constructor RecordRetriever