Пример #1
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="kdb"></param>
        /// <param name="table_name"></param>
        /// <param name="batch_size"></param>
        /// <param name="ktype"></param>
        /// <param name="options"></param>
        /// <param name="workers"></param>
        public KineticaIngestor(Kinetica kdb, string table_name,
                                int batch_size, KineticaType ktype,
                                Dictionary <string, string> options = null,
                                Utils.WorkerList workers            = null)
        {
            this.kineticaDB = kdb;
            this.table_name = table_name;
            this.ktype      = ktype;

            // Validate and save the batch size
            if (batch_size < 1)
            {
                throw new KineticaException($"Batch size must be greater than one; given {batch_size}.");
            }
            this.batch_size = batch_size;

            // Save the options (make it read-only if it exists)
            if (options != null)
            {
                this.options = options;
                //this.options = options.ToImmutableDictionary<string, string>();
            }
            else
            {
                this.options = null;
            }

            // Set up the primary and shard key builders
            // -----------------------------------------
            this.primary_key_builder = new Utils.RecordKeyBuilder <T>(true, this.ktype);
            this.shard_key_builder   = new Utils.RecordKeyBuilder <T>(false, this.ktype);

            // Based on the Java implementation
            if (this.primary_key_builder.hasKey())
            {   // There is a primary key for the given T
                // Now check if there is a distinct shard key
                if (!this.shard_key_builder.hasKey() ||
                    this.shard_key_builder.hasSameKey(this.primary_key_builder))
                {
                    this.shard_key_builder = this.primary_key_builder; // no distinct shard key
                }
            }
            else  // there is no primary key for the given T
            {
                this.primary_key_builder = null;

                // Check if there is shard key for T
                if (!this.shard_key_builder.hasKey())
                {
                    this.shard_key_builder = null;
                }
            }  // done setting up the key builders


            // Set up the worker queues
            // -------------------------
            // Do we update records if there are matching primary keys in the
            // database already?
            bool update_on_existing_pk = ((options != null) &&
                                          options.ContainsKey(InsertRecordsRequest <T> .Options.UPDATE_ON_EXISTING_PK) &&
                                          options[InsertRecordsRequest <T> .Options.UPDATE_ON_EXISTING_PK].Equals(InsertRecordsRequest <T> .Options.TRUE));
            // Do T type records have a primary key?
            bool has_primary_key = (this.primary_key_builder != null);

            this.worker_queues = new List <Utils.WorkerQueue <T> >();
            try
            {
                // If no workers are given, try to get them from Kinetica
                if ((workers == null) || (workers.Count == 0))
                {
                    workers = new Utils.WorkerList(kdb);
                }

                // If we end up with multiple workers, either given by the
                // user or obtained from Kinetica, then use those
                if ((workers != null) && (workers.Count > 0))
                {
                    // Add worker queues per worker
                    foreach (System.Uri worker_url in workers)
                    {
                        string                insert_records_worker_url_str = (worker_url.ToString() + "insert/records");
                        System.Uri            url          = new System.Uri(insert_records_worker_url_str);
                        Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url, batch_size,
                                                                                       has_primary_key,
                                                                                       update_on_existing_pk);
                        this.worker_queues.Add(worker_queue);
                    }

                    // Get the worker rank information from Kinetica
                    this.routing_table = kdb.adminShowShards().rank;
                    // Check that enough worker URLs are specified
                    for (int i = 0; i < routing_table.Count; ++i)
                    {
                        if (this.routing_table[i] > this.worker_queues.Count)
                        {
                            throw new KineticaException("Not enough worker URLs specified.");
                        }
                    }
                }
                else // multihead-ingest is NOT turned on; use the regular Kinetica IP address
                {
                    string                insert_records_url_str = (kdb.URL.ToString() + "insert/records");
                    System.Uri            url          = new System.Uri(insert_records_url_str);
                    Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url, batch_size, has_primary_key, update_on_existing_pk);
                    this.worker_queues.Add(worker_queue);
                    this.routing_table = null;
                }
            }
            catch (Exception ex)
            {
                throw new KineticaException(ex.ToString());
            }

            // Create the random number generator
            this.random = new Random((int)DateTime.Now.Ticks);
        }   // end constructor KineticaIngestor
        /// <summary>
        /// Create a RecordRetriever object with the given parameters.
        /// </summary>
        /// <param name="kdb"></param>
        /// <param name="table_name"></param>
        /// <param name="ktype"></param>
        /// <param name="workers"></param>
        public RecordRetriever(Kinetica kdb, string table_name,
                               KineticaType ktype,
                               Utils.WorkerList workers = null)
        {
            this.kineticaDB = kdb;
            this.table_name = table_name;
            this.ktype      = ktype;

            // Set up the shard key builder
            // ----------------------------
            this.shard_key_builder = new Utils.RecordKeyBuilder <T>(false, this.ktype);
            // Check if there is shard key for T
            if (!this.shard_key_builder.hasKey())
            {
                this.shard_key_builder = null;
            }


            // Set up the worker queues
            // -------------------------
            this.worker_queues = new List <Utils.WorkerQueue <T> >();
            try
            {
                // If no workers are given, try to get them from Kinetica
                if ((workers == null) || (workers.Count == 0))
                {
                    workers = new Utils.WorkerList(kdb);
                }

                // If we end up with multiple workers, either given by the
                // user or obtained from Kinetica, then use those
                if ((workers != null) && (workers.Count > 0))
                {
                    // Add worker queues per worker
                    foreach (System.Uri worker_url in workers)
                    {
                        string                get_records_worker_url_str = (worker_url.ToString() + "get/records");
                        System.Uri            url          = new System.Uri(get_records_worker_url_str);
                        Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url);
                        this.worker_queues.Add(worker_queue);
                    }

                    // Get the worker rank information from Kinetica
                    this.routing_table = kdb.adminShowShards().rank;
                    // Check that enough worker URLs are specified
                    for (int i = 0; i < routing_table.Count; ++i)
                    {
                        if (this.routing_table[i] > this.worker_queues.Count)
                        {
                            throw new KineticaException("Not enough worker URLs specified.");
                        }
                    }
                }
                else // multihead-ingest is NOT turned on; use the regular Kinetica IP address
                {
                    string                get_records_url_str = (kdb.URL.ToString() + "get/records");
                    System.Uri            url          = new System.Uri(get_records_url_str);
                    Utils.WorkerQueue <T> worker_queue = new Utils.WorkerQueue <T>(url);
                    this.worker_queues.Add(worker_queue);
                    this.routing_table = null;
                }
            }
            catch (Exception ex)
            {
                throw new KineticaException(ex.ToString());
            }

            // Create the random number generator
            this.random = new Random((int)DateTime.Now.Ticks);
        }   // end constructor RecordRetriever