/// <summary> /// Creates a <see cref="WorkerList"/> object and automatically populates it with the /// worker URLs from GPUdb to support multi-head ingest. ( If the /// specified GPUdb instance has multi-head ingest disabled, the worker /// list will be empty and multi-head ingest will not be used.) Note that /// in some cases, workers may be configured to use more than one IP /// address, not all of which may be accessible to the client; this /// constructor uses the first IP returned by the server for each worker. /// </summary> /// /// <param name="db">The <see cref="Kinetica"/> instance from which to /// obtain the worker URLs.</param> /// <param name="ip_regex">Optional IP regex to match.</param> public WorkerList(Kinetica db, Regex ip_regex = null) { // Get the system properties from the database server IDictionary <string, string> system_properties = db.showSystemProperties().property_map; // Find out if multi-head ingest is turned on or not string multi_head_ingestion_param; system_properties.TryGetValue(ShowSystemPropertiesResponse.PropertyMap.CONF_ENABLE_WORKER_HTTP_SERVERS, out multi_head_ingestion_param); if (multi_head_ingestion_param == null) { throw new KineticaException("Missing value for " + ShowSystemPropertiesResponse.PropertyMap.CONF_ENABLE_WORKER_HTTP_SERVERS); } bool is_multi_head_ingest_enabled = multi_head_ingestion_param.Equals(ShowSystemPropertiesResponse.PropertyMap.TRUE); // Nothing to do if multi-head ingestion is disabled if (!is_multi_head_ingest_enabled) { return; } // Multi-head ingestion IS enabled; find the worker URLs; then // add them to the worker url vector. // ----------------------------------------------------------- string worker_urls_str; system_properties.TryGetValue("conf.worker_http_server_urls", out worker_urls_str); if (worker_urls_str.Length > 0) // found some URLs { // Parse the URLs // -------------- // Split the strings string[] worker_url_lists = worker_urls_str.Split(';'); // Ignoring the very first rank (rank-0), add all matching // URLs (if any regex is given), or just add the first one for (int i = 1; i < worker_url_lists.Length; ++i) { string url_list = worker_url_lists[i]; // Need to split each of the URL lists on a comma string[] urls = url_list.Split(','); bool matching_url_found = false; // Find at least one URL to work with foreach (string url_str in urls) { // Try to create the URL try { // If a regular expression is given, then see if this one is a match if (ip_regex != null) { matching_url_found = ip_regex.IsMatch(url_str); } else // no regex given, so take the first URL encountered for this worker { matching_url_found = true; } if (matching_url_found) { Uri url = new Uri(url_str); // Add the URL to this WorkerList this.Add(url); break; // don't keep trying to match URLs in this group } // end inner if } // end try catch (Exception ex) { throw new KineticaException(ex.Message); } } // end inner foreach if (!matching_url_found) { throw new KineticaException($"No matching URL found for worker #{i}."); } } // end outer for } else // construct the URLs from IP addresses and ports { // Get the worker IPs and ports string worker_ips_str, worker_ports_str; system_properties.TryGetValue(ShowSystemPropertiesResponse.PropertyMap.CONF_WORKER_HTTP_SERVER_IPS, out worker_ips_str); system_properties.TryGetValue(ShowSystemPropertiesResponse.PropertyMap.CONF_WORKER_HTTP_SERVER_PORTS, out worker_ports_str); // Check that we got them if (worker_ips_str.Length == 0) { throw new KineticaException("Missing value for " + ShowSystemPropertiesResponse.PropertyMap.CONF_WORKER_HTTP_SERVER_IPS); } if (worker_ports_str.Length == 0) { throw new KineticaException("Missing value for " + ShowSystemPropertiesResponse.PropertyMap.CONF_WORKER_HTTP_SERVER_PORTS); } // Parse the IPs and the ports // --------------------------- // Split the strings string[] worker_ip_lists = worker_ips_str.Split(';'); string[] worker_ports = worker_ports_str.Split(';'); // Check that there are the same number of IPs and ports supplied if (worker_ip_lists.Length != worker_ports.Length) { throw new KineticaException("Inconsistent number of values for " + ShowSystemPropertiesResponse.PropertyMap.CONF_WORKER_HTTP_SERVER_IPS + " and " + ShowSystemPropertiesResponse.PropertyMap.CONF_WORKER_HTTP_SERVER_PORTS); } // Create the URLs using the IPs and the ports, but // ignore the very first rank (rank-0) for (int i = 1; i < worker_ip_lists.Length; ++i) { string ip_list = worker_ip_lists[i]; // Need to split each of the IP lists on a comma string[] ips = ip_list.Split(','); bool matching_ip_found = false; // Find at least one IP to work with foreach (string ip in ips) { // Try to create the URL try { // If a regular expression is given, then see if this one is a match if (ip_regex != null) { matching_ip_found = ip_regex.IsMatch(ip); } else // no regex given, so take the first IP encountered for this worker { matching_ip_found = true; } if (matching_ip_found) { UriBuilder uri_builder = new UriBuilder(db.URL.Scheme, ip, Int32.Parse(worker_ports[i])); Uri url = uri_builder.Uri; // Add the URL to this WorkerList this.Add(url); break; // don't keep trying to match IPs in this group } // end inner if } // end try catch (Exception ex) { throw new KineticaException(ex.Message); } } // end inner foreach if (!matching_ip_found) { throw new KineticaException($"No matching IP found for worker #{i}."); } } // end outer for } // end if-else // Check that this list is not empty if (this.Count == 0) { throw new KineticaException("No worker HTTP servers found."); } } // end constructor
} // end run_series_example() /// <summary> /// Shows an example of how to use the multi-head ingestion feature /// of Kinetica. /// </summary> /// <param name="server_url">The URL for the Kinetica server.</param> private static void run_multihead_ingest_example(string server_url) { // Establish a connection with Kinetica Kinetica kdb = new Kinetica(server_url); Console.WriteLine("\n\n"); Console.WriteLine("Example showcasing multihead ingestion(one shard key, two primary keys)"); Console.WriteLine("======================================================================="); Console.WriteLine(); // Create a type for our record_type_1 class // Add some interesting properties for some of the data types IDictionary <string, IList <string> > column_properties = new Dictionary <string, IList <string> >(); // Add a primary key List <string> A_props = new List <string>(); A_props.Add(ColumnProperty.PRIMARY_KEY); column_properties.Add("A", A_props); // And a shard key (must be part of the primary keys, if specified--which we have) List <string> ts_props = new List <string>(); ts_props.Add(ColumnProperty.PRIMARY_KEY); ts_props.Add(ColumnProperty.SHARD_KEY); ts_props.Add(ColumnProperty.TIMESTAMP); column_properties.Add("TIMESTAMP", ts_props); // Create the KineticaType object which facilitates creating types in the database KineticaType type1 = KineticaType.fromClass(typeof(record_type_1), column_properties); // Create the type in the database string type_id = type1.create(kdb); string table_name = "csharp_example_table_01"; // Clear any previously made table with the same name Console.WriteLine("Clearing any existing table named '{0}'", table_name); try { kdb.clearTable(table_name, null); } catch (Exception ex) { /* I don't care if the table doesn't already exists! */ } Console.WriteLine(); // Create a table of the given type Console.WriteLine($"Creating table named '{table_name}'"); kdb.createTable(table_name, type_id); Console.WriteLine(); // Create the ingestor (we're not giving any worker IP addresses; the ingestor class will figure it // out by itself) int batch_size = 100; KineticaIngestor <record_type_1> ingestor = new KineticaIngestor <record_type_1>(kdb, table_name, batch_size, type1); // Generate data to be inserted int num_records = batch_size * 5; List <record_type_1> records = new List <record_type_1>(); Random rng = new Random(); double null_probability = 0.2; for (int i = 0; i < num_records; ++i) { // Restricting string length to 256 int max_str_len = rng.Next(0, 256); record_type_1 record = new record_type_1() { A = rng.Next(), B = rng.Next(), C = System.IO.Path.GetRandomFileName().Truncate(max_str_len), D = System.IO.Path.GetRandomFileName().Truncate(max_str_len), E = (float)(rng.NextDouble() * rng.Next()), F = (rng.NextDouble() < null_probability) ? null : ( double? )(rng.NextDouble() * rng.Next()), TIMESTAMP = (long)rng.Next(-306102240, 293795424) * rng.Next(100000) }; records.Add(record); } // end for loop Console.WriteLine($"Generated {num_records} records."); // Insert the records into the ingestor Console.WriteLine($"Inserting {num_records} records..."); ingestor.insert(records); // Flush the ingestor (which actually inserts the records) Console.WriteLine("\nFlushing any remaining records."); ingestor.flush(); Console.WriteLine(); Console.WriteLine(); } // end run_multihead_ingest_example()
/// <summary> /// Example with a record type with nullable columns, primary keys and shard keys. /// </summary> /// <param name="server_url">The URL for the Kinetica server.</param> private static void run_example(string server_url) { // Establish a connection with Kinetica Kinetica kdb = new Kinetica(server_url); Console.WriteLine("Example with a Record Type with Nullable Columns, Primary Keys and Shard Keys"); Console.WriteLine("============================================================================="); Console.WriteLine(); string table_name = "csharp_example_table"; // Create a type for our record_type_1 class Console.WriteLine("Creating the type in kinetica..."); // Add some interesting properties for some of the data types IDictionary <string, IList <string> > column_properties = new Dictionary <string, IList <string> >(); // Make a string char4 (and nullable) List <string> D_props = new List <string>(); D_props.Add(ColumnProperty.CHAR4); D_props.Add(ColumnProperty.NULLABLE); column_properties.Add("D", D_props); // Let's try another nullable column List <string> E_props = new List <string>(); E_props.Add(ColumnProperty.NULLABLE); column_properties.Add("E", E_props); // And two primary keys (one nullable) List <string> A_props = new List <string>(); A_props.Add(ColumnProperty.PRIMARY_KEY); column_properties.Add("A", A_props); List <string> B_props = new List <string>(); B_props.Add(ColumnProperty.PRIMARY_KEY); column_properties.Add("B", B_props); // And a shard key (must be one of the primary keys, if specified--which we have) B_props.Add(ColumnProperty.SHARD_KEY); // Create the KineticaType object which facilitates creating types in the database KineticaType type1 = KineticaType.fromClass(typeof(record_type_1), column_properties); // Create the type in the database string type_id = type1.create(kdb); Console.WriteLine("ID of the created type: " + type_id); Console.WriteLine(); // Show the type information (fetched from Kinetica) Console.WriteLine("Fetching the newly created type information..."); ShowTypesResponse rsp = kdb.showTypes(type_id, ""); Console.WriteLine("Type properties: "); foreach (var x in rsp.properties[0]) { Console.WriteLine(x.Key + ": " + String.Join(",", x.Value)); } Console.WriteLine(); // Clear any previously made table with the same name Console.WriteLine("Clearing any existing table named '{0}'", table_name); try { kdb.clearTable(table_name, null); } catch (Exception ex) { /* I don't care if the table doesn't already exists! */ } Console.WriteLine(); // Create a table of the given type Console.WriteLine("Creating table named '{0}'", table_name); kdb.createTable(table_name, type_id); Console.WriteLine(); // Call /show/table on the table just created Dictionary <string, string> show_table_options = new Dictionary <string, string>(); show_table_options.Add(ShowTableRequest.Options.GET_SIZES, ShowTableRequest.Options.TRUE); Console.WriteLine("Calling ShowTable on '{0}'", table_name); var response2 = kdb.showTable(table_name, show_table_options); Console.WriteLine("Total size: {0}", response2.total_size); Console.WriteLine("sizes: {0}", string.Join(", ", response2.sizes)); Console.WriteLine(); // Create some data to be added to the table List <record_type_1> newData = new List <record_type_1>() { new record_type_1() { A = 99, B = 11, C = "T0_lksdjfokdj92", D = "D01", E = 2.34F, F = null, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_1() { A = 2, B = 3, C = "T1_asdfghjkl", D = null, E = 5.67F, F = null, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_1() { A = 99, B = 999, C = "T2_oierlwk", D = "D244", E = -45.1F, F = 9899.1, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds } }; // Insert the data into the table Console.WriteLine("Inserting some data in '{0}'", table_name); var insertResponse = kdb.insertRecords(table_name, newData); Console.WriteLine("Inserted {0} records", insertResponse.count_inserted + insertResponse.count_updated); Console.WriteLine(); // Call /show/table on the table after adding data Console.WriteLine("Calling ShowTable on '{0}' after adding data", table_name); var rsp2 = kdb.showTable(table_name, show_table_options); Console.WriteLine("Total size: {0}", rsp2.total_size); Console.WriteLine("sizes: {0}", string.Join(", ", rsp2.sizes)); Console.WriteLine(); // Fetch the data back out of the DB using /get/records Console.WriteLine("Getting records out"); var getRecordsResponse = kdb.getRecords <record_type_1>(table_name, 0, 100); Console.WriteLine("GetRecords got {0} records", getRecordsResponse.data.Count); foreach (var r in getRecordsResponse.data) { Console.WriteLine("\t" + r.ToString()); } Console.WriteLine(); // Do a filter operation Console.WriteLine("Filtering data"); string view_name_1 = "csharp_view_01"; string filter_expression = "E > 0"; var filterResponse = kdb.filter(table_name, view_name_1, filter_expression); Console.WriteLine("Filtered {0} records", filterResponse.count); Console.WriteLine(); // Fetch the data from the filtered view using /get/records/fromcollection Console.WriteLine("Getting records out (using /get/records/fromcollection) from view {0}", view_name_1); IDictionary <string, string> getrecords_fc_options = new Dictionary <string, string>(); getrecords_fc_options.Add("return_record_ids", "true"); var request = new GetRecordsFromCollectionRequest(view_name_1, 0, 100, GetRecordsFromCollectionRequest.Encoding.BINARY, getrecords_fc_options); var getRecordsFromCollectionResponse = kdb.getRecordsFromCollection <record_type_1>(request); Console.WriteLine("GetRecordsFromCollection got {0} records", getRecordsFromCollectionResponse.data.Count); foreach (var r in getRecordsFromCollectionResponse.data) { Console.WriteLine("\t" + r.ToString()); } Console.WriteLine(); // Do an /aggregate/groupby operation on the data Console.WriteLine("Performing a group-by aggregate operation"); IList <string> column_names = new List <string>(); column_names.Add("A"); column_names.Add("D"); var agbResponse = kdb.aggregateGroupBy(table_name, column_names, 0, 100); Console.WriteLine("Group by got {0} records", agbResponse.total_number_of_records); // Print the decoded data out to the console ((List <KineticaRecord>)(agbResponse.data)).ForEach(r => Console.WriteLine(r.ContentsToString())); Console.WriteLine(); // Do an /aggregate/unique operation on column F string col_name = "F"; Console.WriteLine($"Performing a unique aggregate operation on column {col_name}"); var uniqueResponse = kdb.aggregateUnique(table_name, col_name, 0, 100); // Print the decoded data out to the console ((List <KineticaRecord>)(uniqueResponse.data)).ForEach(r => Console.WriteLine(r.ContentsToString())); Console.WriteLine(); // Fetch the data back out of the DB using /get/records/bycolumn Console.WriteLine("Getting records out (using /get/records/bycolumn)"); IList <string> col_names = new List <string>(); col_names.Add("B"); col_names.Add("C"); col_names.Add("E"); var getRecordsByColumnResponse = kdb.getRecordsByColumn(table_name, col_names, 0, 100); Console.WriteLine("GetRecordsByColumn got {0} records", getRecordsByColumnResponse.data.Count); foreach (var r in getRecordsByColumnResponse.data) { Console.WriteLine("\t" + r.ContentsToString()); } Console.WriteLine(); Console.WriteLine(); } // end run_example
} // end run_example /// <summary> /// Example showcasing a record with a series type column. /// </summary> /// <param name="server_url">The URL for the Kinetica server.</param> private static void run_series_example(string server_url) { // Establish a connection with Kinetica Kinetica kdb = new Kinetica(server_url); Console.WriteLine("Example showcasing a record with a series type column"); Console.WriteLine("======================================================"); Console.WriteLine(); // Create the series type record in Kinetica KineticaType type_series = KineticaType.fromClass(typeof(record_type_series)); string series_type_id = type_series.create(kdb); Console.WriteLine("ID of the created series type: " + series_type_id); Console.WriteLine(); // Clear any previously made table with the same name string table_name_series = "csharp_example_series_table"; Console.WriteLine("Clearing any existing table named '{0}'", table_name_series); try { kdb.clearTable(table_name_series, null); } catch (Exception ex) { /* I don't care if the table doesn't already exists! */ } Console.WriteLine(); // Create a table of the given type Console.WriteLine("Creating table named '{0}'", table_name_series); kdb.createTable(table_name_series, series_type_id); Console.WriteLine(); // Create some data to be added to the table string series_1 = "series_1"; string series_2 = "series_2"; List <record_type_series> series_data = new List <record_type_series>() { // Five series points moving horizontally new record_type_series() { x = 30, y = 40, TRACKID = series_1, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_series() { x = 35, y = 40, TRACKID = series_1, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_series() { x = 40, y = 40, TRACKID = series_1, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_series() { x = 45, y = 40, TRACKID = series_1, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_series() { x = 50, y = 40, TRACKID = series_1, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, // Five series points moving vertically new record_type_series() { x = -30, y = -40, TRACKID = series_2, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_series() { x = -30, y = -45, TRACKID = series_2, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_series() { x = -30, y = -50, TRACKID = series_2, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_series() { x = -30, y = -55, TRACKID = series_2, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, new record_type_series() { x = -30, y = -60, TRACKID = series_2, TIMESTAMP = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalMilliseconds }, }; // Insert the data into the table Console.WriteLine("Inserting some data in '{0}'", table_name_series); var insertResponse2 = kdb.insertRecords(table_name_series, series_data); Console.WriteLine("Inserted {0} records", insertResponse2.count_inserted + insertResponse2.count_updated); Console.WriteLine(); // Fetch the data back out of the DB using /get/records Console.WriteLine("Getting records out from {0}", table_name_series); var getRecordsResponse2 = kdb.getRecords <record_type_series>(table_name_series); Console.WriteLine("GetRecords got {0} records", getRecordsResponse2.data.Count); foreach (var r in getRecordsResponse2.data) { Console.WriteLine("\t" + r.ToString()); } Console.WriteLine(); // Filter the series data such that only a few series point from only one of the series // are in the view Console.WriteLine("Filtering data from {0} so that only some points from {1} to make it", table_name_series, series_1); string view_name_2 = "csharp_view_02"; // Only the first, second, and third records from the first series should survive this filter // (for track/series type records, we're filtering the "line segments", so records just outside the // filtered box will also be captured) var filterByBoxResp = kdb.filterByBox(table_name_series, view_name_2, "x", 33, 37, "y", 35, 45); Console.WriteLine("Filtered {0} records", filterByBoxResp.count); Console.WriteLine(); // Show the filtered objects var getRecordsResponse3 = kdb.getRecords <record_type_series>(view_name_2); Console.WriteLine("GetRecords got {0} records from {1}", getRecordsResponse3.data.Count, view_name_2); foreach (var r in getRecordsResponse3.data) { Console.WriteLine("\t" + r.ToString()); } Console.WriteLine(); // Extract the entire series from the source table (based on which ones are in the view) Console.WriteLine("Getting records belonging to one series out from table '{0}' using the partial series in view '{1}'", table_name_series, view_name_2); var getRecordsBySeriesResp = kdb.getRecordsBySeries <record_type_series>(view_name_2, table_name_series); Console.WriteLine("GetRecordsBySeries got {0} list of records (should get one list of five records in it)", getRecordsBySeriesResp.data.Count); foreach (var r_list in getRecordsBySeriesResp.data) { foreach (var r in r_list) { Console.WriteLine("\t" + r.ToString()); } } Console.WriteLine(); Console.WriteLine(); } // end run_series_example()