public AsyncQueryExecutor ( AsyncCluster cluster, QueryPolicy policy, RecordSequenceListener listener, Statement statement, Node[] nodes ) : base(cluster) { this.listener = listener; statement.Prepare(true); // Create commands. AsyncQuery[] tasks = new AsyncQuery[nodes.Length]; int count = 0; foreach (Node node in nodes) { tasks[count++] = new AsyncQuery(this, cluster, (AsyncNode)node, policy, listener, statement); } // Dispatch commands to nodes. if (policy.failOnClusterChange) { ExecuteValidate(tasks, policy.maxConcurrentNodes, statement.ns); } else { Execute(tasks, policy.maxConcurrentNodes); } }
public QueryRecordCommand(QueryPolicy policy, Statement statement, RecordSet recordSet, ulong clusterKey, bool first) : base(statement.ns, clusterKey, first) { this.policy = policy; this.statement = statement; this.recordSet = recordSet; }
public QueryAggregateExecutor(Cluster cluster, QueryPolicy policy, Statement statement) : base(cluster, policy, statement) { inputQueue = new BlockingCollection <object>(500); resultSet = new ResultSet(this, policy.recordQueueSize, cancel.Token); InitializeThreads(); }
public QueryAggregateExecutor(Cluster cluster, QueryPolicy policy, Statement statement) : base(cluster, policy, statement) { inputQueue = new BlockingCollection<object>(500); resultSet = new ResultSet(this, policy.recordQueueSize, cancel.Token); InitializeThreads(); }
/// <summary> /// Copy client policy from another client policy. /// </summary> public ClientPolicy(ClientPolicy other) { this.user = other.user; this.password = other.password; this.clusterName = other.clusterName; this.authMode = other.authMode; this.timeout = other.timeout; this.loginTimeout = other.loginTimeout; this.maxConnsPerNode = other.maxConnsPerNode; this.connPoolsPerNode = other.connPoolsPerNode; this.maxSocketIdle = other.maxSocketIdle; this.tendInterval = other.tendInterval; this.failIfNotConnected = other.failIfNotConnected; this.readPolicyDefault = new Policy(other.readPolicyDefault); this.writePolicyDefault = new WritePolicy(other.writePolicyDefault); this.scanPolicyDefault = new ScanPolicy(other.scanPolicyDefault); this.queryPolicyDefault = new QueryPolicy(other.queryPolicyDefault); this.batchPolicyDefault = new BatchPolicy(other.batchPolicyDefault); this.infoPolicyDefault = new InfoPolicy(other.infoPolicyDefault); this.tlsPolicy = (other.tlsPolicy != null) ? new TlsPolicy(other.tlsPolicy) : null; this.ipMap = other.ipMap; this.useServicesAlternate = other.useServicesAlternate; this.rackAware = other.rackAware; this.rackId = other.rackId; }
public AsyncQueryExecutor(AsyncCluster cluster, QueryPolicy policy, RecordSequenceListener listener, Statement statement) { this.listener = listener; statement.Prepare(true); Node[] nodes = cluster.Nodes; if (nodes.Length == 0) { throw new AerospikeException(ResultCode.SERVER_NOT_AVAILABLE, "Query failed because cluster is empty."); } // Create commands. AsyncQuery[] tasks = new AsyncQuery[nodes.Length]; int count = 0; bool hasClusterStable = true; foreach (Node node in nodes) { if (!node.HasClusterStable) { hasClusterStable = false; } tasks[count++] = new AsyncQuery(this, cluster, (AsyncNode)node, policy, listener, statement); } // Dispatch commands to nodes. if (policy.failOnClusterChange && hasClusterStable) { ExecuteValidate(cluster, tasks, policy.maxConcurrentNodes, statement.ns); } else { Execute(tasks, policy.maxConcurrentNodes); } }
public QueryExecutor(Cluster cluster, QueryPolicy policy, Statement statement, Node[] nodes) { this.cluster = cluster; this.policy = policy; this.statement = statement; this.nodes = nodes; this.threads = new QueryThread[nodes.Length]; this.cancel = new CancellationTokenSource(); // Initialize maximum number of nodes to query in parallel. this.maxConcurrentNodes = (policy.maxConcurrentNodes == 0 || policy.maxConcurrentNodes >= threads.Length) ? threads.Length : policy.maxConcurrentNodes; }
public QueryRecordCommand ( Cluster cluster, Node node, QueryPolicy policy, Statement statement, RecordSet recordSet, ulong clusterKey, bool first ) : base(cluster, policy, node, statement.ns, clusterKey, first) { this.statement = statement; this.recordSet = recordSet; }
public QueryAggregateCommand ( QueryPolicy policy, Statement statement, BlockingCollection <object> inputQueue, CancellationToken cancelToken, ulong clusterKey, bool first ) : base(statement.ns, clusterKey, first) { this.policy = policy; this.statement = statement; this.inputQueue = inputQueue; this.cancelToken = cancelToken; }
public QueryExecutor(Cluster cluster, QueryPolicy policy, Statement statement) { this.policy = policy; this.statement = statement; this.cancel = new CancellationTokenSource(); this.nodes = cluster.Nodes; if (this.nodes.Length == 0) { throw new AerospikeException(ResultCode.SERVER_NOT_AVAILABLE, "Query failed because cluster is empty."); } this.threads = new QueryThread[nodes.Length]; // Initialize maximum number of nodes to query in parallel. this.maxConcurrentNodes = (policy.maxConcurrentNodes == 0 || policy.maxConcurrentNodes >= threads.Length) ? threads.Length : policy.maxConcurrentNodes; }
public AsyncQueryPartitionExecutor ( AsyncCluster cluster, QueryPolicy policy, RecordSequenceListener listener, Statement statement, PartitionTracker tracker ) : base(cluster) { this.policy = policy; this.listener = listener; this.statement = statement; this.tracker = tracker; statement.returnData = true; tracker.SleepBetweenRetries = 0; QueryPartitions(); }
public QueryPartitionExecutor ( Cluster cluster, QueryPolicy policy, Statement statement, int nodeCapacity, PartitionTracker tracker ) { this.cluster = cluster; this.policy = policy; statement.returnData = true; this.statement = statement; this.threads = new List <QueryThread>(nodeCapacity); this.cancel = new CancellationTokenSource(); this.tracker = tracker; this.recordSet = new RecordSet(this, policy.recordQueueSize, cancel.Token); ThreadPool.QueueUserWorkItem(this.Run); }
public AsyncQueryExecutor(AsyncCluster cluster, QueryPolicy policy, RecordSequenceListener listener, Statement statement) { this.listener = listener; statement.Prepare(true); Node[] nodes = cluster.Nodes; if (nodes.Length == 0) { throw new AerospikeException(ResultCode.SERVER_NOT_AVAILABLE, "Query failed because cluster is empty."); } // Create commands. AsyncQuery[] tasks = new AsyncQuery[nodes.Length]; int count = 0; foreach (Node node in nodes) { tasks[count++] = new AsyncQuery(this, cluster, (AsyncNode)node, policy, listener, statement); } // Dispatch commands to nodes. Execute(tasks, policy.maxConcurrentNodes); }
protected internal void SetQuery(Policy policy, Statement statement, bool write) { byte[] functionArgBuffer = null; int fieldCount = 0; int filterSize = 0; int binNameSize = 0; Begin(); if (statement.ns != null) { dataOffset += ByteUtil.EstimateSizeUtf8(statement.ns) + FIELD_HEADER_SIZE; fieldCount++; } if (statement.indexName != null) { dataOffset += ByteUtil.EstimateSizeUtf8(statement.indexName) + FIELD_HEADER_SIZE; fieldCount++; } if (statement.setName != null) { dataOffset += ByteUtil.EstimateSizeUtf8(statement.setName) + FIELD_HEADER_SIZE; fieldCount++; } // Allocate space for TaskId field. dataOffset += 8 + FIELD_HEADER_SIZE; fieldCount++; if (statement.filter != null) { IndexCollectionType type = statement.filter.CollectionType; if (type != IndexCollectionType.DEFAULT) { dataOffset += FIELD_HEADER_SIZE + 1; fieldCount++; } dataOffset += FIELD_HEADER_SIZE; filterSize++; // num filters filterSize += statement.filter.EstimateSize(); dataOffset += filterSize; fieldCount++; // Query bin names are specified as a field (Scan bin names are specified later as operations) if (statement.binNames != null) { dataOffset += FIELD_HEADER_SIZE; binNameSize++; // num bin names foreach (string binName in statement.binNames) { binNameSize += ByteUtil.EstimateSizeUtf8(binName) + 1; } dataOffset += binNameSize; fieldCount++; } } else { // Calling query with no filters is more efficiently handled by a primary index scan. // Estimate scan options size. dataOffset += 2 + FIELD_HEADER_SIZE; fieldCount++; // Estimate scan timeout size. dataOffset += 4 + FIELD_HEADER_SIZE; fieldCount++; } PredExp[] predExp = statement.PredExp; int predSize = 0; if (predExp != null) { dataOffset += FIELD_HEADER_SIZE; predSize = PredExp.EstimateSize(predExp); dataOffset += predSize; fieldCount++; } if (statement.functionName != null) { dataOffset += FIELD_HEADER_SIZE + 1; // udf type dataOffset += ByteUtil.EstimateSizeUtf8(statement.packageName) + FIELD_HEADER_SIZE; dataOffset += ByteUtil.EstimateSizeUtf8(statement.functionName) + FIELD_HEADER_SIZE; if (statement.functionArgs.Length > 0) { functionArgBuffer = Packer.Pack(statement.functionArgs); } else { functionArgBuffer = new byte[0]; } dataOffset += FIELD_HEADER_SIZE + functionArgBuffer.Length; fieldCount += 4; } if (statement.filter == null) { if (statement.binNames != null) { foreach (string binName in statement.binNames) { EstimateOperationSize(binName); } } } SizeBuffer(); int operationCount = (statement.filter == null && statement.binNames != null) ? statement.binNames.Length : 0; if (write) { WriteHeader((WritePolicy)policy, Command.INFO1_READ, Command.INFO2_WRITE, fieldCount, operationCount); } else { QueryPolicy qp = (QueryPolicy)policy; int readAttr = qp.includeBinData ? Command.INFO1_READ : Command.INFO1_READ | Command.INFO1_NOBINDATA; WriteHeader(policy, readAttr, 0, fieldCount, operationCount); } if (statement.ns != null) { WriteField(statement.ns, FieldType.NAMESPACE); } if (statement.indexName != null) { WriteField(statement.indexName, FieldType.INDEX_NAME); } if (statement.setName != null) { WriteField(statement.setName, FieldType.TABLE); } // Write taskId field WriteFieldHeader(8, FieldType.TRAN_ID); ByteUtil.LongToBytes(statement.taskId, dataBuffer, dataOffset); dataOffset += 8; if (statement.filter != null) { IndexCollectionType type = statement.filter.CollectionType; if (type != IndexCollectionType.DEFAULT) { WriteFieldHeader(1, FieldType.INDEX_TYPE); dataBuffer[dataOffset++] = (byte)type; } WriteFieldHeader(filterSize, FieldType.INDEX_RANGE); dataBuffer[dataOffset++] = (byte)1; dataOffset = statement.filter.Write(dataBuffer, dataOffset); // Query bin names are specified as a field (Scan bin names are specified later as operations) if (statement.binNames != null) { WriteFieldHeader(binNameSize, FieldType.QUERY_BINLIST); dataBuffer[dataOffset++] = (byte)statement.binNames.Length; foreach (string binName in statement.binNames) { int len = ByteUtil.StringToUtf8(binName, dataBuffer, dataOffset + 1); dataBuffer[dataOffset] = (byte)len; dataOffset += len + 1; } } } else { // Calling query with no filters is more efficiently handled by a primary index scan. WriteFieldHeader(2, FieldType.SCAN_OPTIONS); byte priority = (byte)policy.priority; priority <<= 4; dataBuffer[dataOffset++] = priority; dataBuffer[dataOffset++] = (byte)100; // Write scan timeout WriteFieldHeader(4, FieldType.SCAN_TIMEOUT); dataOffset += ByteUtil.IntToBytes((uint)policy.socketTimeout, dataBuffer, dataOffset); } if (predExp != null) { WriteFieldHeader(predSize, FieldType.PREDEXP); dataOffset = PredExp.Write(predExp, dataBuffer, dataOffset); } if (statement.functionName != null) { WriteFieldHeader(1, FieldType.UDF_OP); dataBuffer[dataOffset++] = (statement.returnData) ? (byte)1 : (byte)2; WriteField(statement.packageName, FieldType.UDF_PACKAGE_NAME); WriteField(statement.functionName, FieldType.UDF_FUNCTION); WriteField(functionArgBuffer, FieldType.UDF_ARGLIST); } // Scan bin names are specified after all fields. if (statement.filter == null) { if (statement.binNames != null) { foreach (string binName in statement.binNames) { WriteOperation(binName, Operation.Type.READ); } } } End(); }
//-------------------------------------------------------- // Query functions (Supported by Aerospike 3 servers only) //-------------------------------------------------------- /// <summary> /// Execute query and call action for each record returned from server. /// </summary> /// <param name="policy">generic configuration parameters, pass in null for defaults</param> /// <param name="statement">database query command</param> /// <param name="action">action methods to be called for each record</param> /// <exception cref="AerospikeException">if query fails</exception> public void Query(QueryPolicy policy, Statement statement, Action<Key, Record> action) { using (RecordSet rs = Query(policy, statement)) { while (rs.Next()) { action(rs.Key, rs.Record); } } }
public PartitionTracker(QueryPolicy policy, Node[] nodes, PartitionFilter filter) : this((Policy)policy, nodes, filter) { this.maxRecords = policy.maxRecords; }
public PartitionTracker(QueryPolicy policy, Node[] nodes) : this((Policy)policy, nodes) { this.maxRecords = policy.maxRecords; }
/// <summary> /// Execute query and return record iterator. The query executor puts records on a queue in /// separate threads. The calling thread concurrently pops records off the queue through the /// record iterator. /// <para> /// This method is only supported by Aerospike 3 servers. /// </para> /// </summary> /// <param name="policy">generic configuration parameters, pass in null for defaults</param> /// <param name="statement">database query command</param> /// <exception cref="AerospikeException">if query fails</exception> public RecordSet Query(QueryPolicy policy, Statement statement) { if (policy == null) { policy = queryPolicyDefault; } QueryRecordExecutor executor = new QueryRecordExecutor(cluster, policy, statement); executor.Execute(); return executor.RecordSet; }
/// <summary> /// Execute query, apply statement's aggregation function, and return result iterator. /// The aggregation function should be located in a Lua script file that can be found from the /// "LuaConfig.PackagePath" paths static variable. The default package path is "udf/?.lua" /// where "?" is the packageName. /// <para> /// The query executor puts results on a queue in separate threads. The calling thread /// concurrently pops results off the queue through the ResultSet iterator. /// The aggregation function is called on both server and client (final reduce). /// Therefore, the Lua script file must also reside on both server and client. /// </para> /// <para> /// This method is only supported by Aerospike 3 servers. /// </para> /// </summary> /// <param name="policy">generic configuration parameters, pass in null for defaults</param> /// <param name="statement">database query command</param> /// <param name="packageName">server package where user defined function resides</param> /// <param name="functionName">aggregation function name</param> /// <param name="functionArgs">arguments to pass to function name, if any</param> /// <exception cref="AerospikeException">if query fails</exception> public ResultSet QueryAggregate(QueryPolicy policy, Statement statement, string packageName, string functionName, params Value[] functionArgs) { statement.SetAggregateFunction(packageName, functionName, functionArgs); return QueryAggregate(policy, statement); }
/// <summary> /// Execute query, apply statement's aggregation function, call action for each aggregation /// object returned from server. /// </summary> /// <param name="policy">generic configuration parameters, pass in null for defaults</param> /// <param name="statement">database query command with aggregate functions already initialized by SetAggregateFunction()</param> /// <param name="action">action methods to be called for each aggregation object</param> /// <exception cref="AerospikeException">if query fails</exception> public void QueryAggregate(QueryPolicy policy, Statement statement, Action<Object> action) { using (ResultSet rs = QueryAggregate(policy, statement)) { while (rs.Next()) { action(rs.Object); } } }
//------------------------------------------------------- // Query Operations //------------------------------------------------------- /// <summary> /// Asynchronously execute query on all server nodes. The query policy's /// <code>maxConcurrentNodes</code> dictate how many nodes can be queried in parallel. /// The default is to query all nodes in parallel. /// <para> /// This method schedules the node's query commands with channel selectors and returns. /// Selector threads will process the commands and send the results to the listener. /// </para> /// </summary> /// <param name="policy">query configuration parameters, pass in null for defaults</param> /// <param name="listener">where to send results</param> /// <param name="statement">database query command parameters</param> /// <exception cref="AerospikeException">if query fails</exception> public void Query(QueryPolicy policy, RecordSequenceListener listener, Statement statement) { if (policy == null) { policy = queryPolicyDefault; } new AsyncQueryExecutor(cluster, policy, listener, statement); }
/// <summary> /// Execute query, apply statement's aggregation function, and return result iterator. /// The aggregation function should be initialized via the statement's SetAggregateFunction() /// and should be located in a Lua resource file located in an assembly. /// <para> /// The query executor puts results on a queue in separate threads. The calling thread /// concurrently pops results off the queue through the ResultSet iterator. /// The aggregation function is called on both server and client (final reduce). /// Therefore, the Lua script file must also reside on both server and client. /// </para> /// <para> /// This method is only supported by Aerospike 3 servers. /// </para> /// </summary> /// <param name="policy">generic configuration parameters, pass in null for defaults</param> /// <param name="statement">database query command with aggregate functions already initialized by SetAggregateFunction()</param> /// <exception cref="AerospikeException">if query fails</exception> public ResultSet QueryAggregate(QueryPolicy policy, Statement statement) { if (policy == null) { policy = queryPolicyDefault; } statement.Prepare(true); QueryAggregateExecutor executor = new QueryAggregateExecutor(cluster, policy, statement); executor.Execute(); return executor.ResultSet; }
/// <summary> /// Initialize Aerospike client with suitable hosts to seed the cluster map. /// The client policy is used to set defaults and size internal data structures. /// For each host connection that succeeds, the client will: /// <list type="bullet"> /// <item>Add host to the cluster map</item> /// <item>Request host's list of other nodes in cluster</item> /// <item>Add these nodes to cluster map</item> /// </list> /// <para> /// In most cases, only one host is necessary to seed the cluster. The remaining hosts /// are added as future seeds in case of a complete network failure. /// </para> /// <para> /// If one connection succeeds, the client is ready to process database requests. /// If all connections fail and the policy's failIfNotConnected is true, a connection /// exception will be thrown. Otherwise, the cluster will remain in a disconnected state /// until the server is activated. /// </para> /// </summary> /// <param name="policy">client configuration parameters, pass in null for defaults</param> /// <param name="hosts">array of potential hosts to seed the cluster</param> /// <exception cref="AerospikeException">if all host connections fail</exception> public AerospikeClient(ClientPolicy policy, params Host[] hosts) { if (policy == null) { policy = new ClientPolicy(); } this.readPolicyDefault = policy.readPolicyDefault; this.writePolicyDefault = policy.writePolicyDefault; this.scanPolicyDefault = policy.scanPolicyDefault; this.queryPolicyDefault = policy.queryPolicyDefault; this.batchPolicyDefault = policy.batchPolicyDefault; this.infoPolicyDefault = policy.infoPolicyDefault; cluster = new Cluster(policy, hosts); cluster.InitTendThread(policy.failIfNotConnected); }
/// <summary> /// Construct client without initialization. /// Should only be used by classes inheriting from this client. /// </summary> protected internal AerospikeClient(ClientPolicy policy) { if (policy != null) { this.readPolicyDefault = policy.readPolicyDefault; this.writePolicyDefault = policy.writePolicyDefault; this.scanPolicyDefault = policy.scanPolicyDefault; this.queryPolicyDefault = policy.queryPolicyDefault; this.batchPolicyDefault = policy.batchPolicyDefault; this.infoPolicyDefault = policy.infoPolicyDefault; } else { this.readPolicyDefault = new Policy(); this.writePolicyDefault = new WritePolicy(); this.scanPolicyDefault = new ScanPolicy(); this.queryPolicyDefault = new QueryPolicy(); this.batchPolicyDefault = new BatchPolicy(); this.infoPolicyDefault = new InfoPolicy(); } }