public AsyncScanExecutor ( AsyncCluster cluster, ScanPolicy policy, RecordSequenceListener listener, string ns, string setName, string[] binNames ) { this.listener = listener; Node[] nodes = cluster.Nodes; if (nodes.Length == 0) { throw new AerospikeException(ResultCode.SERVER_NOT_AVAILABLE, "Scan failed because cluster is empty."); } completedSize = nodes.Length; long taskId = Environment.TickCount; foreach (Node node in nodes) { AsyncScan async = new AsyncScan(this, cluster, (AsyncNode)node, policy, listener, ns, setName, binNames, taskId); async.Execute(); } }
public AsyncScanExecutor ( AsyncCluster cluster, ScanPolicy policy, RecordSequenceListener listener, string ns, string setName, string[] binNames ) { this.listener = listener; Node[] nodes = cluster.Nodes; if (nodes.Length == 0) { throw new AerospikeException(ResultCode.SERVER_NOT_AVAILABLE, "Scan failed because cluster is empty."); } ulong taskId = RandomShift.ThreadLocalInstance.NextLong(); // Create commands. AsyncScan[] tasks = new AsyncScan[nodes.Length]; int count = 0; foreach (Node node in nodes) { tasks[count++] = new AsyncScan(this, cluster, (AsyncNode)node, policy, listener, ns, setName, binNames, taskId); } // Dispatch commands to nodes. Execute(tasks, policy.maxConcurrentNodes); }
public AsyncScanExecutor( AsyncCluster cluster, ScanPolicy policy, RecordSequenceListener listener, string ns, string setName, string[] binNames ) { this.listener = listener; Node[] nodes = cluster.Nodes; if (nodes.Length == 0) { throw new AerospikeException(ResultCode.SERVER_NOT_AVAILABLE, "Scan failed because cluster is empty."); } ulong taskId = RandomShift.ThreadLocalInstance.NextLong(); // Create commands. AsyncScan[] tasks = new AsyncScan[nodes.Length]; int count = 0; foreach (Node node in nodes) { tasks[count++] = new AsyncScan(this, cluster, (AsyncNode)node, policy, listener, ns, setName, binNames, taskId); } // Dispatch commands to nodes. Execute(tasks, policy.maxConcurrentNodes); }
/// <summary> /// Copy client policy from another client policy. /// </summary> public ClientPolicy(ClientPolicy other) { this.user = other.user; this.password = other.password; this.clusterName = other.clusterName; this.authMode = other.authMode; this.timeout = other.timeout; this.loginTimeout = other.loginTimeout; this.maxConnsPerNode = other.maxConnsPerNode; this.connPoolsPerNode = other.connPoolsPerNode; this.maxSocketIdle = other.maxSocketIdle; this.tendInterval = other.tendInterval; this.failIfNotConnected = other.failIfNotConnected; this.readPolicyDefault = new Policy(other.readPolicyDefault); this.writePolicyDefault = new WritePolicy(other.writePolicyDefault); this.scanPolicyDefault = new ScanPolicy(other.scanPolicyDefault); this.queryPolicyDefault = new QueryPolicy(other.queryPolicyDefault); this.batchPolicyDefault = new BatchPolicy(other.batchPolicyDefault); this.infoPolicyDefault = new InfoPolicy(other.infoPolicyDefault); this.tlsPolicy = (other.tlsPolicy != null) ? new TlsPolicy(other.tlsPolicy) : null; this.ipMap = other.ipMap; this.useServicesAlternate = other.useServicesAlternate; this.rackAware = other.rackAware; this.rackId = other.rackId; }
public static void ScanNodes(Cluster cluster, ScanPolicy policy, string ns, string setName, string[] binNames, ScanCallback callback, Node[] nodes) { policy.Validate(); // Detect cluster migrations when performing scan. ulong taskId = RandomShift.ThreadLocalInstance.NextLong(); ulong clusterKey = policy.failOnClusterChange ? QueryValidate.ValidateBegin(nodes[0], ns) : 0; bool first = true; if (policy.concurrentNodes && nodes.Length > 1) { Executor executor = new Executor(nodes.Length); foreach (Node node in nodes) { ScanCommand command = new ScanCommand(cluster, node, policy, ns, setName, binNames, callback, taskId, clusterKey, first); executor.AddCommand(command); first = false; } executor.Execute(policy.maxConcurrentNodes); } else { foreach (Node node in nodes) { ScanCommand command = new ScanCommand(cluster, node, policy, ns, setName, binNames, callback, taskId, clusterKey, first); command.Execute(); first = false; } } }
public void AsyncScan() { recordCount = 0; ScanPolicy policy = new ScanPolicy(); client.ScanAll(policy, new RecordSequenceHandler(this), args.ns, args.set); WaitTillComplete(); }
/// <summary> /// Copy scan policy from another scan policy. /// </summary> public ScanPolicy(ScanPolicy other) : base(other) { this.maxRecords = other.maxRecords; this.scanPercent = other.scanPercent; this.recordsPerSecond = other.recordsPerSecond; this.maxConcurrentNodes = other.maxConcurrentNodes; this.concurrentNodes = other.concurrentNodes; this.includeBinData = other.includeBinData; this.failOnClusterChange = other.failOnClusterChange; }
/// <summary> /// Asynchronous scan example. /// </summary> public override void RunExample(AsyncClient client, Arguments args) { console.Info("Asynchronous scan: namespace=" + args.ns + " set=" + args.set); recordCount = 0; completed = false; DateTime begin = DateTime.Now; ScanPolicy policy = new ScanPolicy(); client.ScanAll(policy, new RecordSequenceHandler(this, begin), args.ns, args.set); WaitTillComplete(); }
public static void ScanPartitions(Cluster cluster, ScanPolicy policy, string ns, string setName, string[] binNames, ScanCallback callback, PartitionTracker tracker) { policy.Validate(); while (true) { ulong taskId = RandomShift.ThreadLocalInstance.NextLong(); try { List <NodePartitions> list = tracker.AssignPartitionsToNodes(cluster, ns); if (policy.concurrentNodes && list.Count > 1) { Executor executor = new Executor(list.Count); foreach (NodePartitions nodePartitions in list) { ScanPartitionCommand command = new ScanPartitionCommand(cluster, policy, ns, setName, binNames, callback, taskId, tracker, nodePartitions); executor.AddCommand(command); } executor.Execute(policy.maxConcurrentNodes); } else { foreach (NodePartitions nodePartitions in list) { ScanPartitionCommand command = new ScanPartitionCommand(cluster, policy, ns, setName, binNames, callback, taskId, tracker, nodePartitions); command.Execute(); } } } catch (AerospikeException ae) { ae.Iteration = tracker.iteration; throw ae; } if (tracker.IsComplete(policy)) { // Scan is complete. return; } if (policy.sleepBetweenRetries > 0) { // Sleep before trying again. Util.Sleep(policy.sleepBetweenRetries); } } }
/// <summary> /// Scan all nodes in parallel and read all records in a set. /// </summary> public override void RunExample(AerospikeClient client, Arguments args) { console.Info("Scan parallel: namespace=" + args.ns + " set=" + args.set); recordCount = 0; DateTime begin = DateTime.Now; ScanPolicy policy = new ScanPolicy(); client.ScanAll(policy, args.ns, args.set, ScanCallback); DateTime end = DateTime.Now; double seconds = end.Subtract(begin).TotalSeconds; console.Info("Total records returned: " + recordCount); console.Info("Elapsed time: " + seconds + " seconds"); double performance = Math.Round((double)recordCount / seconds); console.Info("Records/second: " + performance); }
public ScanCommand ( ScanPolicy policy, string ns, string setName, ScanCallback callback, string[] binNames, ulong taskId ) : base(true) { this.policy = policy; this.ns = ns; this.setName = setName; this.callback = callback; this.binNames = binNames; this.taskId = taskId; }
public ScanCommand( Node node, ScanPolicy policy, string ns, string setName, ScanCallback callback, string[] binNames, long taskId ) : base(node, true) { this.policy = policy; this.ns = ns; this.setName = setName; this.callback = callback; this.binNames = binNames; this.taskId = taskId; }
public ScanCommand ( ScanPolicy policy, string ns, string setName, ScanCallback callback, string[] binNames, ulong taskId, ulong clusterKey, bool first ) : base(ns, clusterKey, first) { this.policy = policy; this.setName = setName; this.callback = callback; this.binNames = binNames; this.taskId = taskId; }
public ScanCommand ( Cluster cluster, Node node, ScanPolicy scanPolicy, string ns, string setName, string[] binNames, ScanCallback callback, ulong taskId, ulong clusterKey, bool first ) : base(cluster, scanPolicy, node, ns, clusterKey, first) { this.scanPolicy = scanPolicy; this.setName = setName; this.binNames = binNames; this.callback = callback; this.taskId = taskId; }
public void ScanSeries() { // Use low scan priority. This will take more time, but it will reduce // the load on the server. ScanPolicy policy = new ScanPolicy(); policy.maxRetries = 1; policy.priority = Priority.LOW; Node[] nodes = client.Nodes; foreach (Node node in nodes) { client.ScanNode(policy, node, args.ns, args.set, ScanCallback); foreach (KeyValuePair<string, Metrics> entry in setMap) { entry.Value.count = 0; } } }
/// <summary> /// Scan all nodes in series and read all records in all sets. /// </summary> public override void RunExample(AerospikeClient client, Arguments args) { console.Info("Scan series: namespace=" + args.ns + " set=" + args.set); setMap.Clear(); // Use low scan priority. This will take more time, but it will reduce // the load on the server. ScanPolicy policy = new ScanPolicy(); policy.maxRetries = 1; policy.priority = Priority.LOW; Node[] nodes = client.Nodes; DateTime begin = DateTime.Now; foreach (Node node in nodes) { console.Info("Scan node " + node.Name); client.ScanNode(policy, node, args.ns, args.set, ScanCallback); foreach (KeyValuePair<string, Metrics> entry in setMap) { console.Info("Node " + node.Name + " set " + entry.Key + " count: " + entry.Value.count); entry.Value.count = 0; } } DateTime end = DateTime.Now; double seconds = end.Subtract(begin).TotalSeconds; console.Info("Elapsed time: " + seconds + " seconds"); long total = 0; foreach (KeyValuePair<string, Metrics> entry in setMap) { console.Info("Total set " + entry.Key + " count: " + entry.Value.total); total += entry.Value.total; } console.Info("Grand total: " + total); double performance = Math.Round((double)total / seconds); console.Info("Records/second: " + performance); }
public ScanPartitionCommand ( Cluster cluster, ScanPolicy scanPolicy, string ns, string setName, string[] binNames, ScanCallback callback, ulong taskId, PartitionTracker tracker, NodePartitions nodePartitions ) : base(cluster, scanPolicy, nodePartitions.node, ns, tracker.socketTimeout, tracker.totalTimeout) { this.scanPolicy = scanPolicy; this.setName = setName; this.binNames = binNames; this.callback = callback; this.taskId = taskId; this.tracker = tracker; this.nodePartitions = nodePartitions; }
public AsyncScanExecutor ( AsyncCluster cluster, ScanPolicy policy, RecordSequenceListener listener, string ns, string setName, string[] binNames, Node[] nodes ) : base(cluster) { this.listener = listener; policy.Validate(); ulong taskId = RandomShift.ThreadLocalInstance.NextLong(); // Create commands. AsyncScan[] tasks = new AsyncScan[nodes.Length]; int count = 0; bool hasClusterStable = true; foreach (Node node in nodes) { if (!node.HasClusterStable) { hasClusterStable = false; } tasks[count++] = new AsyncScan(this, cluster, (AsyncNode)node, policy, listener, ns, setName, binNames, taskId); } // Dispatch commands to nodes. if (policy.failOnClusterChange && hasClusterStable) { ExecuteValidate(tasks, policy.maxConcurrentNodes, ns); } else { Execute(tasks, policy.maxConcurrentNodes); } }
public AsyncScanPartitionExecutor ( AsyncCluster cluster, ScanPolicy policy, RecordSequenceListener listener, string ns, string setName, string[] binNames, PartitionTracker tracker ) : base(cluster) { this.policy = policy; this.listener = listener; this.ns = ns; this.setName = setName; this.binNames = binNames; this.tracker = tracker; policy.Validate(); tracker.SleepBetweenRetries = 0; ScanPartitions(); }
public void scanAllTweetsForAllUsers() { ScanPolicy policy = new ScanPolicy(); policy.concurrentNodes = true; policy.priority = Priority.LOW; policy.includeBinData = true; client.ScanAll(policy, "test", "tweets", scanTweetsCallback, "tweet"); }
/// <summary> /// Read all records in specified namespace and set for one node only. /// <para> /// This call will block until the scan is complete - callbacks are made /// within the scope of this call. /// </para> /// </summary> /// <param name="policy">scan configuration parameters, pass in null for defaults</param> /// <param name="node">server node</param> /// <param name="ns">namespace - equivalent to database name</param> /// <param name="setName">optional set name - equivalent to database table</param> /// <param name="callback">read callback method - called with record data</param> /// <param name="binNames"> /// optional bin to retrieve. All bins will be returned if not specified. /// Aerospike 2 servers ignore this parameter. /// </param> /// <exception cref="AerospikeException">if transaction fails</exception> public void ScanNode(ScanPolicy policy, Node node, string ns, string setName, ScanCallback callback, params string[] binNames) { if (policy == null) { policy = scanPolicyDefault; } long taskId = Environment.TickCount; ScanCommand command = new ScanCommand(node, policy, ns, setName, callback, binNames, taskId); command.Execute(); }
/// <summary> /// Construct client without initialization. /// Should only be used by classes inheriting from this client. /// </summary> protected internal AerospikeClient(ClientPolicy policy) { if (policy != null) { this.readPolicyDefault = policy.readPolicyDefault; this.writePolicyDefault = policy.writePolicyDefault; this.scanPolicyDefault = policy.scanPolicyDefault; this.queryPolicyDefault = policy.queryPolicyDefault; this.batchPolicyDefault = policy.batchPolicyDefault; this.infoPolicyDefault = policy.infoPolicyDefault; } else { this.readPolicyDefault = new Policy(); this.writePolicyDefault = new WritePolicy(); this.scanPolicyDefault = new ScanPolicy(); this.queryPolicyDefault = new QueryPolicy(); this.batchPolicyDefault = new BatchPolicy(); this.infoPolicyDefault = new InfoPolicy(); } }
public void SetScan(ScanPolicy policy, string ns, string setName, string[] binNames, ulong taskId) { Begin(); int fieldCount = 0; if (ns != null) { dataOffset += ByteUtil.EstimateSizeUtf8(ns) + FIELD_HEADER_SIZE; fieldCount++; } if (setName != null) { dataOffset += ByteUtil.EstimateSizeUtf8(setName) + FIELD_HEADER_SIZE; fieldCount++; } // Estimate scan options size. dataOffset += 2 + FIELD_HEADER_SIZE; fieldCount++; // Estimate scan timeout size. dataOffset += 4 + FIELD_HEADER_SIZE; fieldCount++; // Estimate taskId size. dataOffset += 8 + FIELD_HEADER_SIZE; fieldCount++; if (binNames != null) { foreach (String binName in binNames) { EstimateOperationSize(binName); } } SizeBuffer(); byte readAttr = (byte)Command.INFO1_READ; if (!policy.includeBinData) { readAttr |= (byte)Command.INFO1_NOBINDATA; } int operationCount = (binNames == null) ? 0 : binNames.Length; WriteHeader(policy, readAttr, 0, fieldCount, operationCount); if (ns != null) { WriteField(ns, FieldType.NAMESPACE); } if (setName != null) { WriteField(setName, FieldType.TABLE); } WriteFieldHeader(2, FieldType.SCAN_OPTIONS); byte priority = (byte)policy.priority; priority <<= 4; if (policy.failOnClusterChange) { priority |= 0x08; } dataBuffer[dataOffset++] = priority; dataBuffer[dataOffset++] = (byte)policy.scanPercent; // Write scan timeout WriteFieldHeader(4, FieldType.SCAN_TIMEOUT); dataOffset += ByteUtil.IntToBytes((uint)policy.socketTimeout, dataBuffer, dataOffset); // Write taskId field WriteFieldHeader(8, FieldType.TRAN_ID); dataOffset += ByteUtil.LongToBytes(taskId, dataBuffer, dataOffset); if (binNames != null) { foreach (String binName in binNames) { WriteOperation(binName, Operation.Type.READ); } } End(); }
public PartitionTracker(ScanPolicy policy, Node[] nodes, PartitionFilter filter) : this((Policy)policy, nodes, filter) { this.maxRecords = policy.maxRecords; }
/// <summary> /// Initialize Aerospike client with suitable hosts to seed the cluster map. /// The client policy is used to set defaults and size internal data structures. /// For each host connection that succeeds, the client will: /// <list type="bullet"> /// <item>Add host to the cluster map</item> /// <item>Request host's list of other nodes in cluster</item> /// <item>Add these nodes to cluster map</item> /// </list> /// <para> /// In most cases, only one host is necessary to seed the cluster. The remaining hosts /// are added as future seeds in case of a complete network failure. /// </para> /// <para> /// If one connection succeeds, the client is ready to process database requests. /// If all connections fail and the policy's failIfNotConnected is true, a connection /// exception will be thrown. Otherwise, the cluster will remain in a disconnected state /// until the server is activated. /// </para> /// </summary> /// <param name="policy">client configuration parameters, pass in null for defaults</param> /// <param name="hosts">array of potential hosts to seed the cluster</param> /// <exception cref="AerospikeException">if all host connections fail</exception> public AerospikeClient(ClientPolicy policy, params Host[] hosts) { if (policy == null) { policy = new ClientPolicy(); } this.readPolicyDefault = policy.readPolicyDefault; this.writePolicyDefault = policy.writePolicyDefault; this.scanPolicyDefault = policy.scanPolicyDefault; this.queryPolicyDefault = policy.queryPolicyDefault; this.batchPolicyDefault = policy.batchPolicyDefault; this.infoPolicyDefault = policy.infoPolicyDefault; cluster = new Cluster(policy, hosts); cluster.InitTendThread(policy.failIfNotConnected); }
//------------------------------------------------------- // Scan Operations //------------------------------------------------------- /// <summary> /// Read all records in specified namespace and set. If the policy's /// concurrentNodes is specified, each server node will be read in /// parallel. Otherwise, server nodes are read in series. /// <para> /// This call will block until the scan is complete - callbacks are made /// within the scope of this call. /// </para> /// </summary> /// <param name="policy">scan configuration parameters, pass in null for defaults</param> /// <param name="ns">namespace - equivalent to database name</param> /// <param name="setName">optional set name - equivalent to database table</param> /// <param name="callback">read callback method - called with record data</param> /// <param name="binNames"> /// optional bin to retrieve. All bins will be returned if not specified. /// Aerospike 2 servers ignore this parameter. /// </param> /// <exception cref="AerospikeException">if scan fails</exception> public void ScanAll(ScanPolicy policy, string ns, string setName, ScanCallback callback, params string[] binNames) { if (policy == null) { policy = scanPolicyDefault; } Node[] nodes = cluster.Nodes; if (nodes.Length == 0) { throw new AerospikeException(ResultCode.SERVER_NOT_AVAILABLE, "Scan failed because cluster is empty."); } if (policy.concurrentNodes) { Executor executor = new Executor(nodes.Length); ulong taskId = RandomShift.ThreadLocalInstance.NextLong(); foreach (Node node in nodes) { ScanCommand command = new ScanCommand(node, policy, ns, setName, callback, binNames, taskId); executor.AddCommand(command); } executor.Execute(policy.maxConcurrentNodes); } else { foreach (Node node in nodes) { ScanNode(policy, node, ns, setName, callback, binNames); } } }
public PartitionTracker(ScanPolicy policy, Node[] nodes) : this((Policy)policy, nodes) { this.maxRecords = policy.maxRecords; }
/// <summary> /// Read all records in specified namespace and set for one node only. /// The node is specified by name. /// <para> /// This call will block until the scan is complete - callbacks are made /// within the scope of this call. /// </para> /// </summary> /// <param name="policy">scan configuration parameters, pass in null for defaults</param> /// <param name="nodeName">server node name</param> /// <param name="ns">namespace - equivalent to database name</param> /// <param name="setName">optional set name - equivalent to database table</param> /// <param name="callback">read callback method - called with record data</param> /// <param name="binNames"> /// optional bin to retrieve. All bins will be returned if not specified. /// Aerospike 2 servers ignore this parameter. /// </param> /// <exception cref="AerospikeException">if scan fails</exception> public void ScanNode(ScanPolicy policy, string nodeName, string ns, string setName, ScanCallback callback, params string[] binNames) { Node node = cluster.GetNode(nodeName); ScanNode(policy, node, ns, setName, callback, binNames); }
//------------------------------------------------------- // Scan Operations //------------------------------------------------------- /// <summary> /// Asynchronously read all records in specified namespace and set. If the policy's /// concurrentNodes is specified, each server node will be read in /// parallel. Otherwise, server nodes are read in series. /// <para> /// This method schedules the scan command with a channel selector and returns. /// Another thread will process the command and send the results to the listener. /// </para> /// </summary> /// <param name="policy">scan configuration parameters, pass in null for defaults</param> /// <param name="listener">where to send results, pass in null for fire and forget</param> /// <param name="ns">namespace - equivalent to database name</param> /// <param name="setName">optional set name - equivalent to database table</param> /// <param name="binNames"> /// optional bin to retrieve. All bins will be returned if not specified. /// Aerospike 2 servers ignore this parameter. /// </param> /// <exception cref="AerospikeException">if queue is full</exception> public void ScanAll(ScanPolicy policy, RecordSequenceListener listener, string ns, string setName, params string[] binNames) { if (policy == null) { policy = scanPolicyDefault; } new AsyncScanExecutor(cluster, policy, listener, ns, setName, binNames); }
public void SetScan(ScanPolicy policy, string ns, string setName, string[] binNames, ulong taskId) { Begin(); int fieldCount = 0; if (ns != null) { dataOffset += ByteUtil.EstimateSizeUtf8(ns) + FIELD_HEADER_SIZE; fieldCount++; } if (setName != null) { dataOffset += ByteUtil.EstimateSizeUtf8(setName) + FIELD_HEADER_SIZE; fieldCount++; } // Estimate scan options size. dataOffset += 2 + FIELD_HEADER_SIZE; fieldCount++; // Estimate taskId size. dataOffset += 8 + FIELD_HEADER_SIZE; fieldCount++; if (binNames != null) { foreach (String binName in binNames) { EstimateOperationSize(binName); } } SizeBuffer(); byte readAttr = (byte)Command.INFO1_READ; if (!policy.includeBinData) { readAttr |= (byte)Command.INFO1_NOBINDATA; } int operationCount = (binNames == null) ? 0 : binNames.Length; WriteHeader(policy, readAttr, 0, fieldCount, operationCount); if (ns != null) { WriteField(ns, FieldType.NAMESPACE); } if (setName != null) { WriteField(setName, FieldType.TABLE); } WriteFieldHeader(2, FieldType.SCAN_OPTIONS); byte priority = (byte)policy.priority; priority <<= 4; if (policy.failOnClusterChange) { priority |= 0x08; } if (policy.includeLDT) { priority |= 0x02; } dataBuffer[dataOffset++] = priority; dataBuffer[dataOffset++] = (byte)policy.scanPercent; // Write taskId field WriteFieldHeader(8, FieldType.TRAN_ID); ByteUtil.LongToBytes(taskId, dataBuffer, dataOffset); dataOffset += 8; if (binNames != null) { foreach (String binName in binNames) { WriteOperation(binName, Operation.Type.READ); } } End(); }
public void ScanParallel() { ScanPolicy policy = new ScanPolicy(); client.ScanAll(policy, args.ns, args.set, ScanCallback); }
/// <summary> /// Read all records in specified namespace and set for one node only. /// <para> /// This call will block until the scan is complete - callbacks are made /// within the scope of this call. /// </para> /// </summary> /// <param name="policy">scan configuration parameters, pass in null for defaults</param> /// <param name="node">server node</param> /// <param name="ns">namespace - equivalent to database name</param> /// <param name="setName">optional set name - equivalent to database table</param> /// <param name="callback">read callback method - called with record data</param> /// <param name="binNames"> /// optional bin to retrieve. All bins will be returned if not specified. /// Aerospike 2 servers ignore this parameter. /// </param> /// <exception cref="AerospikeException">if transaction fails</exception> public void ScanNode(ScanPolicy policy, Node node, string ns, string setName, ScanCallback callback, params string[] binNames) { if (policy == null) { policy = scanPolicyDefault; } ulong taskId = RandomShift.ThreadLocalInstance.NextLong(); ScanCommand command = new ScanCommand(node, policy, ns, setName, callback, binNames, taskId); command.Execute(); }