/// <summary> /// Initializes a new instance of the <see cref="Producer<TKey, TData>"/> class. /// </summary> /// <param name="config">The config object.</param> /// <param name="partitioner">The partitioner that implements <see cref="IPartitioner<TKey>" /> /// used to supply a custom partitioning strategy based on the message key.</param> /// <param name="producerPool">Pool of producers, one per broker.</param> /// <param name="populateProducerPool">if set to <c>true</c>, producers should be populated.</param> /// <remarks> /// Should be used for testing purpose only. /// </remarks> internal Producer( ProducerConfiguration config, IPartitioner <TKey> partitioner, IProducerPool <TData> producerPool, bool populateProducerPool = true) { Guard.NotNull(config, "config"); Guard.NotNull(producerPool, "producerPool"); this.config = config; this.partitioner = partitioner ?? new DefaultPartitioner <TKey>(); this.populateProducerPool = populateProducerPool; this.producerPool = producerPool; if (this.config.IsZooKeeperEnabled) { this.brokerPartitionInfo = new ZKBrokerPartitionInfo(this.config, this.Callback); } else { this.brokerPartitionInfo = new ConfigBrokerPartitionInfo(this.config); } if (this.populateProducerPool) { IDictionary <int, Broker> allBrokers = this.brokerPartitionInfo.GetAllBrokerInfo(); foreach (var broker in allBrokers) { this.producerPool.AddProducer( new Broker(broker.Key, broker.Value.Host, broker.Value.Host, broker.Value.Port)); } } }
/// <summary> /// Specify a concurrency limit for tasks executing through the filter. No more than the specified /// number of tasks will be allowed to execute concurrently. /// </summary> /// <typeparam name="TActivity"></typeparam> /// <typeparam name="TLog"></typeparam> /// <param name="configurator"></param> /// <param name="partitioner">An existing partitioner to share</param> /// <param name="keyProvider">Provides the key from the message</param> public static void UsePartitioner <TActivity, TLog>(this IPipeConfigurator <CompensateActivityContext <TActivity, TLog> > configurator, IPartitioner partitioner, Func <CompensateActivityContext <TActivity, TLog>, Guid> keyProvider) where TActivity : class, ICompensateActivity <TLog> where TLog : class { if (configurator == null) { throw new ArgumentNullException(nameof(configurator)); } if (partitioner == null) { throw new ArgumentNullException(nameof(partitioner)); } if (keyProvider == null) { throw new ArgumentNullException(nameof(keyProvider)); } byte[] PartitionKeyProvider(CompensateActivityContext <TActivity, TLog> context) { return(keyProvider(context).ToByteArray()); } var specification = new PartitionerPipeSpecification <CompensateActivityContext <TActivity, TLog> >(PartitionKeyProvider, partitioner); configurator.AddPipeSpecification(specification); }
public ExportRequest( Guild guild, Channel channel, string outputPath, ExportFormat format, Snowflake?after, Snowflake?before, IPartitioner partitioner, bool shouldDownloadMedia, bool shouldReuseMedia, string dateFormat) { Guild = guild; Channel = channel; OutputPath = outputPath; Format = format; After = after; Before = before; Partitoner = partitioner; ShouldDownloadMedia = shouldDownloadMedia; ShouldReuseMedia = shouldReuseMedia; DateFormat = dateFormat; OutputBaseFilePath = GetOutputBaseFilePath( guild, channel, outputPath, format, after, before ); OutputBaseDirPath = Path.GetDirectoryName(OutputBaseFilePath) ?? OutputPath; OutputMediaDirPath = $"{OutputBaseFilePath}_Files{Path.DirectorySeparatorChar}"; }
public PartitionMessageConfigurationObserver(IConsumePipeConfigurator configurator, IPartitioner partitioner) : base(configurator) { _partitioner = partitioner; Connect(this); }
/// <summary> /// Specify a concurrency limit for tasks executing through the filter. No more than the specified /// number of tasks will be allowed to execute concurrently. /// </summary> /// <typeparam name="TActivity"></typeparam> /// <typeparam name="TLog"></typeparam> /// <param name="configurator"></param> /// <param name="partitioner">An existing partitioner to share</param> /// <param name="keyProvider">Provides the key from the message</param> /// <param name="encoding">The text encoding to use to convert the string to byte[] (defaults to UTF8)</param> public static void UsePartitioner <TActivity, TLog>(this IPipeConfigurator <CompensateActivityContext <TActivity, TLog> > configurator, IPartitioner partitioner, Func <CompensateActivityContext <TActivity, TLog>, string> keyProvider, Encoding encoding = null) where TActivity : class, CompensateActivity <TLog> where TLog : class { if (configurator == null) { throw new ArgumentNullException(nameof(configurator)); } if (partitioner == null) { throw new ArgumentNullException(nameof(partitioner)); } if (keyProvider == null) { throw new ArgumentNullException(nameof(keyProvider)); } var textEncoding = encoding ?? Encoding.UTF8; PartitionKeyProvider <CompensateActivityContext <TActivity, TLog> > provider = context => { var key = keyProvider(context) ?? ""; return(textEncoding.GetBytes(key)); }; var specification = new PartitionerPipeSpecification <CompensateActivityContext <TActivity, TLog> >(provider, partitioner); configurator.AddPipeSpecification(specification); }
public Partitions(IPartitionKeyGenerator partitionKeyGenerator, IStringHasher <THashOutput> hasher, IPartitioner <THashOutput> partitioner, int partitionCount) { _partitionCount = partitionCount; _partitionKeyGenerator = partitionKeyGenerator; _partitions = new Partition[_partitionCount]; _partitioner = partitioner; _hasher = hasher; }
/// <summary> /// Send binary message to specified topic with specified key, using specified partitioner. /// </summary> public void Send(String topic, byte[] payload, Object key, IPartitioner partitioner) { var partitionsNumber = GetNumberOfPartitionsForTopic(topic); var partition = _partitioner.SelectPartition(null, partitionsNumber); var request = new AppendRequest(topic, partition, Message.CreateMessage(payload)); _pushSender.Push(request); }
/// <summary> /// Constructs a <see cref="SimpleStepExecutionSplitter"/> from its mandatory properties. /// </summary> /// <param name="jobRepository"></param> /// <param name="allowStartIfComplete">flag specifying preferences on restart</param> /// <param name="stepName"></param> /// <param name="partitioner">a IPartitioner to use for generating input parameters</param> public SimpleStepExecutionSplitter(IJobRepository jobRepository, bool allowStartIfComplete, string stepName, IPartitioner partitioner) { JobRepository = jobRepository; AllowStartIfComplete = allowStartIfComplete; Partitioner = partitioner; StepName = stepName; }
public TuplePartitioner(IPartitioner partitioner, int numberReducers) { Partitioner = partitioner; NumberReducers = numberReducers; for (int i = 0; i < NumberReducers; i++) { partitions.Add(new JArray()); } }
public SingleConnectionPerEndpointCluster(IEndpointStrategy endpointStrategy, ILogger logger, IConnectionFactory connectionFactory, IRecoveryService recoveryService, IPartitioner partitioner) { _ip2Connection = new Dictionary <IPAddress, IConnection>(); _endpointStrategy = endpointStrategy; _logger = logger; _connectionFactory = connectionFactory; _recoveryService = recoveryService; Partitioner = partitioner; }
protected WriteableSpaceBase(WriteableSpaceBase <T> parent, Func <T, Vector2> getPoint, Func <T, TopLeftRectangle> getBox, IPartitioner partitioner = null) { Parent = parent; GetPoint = getPoint; GetBox = getBox; _partitions = new List <IWriteableSpace <T> >(); _readOnlyPartitions = _partitions.AsReadOnly(); _partitioner = partitioner; }
public IEnumerable <Observation> Aggregate(IPartitioner partitioner, ICalculator calculator) { var partitions = partitioner.Partition(_observations); foreach (var partition in partitions) { yield return(calculator.AggregateCollection(partition)); } }
public SingleConnectionPerEndpointCluster(IEndpointStrategy endpointStrategy, ILogger logger, IConnectionFactory connectionFactory, IRecoveryService recoveryService, IPartitioner partitioner) { _ip2Connection = new Dictionary<IPAddress, IConnection>(); _endpointStrategy = endpointStrategy; _logger = logger; _connectionFactory = connectionFactory; _recoveryService = recoveryService; Partitioner = partitioner; }
/// <summary> /// Configure the routing slip activity event consumers on a receive endpoint /// </summary> /// <param name="configurator"></param> /// <param name="persister"></param> /// <param name="partitioner">Use a partitioner to reduce duplicate key errors</param> public static void RoutingSlipActivityEventConsumers(this IReceiveEndpointConfigurator configurator, IRoutingSlipEventPersister persister, IPartitioner partitioner) { configurator.Consumer(() => new RoutingSlipActivityCompensatedConsumer(persister), x => x.ConfigureMessage<RoutingSlipActivityCompensated>(y => y.UsePartitioner(partitioner, p => p.Message.TrackingNumber))); configurator.Consumer(() => new RoutingSlipActivityCompletedConsumer(persister), x => x.ConfigureMessage<RoutingSlipActivityCompleted>(y => y.UsePartitioner(partitioner, p => p.Message.TrackingNumber))); configurator.Consumer(() => new RoutingSlipActivityFaultedConsumer(persister), x => x.ConfigureMessage<RoutingSlipActivityFaulted>(y => y.UsePartitioner(partitioner, p => p.Message.TrackingNumber))); configurator.Consumer(() => new RoutingSlipActivityCompensationFailedConsumer(persister), x => x.ConfigureMessage<RoutingSlipActivityCompensationFailed>(y => y.UsePartitioner(partitioner, p => p.Message.TrackingNumber))); }
public ConnectionPoolEndpointCluster(IEndpointStrategy endpointStrategy, ILogger logger, IConnectionFactory connectionFactory, IRecoveryService recoveryService, IPartitioner partitioner) { _ip2Connection = new ConcurrentDictionary <IPAddress, IConnection[]>(); _endpointStrategy = endpointStrategy; _logger = logger; _connectionFactory = connectionFactory; _recoveryService = recoveryService; Partitioner = partitioner; logger?.Info("ConnectionPoolEndpointCluster created."); }
private Partition CallCustomPartitioner(string topic, IPartitioner partitioner, IntPtr keydata, UIntPtr keylen, int partition_cnt, IntPtr rkt_opaque, IntPtr msg_opaque) { try { return(partitioner.Partition(topic, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque)); } catch { return(Partition.Any); } }
/// <summary> /// Configure the routing slip activity event consumers on a receive endpoint /// </summary> /// <param name="configurator"></param> /// <param name="persister"></param> /// <param name="partitioner">Use a partitioner to reduce duplicate key errors</param> public static void RoutingSlipActivityEventConsumers(this IReceiveEndpointConfigurator configurator, IRoutingSlipEventPersister persister, IPartitioner partitioner) { configurator.Consumer(() => new RoutingSlipActivityCompensatedConsumer(persister), x => x.Message <RoutingSlipActivityCompensated>(y => y.UsePartitioner(partitioner, p => p.Message.TrackingNumber))); configurator.Consumer(() => new RoutingSlipActivityCompletedConsumer(persister), x => x.Message <RoutingSlipActivityCompleted>(y => y.UsePartitioner(partitioner, p => p.Message.TrackingNumber))); configurator.Consumer(() => new RoutingSlipActivityFaultedConsumer(persister), x => x.Message <RoutingSlipActivityFaulted>(y => y.UsePartitioner(partitioner, p => p.Message.TrackingNumber))); configurator.Consumer(() => new RoutingSlipActivityCompensationFailedConsumer(persister), x => x.Message <RoutingSlipActivityCompensationFailed>(y => y.UsePartitioner(partitioner, p => p.Message.TrackingNumber))); }
/// <summary> /// Initializes a new instance of the <see cref="Producer<TKey, TData>"/> class. /// </summary> /// <param name="config">The config object.</param> /// <param name="partitioner">The partitioner that implements <see cref="IPartitioner<TKey>" /> /// used to supply a custom partitioning strategy based on the message key.</param> /// <param name="encoder">The encoder that implements <see cref="IEncoder<TData>" /> /// used to convert an object of type TData to <see cref="Message" />.</param> /// <remarks> /// Can be used to provide pre-instantiated objects for all config parameters /// that would otherwise be instantiated via reflection. /// </remarks> public Producer( ProducerConfiguration config, IPartitioner <TKey> partitioner, IEncoder <TData> encoder) : this( config, partitioner, ProducerPool <TData> .CreatePool(config, encoder, null)) { Guard.NotNull(config, "config"); Guard.NotNull(encoder, "encoder"); }
public DefaultCallbackHandler(ProducerConfiguration config, IPartitioner <TK> partitioner, IEncoder <TV> encoder, IBrokerPartitionInfo brokerPartitionInfo, ISyncProducerPool syncProducerPool) { this.producerConfig = config; this.partitioner = partitioner; Logger.DebugFormat("partitioner {0}", this.partitioner == null ? "Null" : this.partitioner.GetType().ToString()); this.encoder = encoder; this.syncProducerPool = syncProducerPool; this.brokerPartitionInfo = brokerPartitionInfo; }
public PartitionConsumerSpecification(IPartitioner partitioner, PartitionKeyProvider <ConsumerConsumeContext <TConsumer> > keyProvider) { if (partitioner == null) { throw new ArgumentNullException(nameof(partitioner)); } if (keyProvider == null) { throw new ArgumentNullException(nameof(keyProvider)); } _partitioner = partitioner; _keyProvider = keyProvider; }
private static string SerializePartitioner(IPartitioner partitioner) { if (partitioner is ContinuousPartitioner) { return(SerializePartitioner((ContinuousPartitioner)partitioner)); } else if (partitioner is CategoricalPartitioner) { return(SerializePartitioner((CategoricalPartitioner)partitioner)); } else { return(string.Format("<{0}>", partitioner)); } }
private static void Run() { ISagaRepository <RegistrationStateInstance> sagaRepository = new EntityFrameworkSagaRepository <RegistrationStateInstance>(SagaDbContextFactory); IBusControl busControl = Bus.Factory.CreateUsingRabbitMq(cfg => { IRabbitMqHost host = cfg.Host(new Uri("rabbitmq://localhost"), h => { h.Username("guest"); h.Password("guest"); }); EndpointConvention.Map <IProcessRegistration>(new Uri("rabbitmq://localhost/registration.state.queue")); cfg.ReceiveEndpoint(host, "registration.state.queue", endpointConfigurator => { endpointConfigurator.PrefetchCount = 16; IPartitioner partitioner = cfg.CreatePartitioner(8); RegistrationStateMachine machine = new RegistrationStateMachine(); endpointConfigurator.StateMachineSaga(machine, sagaRepository, sagaConfigurator => { sagaConfigurator.Message <IRegistrationReceived>(m => m.UsePartitioner(partitioner, p => p.Message.SubmissionId)); sagaConfigurator.Message <IRegistrationCompleted>(m => m.UsePartitioner(partitioner, p => p.Message.SubmissionId)); }); }); }); busControl.StartAsync(); ReadKey(); busControl.StopAsync(); DbContext SagaDbContextFactory() { DbContextOptionsBuilder optionsBuilder = new DbContextOptionsBuilder(); optionsBuilder.UseSqlServer(@"Data Source=(LocalDB)\MSSQLLocalDB;Initial Catalog=RegistrationDemo;Integrated Security=True"); SagaDbContext <RegistrationStateInstance, RegistrationStateInstanceMap> context = new SagaDbContext <RegistrationStateInstance, RegistrationStateInstanceMap>(optionsBuilder.Options); context.Database.EnsureCreated(); return(context); } }
/// <summary> /// Set the partitioner to be called when a <seealso cref="Partition"/> needs to be determined for a <seealso cref="Message{TKey, TValue}"/>. /// </summary> /// <remarks> /// This handler is only called when the <seealso cref="Message{TKey, TValue}"/>'s partition is set to <seealso cref="Partition.Any"/>. /// </remarks> public ProducerBuilder <TKey, TValue> SetPartitioner(string topic, IPartitioner partitioner) { if (string.IsNullOrWhiteSpace(topic)) { throw new ArgumentNullException(nameof(topic)); } if (this.Partitioners.ContainsKey(topic)) { this.Partitioners[topic] = partitioner; } else { this.Partitioners.Add(topic, partitioner); } return(this); }
public ICluster GetCluster(ClusterConfig clusterConfig) { clusterConfig.CheckArgumentNotNull("clusterConfig"); clusterConfig.Endpoints.CheckArgumentNotNull("clusterConfig.Endpoints"); TransportConfig transportConfig = clusterConfig.Transport ?? new TransportConfig(); IRecoveryService recoveryService = GetRecoveryService(transportConfig.Recoverable); KeyspaceConfig keyspaceConfig = clusterConfig.DefaultKeyspace ?? new KeyspaceConfig(); // create endpoints IEndpointSnitch snitch = ServiceActivator <Factory> .Create <IEndpointSnitch>(clusterConfig.Endpoints.Snitch, _logger); IEnumerable <IPAddress> endpoints = clusterConfig.Endpoints.Servers.Select(Network.Find).Where(x => null != x).ToArray(); if (!endpoints.Any()) { throw new ArgumentException("Expecting at least one valid endpoint"); } // create required services IEndpointStrategy endpointsManager = ServiceActivator <EndpointStrategy.Factory> .Create <IEndpointStrategy>(clusterConfig.Endpoints.Strategy, endpoints, snitch, _logger, clusterConfig.Endpoints); IConnectionFactory connectionFactory = ServiceActivator <Transport.Factory> .Create <IConnectionFactory>(transportConfig.Type, transportConfig, keyspaceConfig, _logger, _instrumentation); IPartitioner partitioner = ServiceActivator <Partitioner.Factory> .Create <IPartitioner>(clusterConfig.Partitioner); // create the cluster now ICluster cluster = ServiceActivator <Cluster.Factory> .Create <ICluster>(clusterConfig.Type, endpointsManager, _logger, connectionFactory, recoveryService, partitioner, clusterConfig); IDiscoveryService discoveryService = ServiceActivator <Discovery.Factory> .Create <IDiscoveryService>(clusterConfig.Endpoints.Discovery.Type, clusterConfig.Endpoints.Discovery, _logger, cluster); discoveryService.OnTopologyUpdate += endpointsManager.Update; cluster.OnClosed += discoveryService.SafeDispose; return(cluster); }
public DefaultEventHandler(ProducerConfig config, IPartitioner partitioner, IEncoder <TValue> encoder, IEncoder <TKey> keyEncoder, ProducerPool producerPool, Dictionary <string, TopicMetadata> topicPartitionInfos = null) { this.Config = config; this.partitioner = partitioner; this.encoder = encoder; this.keyEncoder = keyEncoder; this.producerPool = producerPool; if (topicPartitionInfos != null) { this.topicPartitionInfos = topicPartitionInfos; } this.brokerPartitionInfo = new BrokerPartitionInfo(this.Config, this.producerPool, this.topicPartitionInfos); this.topicMetadataRefreshInterval = TimeSpan.FromMilliseconds(config.TopicMetadataRefreshIntervalMs); this.producerStats = ProducerStatsRegistry.GetProducerStats(config.ClientId); this.producerTopicStats = ProducerTopicStatsRegistry.GetProducerTopicStats(config.ClientId); }
/// <summary> /// Specify a concurrency limit for tasks executing through the filter. No more than the specified /// number of tasks will be allowed to execute concurrently. /// </summary> /// <typeparam name="TActivity"></typeparam> /// <typeparam name="TArguments"></typeparam> /// <param name="configurator"></param> /// <param name="partitioner">An existing partitioner to share</param> /// <param name="keyProvider">Provides the key from the message</param> public static void UsePartitioner <TActivity, TArguments>(this IPipeConfigurator <ExecuteActivityContext <TActivity, TArguments> > configurator, IPartitioner partitioner, Func <ExecuteActivityContext <TActivity, TArguments>, Guid> keyProvider) where TActivity : class, ExecuteActivity <TArguments> where TArguments : class { if (configurator == null) { throw new ArgumentNullException(nameof(configurator)); } if (partitioner == null) { throw new ArgumentNullException(nameof(partitioner)); } if (keyProvider == null) { throw new ArgumentNullException(nameof(keyProvider)); } PartitionKeyProvider <ExecuteActivityContext <TActivity, TArguments> > provider = context => keyProvider(context).ToByteArray(); var specification = new PartitionerPipeSpecification <ExecuteActivityContext <TActivity, TArguments> >(provider, partitioner); configurator.AddPipeSpecification(specification); }
public MessagingSpace(ISubscribableRouter router, Func <T, TopLeftRectangle> getBox, IPartitioner partitioner = null) : this(router, x => getBox(x).Center, getBox, partitioner) { }
public MessagingSpace(ISubscribableRouter router, Func <T, Vector2> getPoint, IPartitioner partitioner = null) : this(router, getPoint, x => new CenteredRectangle(getPoint(x), 0, 0), partitioner) { }
/// <summary> /// Sets the partitioner. /// </summary> /// <param name="slaveStepName">the name of the partitioned step</param> /// <param name="name">the partitioner</param> /// <returns>the current step builder</returns> public PartitionStepBuilder Partitioner(string slaveStepName, IPartitioner name) { _stepName = slaveStepName; _partitioner = name; return this; }
/// <summary> /// Specify a concurrency limit for tasks executing through the filter. No more than the specified /// number of tasks will be allowed to execute concurrently. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="configurator"></param> /// <param name="partitioner">An existing partitioner that is shared</param> /// <param name="keyProvider">Provides the key from the message</param> public static void UsePartitioner <T>(this IPipeConfigurator <T> configurator, IPartitioner partitioner, Func <T, Guid> keyProvider) where T : class, PipeContext { if (configurator == null) { throw new ArgumentNullException(nameof(configurator)); } if (partitioner == null) { throw new ArgumentNullException(nameof(partitioner)); } if (keyProvider == null) { throw new ArgumentNullException(nameof(keyProvider)); } PartitionKeyProvider <T> provider = context => keyProvider(context).ToByteArray(); var specification = new PartitionerPipeSpecification <T>(provider, partitioner); configurator.AddPipeSpecification(specification); }
/// <summary> /// Send text message to specified topic with specified key, using default UTF-8 encoding and specified partitioner /// </summary> public void Send(String topic, String message, Object key, IPartitioner partitioner) { Send(topic, message, key, partitioner, _encoding); }
public FluentMapAndReduce ShuffleUsing(IPartitioner partitioner) { job.ShufflePartitioner = partitioner; return this; }
public FluentMapAndReduce ShuffleInMemoryUsing(IPartitioner partitioner) { job.ShufflePartitioner = partitioner; job.ShufflerFitsInMemory = true; return this; }
public FluentMapAndReduce PartitionInputUsing(IPartitioner partitioner) { job.InputPartitioner = partitioner; return this; }
public FluentMapAndReduce PartitionInputInMemoryUsing(IPartitioner partitioner) { job.InputPartitioner = partitioner; job.InputFitsInMemory = true; return this; }
public MessagingSpace(ISubscribableRouter router, Func <T, Vector2> getPoint, Func <T, TopLeftRectangle> getBox, IPartitioner partitioner = null) { _router = router; _space = new Space <T>(getPoint, getBox, partitioner); _newInstances = new List <T>(); NewInstances = new ReadOnlyCollection <T>(_newInstances); _router.Add <ICompositionMessage <T> >(Interpret); _router.Add <IDecompositionMessage <T> >(Interpret); }
public void AddPartitioner(IPartitioner <THashOutput> partitioner) { _partitioners.Add(partitioner); }
/// <summary> /// Send text message to specified topic with specified key, using specified encoding and partitioner. /// </summary> public void Send(String topic, String message, Object key, IPartitioner partitioner, Encoding encoding) { Send(topic, encoding.GetBytes(message), key, partitioner); }
/// <summary> /// Specify a concurrency limit for tasks executing through the filter. No more than the specified /// number of tasks will be allowed to execute concurrently. /// </summary> /// <typeparam name="T"></typeparam> /// <param name="configurator"></param> /// <param name="partitioner">An existing partitioner that is shared</param> /// <param name="keyProvider">Provides the key from the message</param> public static void UsePartitioner <T>(this IPipeConfigurator <T> configurator, IPartitioner partitioner, Func <T, long> keyProvider) where T : class, PipeContext { if (configurator == null) { throw new ArgumentNullException(nameof(configurator)); } if (partitioner == null) { throw new ArgumentNullException(nameof(partitioner)); } if (keyProvider == null) { throw new ArgumentNullException(nameof(keyProvider)); } PartitionKeyProvider <T> provider = context => { var key = keyProvider(context); return(BitConverter.GetBytes(key)); }; var specification = new PartitionerPipeSpecification <T>(provider, partitioner); configurator.AddPipeSpecification(specification); }