public BrokerConfigurationBuilder() { this.brokerConfiguration = new BrokerConfiguration { Topics = new List <string>() }; }
//public KafkaDispatcherMS(ProducerConfigManager producerConfigManager) //{ // this._producerConfigManager = producerConfigManager; //} public virtual async Task UpsertDocument <TDocument>(IUpsertDocumentContext <TDocument> context) where TDocument : class { try { var topicName = context.Document.GetType().Name; var brokerConfig = new BrokerConfiguration() { BrokerId = 0, Host = "localhost", Port = 9092 }; var config = new ProducerConfiguration(new List <BrokerConfiguration> { brokerConfig }); //var serializer = new BinarySerializer<TDocument>(); var json = this._serializer.Serialize(context.Document); var content = Encoding.Default.GetBytes(json); var msg = new Message(content); var kafkaProducer = new Producer(config); var batch = new ProducerData <string, Message>(topicName, msg); kafkaProducer.Send(batch); } catch (Exception e) { } }
private static void MicrosoftKafkaTest() { var brokerConfig = new BrokerConfiguration() { Host = "0.tcp.ngrok.io", Port = 15069 }; var config = new ProducerConfiguration(new List <BrokerConfiguration> { brokerConfig }); var producer = new Kafka.Client.Producers.Producer(config); var payload = Encoding.ASCII.GetBytes(@" { 'method': 'POST', 'timeStamp': 1508164715, 'foo': { 'id': 123 } }"); var message = new Kafka.Client.Messages.Message(payload); var data = new ProducerData <string, Kafka.Client.Messages.Message>("fun-topic", message); try { producer.Send(data); } catch (Exception ex) { Console.WriteLine(ex.Message); } }
private IList <BrokerConfiguration> GetBrokerConfigurations() { List <BrokerConfiguration> brokerConfigurations = new List <BrokerConfiguration>(); string brokers = ConfigurationManager.AppSettings["Brokers"]; string[] brokerArray = brokers.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries); int port = String.IsNullOrEmpty(ConfigurationManager.AppSettings["BrokerPort"]) ? 8080 : Convert.ToInt32(ConfigurationManager.AppSettings["BrokerPort"]); if (brokerArray.Count() <= 0) { Console.WriteLine("No brokers found!"); } var broker = default(BrokerConfiguration); for (int i = 0; i < brokerArray.Length; i++) { broker = new BrokerConfiguration() { //BrokerId = i + 1, Host = brokerArray[i].Trim(), Port = port }; brokerConfigurations.Add(broker); } return(brokerConfigurations); }
public TradeUtils(BrokerConfiguration brokerConfig, RuleConfiguration ruleConfig) { this.BrokerConfig = brokerConfig; TradeVolumeCalculator = new TradeVolumeCalculator(ruleConfig); OrderSpacingCalculator = new OrderSpacingCalculator(ruleConfig); }
// public void CreateVHost(BrokerConfiguration configuration, Assembly callerAssembly) // { // AssemblyBuildInfoAttribute buildInfo = BuildHelper.GetBuildInfo(callerAssembly); // if (buildInfo != null) // { // string vHostTestName = $@"{configuration.Vhost}_{buildInfo.BuildDate}_{buildInfo.BranchName.Replace('/', '_')}_{buildInfo.BuildNumber}"; // #if DEBUG // vHostTestName = $@"{configuration.Vhost}_{Environment.MachineName}_{Environment.UserName}"; // #endif // configuration.Vhost = CreateVHost(configuration, vHostTestName); // } // } private T HTTPCall <T>(BrokerConfiguration cfg, Func <T> httpAction) { _httpConnector.Credentials = new NetworkCredential { UserName = cfg.UserName, Password = cfg.Password }; _httpConnector.Uri = new Uri($"http://{cfg.HostName}:1{cfg.Port}/"); return(httpAction()); }
private void CallPutRabbitMqApi(BrokerConfiguration cfg, string call, object body = null) { HTTPCall(cfg, () => { var content = new StringContent(body != null ? _jsonSerializer.SerializeObject(body) : "", Encoding.UTF8, "application/json"); return(_httpConnector.Put <string>(call, content)); }); }
public void AddHost(BrokerConfiguration configuration, string vHostName) { if (!_vhosts.ContainsKey(configuration)) { _vhosts[configuration] = new HashSet <string>(); } _vhosts[configuration].Add(vHostName); }
private void DeleteVHost(BrokerConfiguration configuration, string vHostName) { var vhosts = GetVHost(configuration); string httpVHost = vHostName.Replace(@"/", @"%2f"); VHost vhost = vhosts.FirstOrDefault(vh => vh.Name == vHostName); if (vhost == null) { return; } CallDeleteRabbitMqApi(configuration, $"api/vhosts/{httpVHost}"); }
private void CreateConnection(BrokerConfiguration configuration) { try { if (!IsConnect) { if (_factory == null) { _factory = new ConnectionFactory { HostName = configuration.HostName, Port = configuration.Port, UserName = configuration.UserName, Password = configuration.Password, VirtualHost = configuration.Vhost, }; #if DEBUG // Force to use Ipv4 instead of trying ipv6 and try catch exception in his code. _factory.SocketFactory = family => ConnectionFactory.DefaultSocketFactory(AddressFamily.InterNetwork); #endif } RegisteredConnectors.Clear(); _connection = _factory.CreateConnection($"{Environment.MachineName}:{Environment.UserName}"); _logger.LogInformation($"Connected on broker {_factory.HostName} - vHost: {_factory.VirtualHost}"); using (IModel channel = _connection.CreateModel()) { CreateDeadLetter(channel); } _connection.ConnectionShutdown += OnConnectionShutdown; _configuration = configuration; } if (IsConnect) { OnConnect?.Invoke(this, EventArgs.Empty); } } catch (Exception e) { _logger.LogError(e, "Connection failed."); } }
public static void Main(string[] args) { var section = (BrokerConfigurationSection)ConfigurationManager.GetSection("brodBroker"); var configuration = BrokerConfiguration.FromConfigurationSection(section); Console.WriteLine("Brod Broker, v0.0.0.0.1"); Console.WriteLine("-----------------------"); Console.WriteLine(" Storage Directory: {0}", configuration.StorageDirectory); Console.WriteLine(" Default number of partitions: {0}", configuration.NumberOfPartitions); //var socketServer = new SocketServer(configuration.ConsumerPort, configuration.ProducerPort, 1); //socketServer.Startup(); var server = new Broker(configuration); server.Start(); }
private string CreateVHost(BrokerConfiguration configuration, string vHostName) { var vhosts = GetVHost(configuration); string httpVHost = vHostName.Replace(@"/", @"%2f"); VHost vhost = vhosts.FirstOrDefault(vh => vh.Name == vHostName); if (vhost == null) { CallPutRabbitMqApi(configuration, $"api/vhosts/{httpVHost}"); CallPutRabbitMqApi(configuration, $"api/permissions/{httpVHost}/{configuration.UserName}", new Permission()); if (!_vhosts.ContainsKey(configuration)) { _vhosts[configuration] = new HashSet <string>(); } _vhosts[configuration].Add(vHostName); } return(vHostName); }
public void ValidatorResolution() { var container = CreateContainer(); var meConfig = new SimpleMappingEngineConfiguration(container); meConfig.Configure(); var repository = new Mock <IRepository>(); container.RegisterInstance(repository.Object); var config = new BrokerConfiguration(container); config.Configure(); var validator = container.Resolve <IValidator <Broker> >("broker"); // Assert Assert.IsNotNull(validator, "Validator resolution failed"); }
/// <summary> /// Convert a broker string into a BrokerConfiguration. /// </summary> /// <param name="brokerAddr">Broker string must follow the format of {Broker-ID},{Host}:{Port}</param> /// <returns></returns> public static BrokerConfiguration ToBrokerConfig(string brokerAddr) { if (string.IsNullOrEmpty(brokerAddr)) { return(null); } var brokerParams = brokerAddr.Split(','); if (brokerParams == null || brokerParams.Length != 2) { return(null); } var brokerId = -1; if (!int.TryParse(brokerParams[0], out brokerId)) { return(null); } var hostParams = brokerParams[1].Split(':'); if (hostParams == null || hostParams.Length != 2) { return(null); } var portNumber = -1; if (!int.TryParse(hostParams[1], out portNumber)) { return(null); } var broker = new BrokerConfiguration { BrokerId = brokerId, Host = hostParams[0], Port = portNumber }; return(broker); }
public override bool LoadResources() { var brokerConfig = new BrokerConfiguration() { BrokerId = BrokerId, Host = Host, Port = Port }; var config = new ProducerConfiguration(new List <BrokerConfiguration>() { brokerConfig }) { PartitionerClass = typeof(ProducerKeyPatition).AssemblyQualifiedName }; _producer = new ProducerDevId(config); return(true); }
public BrokerFactory(BrokerConfiguration brokerConfiguration) { this.brokerConfiguration = brokerConfiguration; }
private IEnumerable <VHost> GetVHost(BrokerConfiguration cfg) => CallGetRabbitMqApi <IEnumerable <VHost> >(cfg, "api/vhosts").Result;
private void CallDeleteRabbitMqApi(BrokerConfiguration cfg, string call) => HTTPCall(cfg, () => _httpConnector.Delete <string>(call));
public static PullResponse PullMessage( ConsumerConfiguration consumerConfig, BrokerConfiguration leaderBrokerConfig, int correlationID, string topic, int partitionIndex, long fetchOffset, KafkaClientHelperConfiguration helperConfiguration, out long offsetNew) { offsetNew = -1; PullResponse result = null; int payloadCount = 0; // at least retry once int maxRetry = 1; int retryCount = 0; string s = string.Empty; bool success = false; while (!success && retryCount < maxRetry) { try { var requestMap = new Dictionary <string, List <PartitionFetchInfo> >(); requestMap.Add( topic, new List <PartitionFetchInfo>() { new PartitionFetchInfo( partitionIndex, fetchOffset, helperConfiguration.FetchSize) }); using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port)) { var response = consumer.Fetch(new FetchRequest( correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id helperConfiguration.MaxWaitTime, helperConfiguration.MinWaitBytes, requestMap)); if (response == null) { throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, leaderBrokerConfig, topic, partitionIndex)); } var partitionData = response.PartitionData(topic, partitionIndex); if (partitionData == null) { throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}", fetchOffset, leaderBrokerConfig, topic, partitionIndex)); } if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode) { s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}"; Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); return(null); } if (partitionData.Error != ErrorMapping.NoError) { s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}"; Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); return(null); } var messages = partitionData.MessageSet.Messages; s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}"; Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry); success = true; result = new PullResponse(partitionData); if (null != messages && messages.Count() > 0) { payloadCount = messages.Count(); long lastOffset = messages.Last().Offset; if ((payloadCount + fetchOffset) != (lastOffset + 1)) { s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}"; Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry); } offsetNew = messages.Last().Offset + 1; } return(result); } } catch (Exception) { if (retryCount >= maxRetry) { throw; } } finally { retryCount++; } } // end of while loop return(result); }
private async Task <T> CallGetRabbitMqApi <T>(BrokerConfiguration cfg, string call) where T : class { return(await HTTPCall(cfg, async() => await _httpConnector.Get <T>(call))); }
private void CreateProducerOfOnePartition(string topic, int partitionId, BrokerConfiguration broker, ProducerConfiguration producerConfigTemplate, bool forceRecreateEvenHostPortSame) { Logger.InfoFormat("CreateProducer == enter: Topic:{0} partitionId:{1} broker:{2} forceRecreateEvenHostPortSame:{3} ", topic, partitionId, broker, forceRecreateEvenHostPortSame); //Explicitly set partitionID ProducerConfiguration producerConfig = new ProducerConfiguration(producerConfigTemplate, new List <BrokerConfiguration> { broker }, partitionId) { ForceToPartition = partitionId }; if (!this.TopicPartitionsLeaderProducers.ContainsKey(topic)) { this.TopicPartitionsLeaderProducers.TryAdd(topic, new ConcurrentDictionary <int, Producer <TKey, TData> >()); } ConcurrentDictionary <int, Producer <TKey, TData> > dictPartitionLeaderProducersOfOneTopic = this.TopicPartitionsLeaderProducers[topic]; bool needRecreate = true; Producer <TKey, TData> oldProducer = null; if (dictPartitionLeaderProducersOfOneTopic.TryGetValue(partitionId, out oldProducer)) { if (oldProducer.Config.Brokers.Any() && oldProducer.Config.Brokers[0].Equals(producerConfig.Brokers[0])) { needRecreate = false; } } if (forceRecreateEvenHostPortSame) { needRecreate = true; } if (needRecreate) { lock (GetProduceLockOfTopic(topic, partitionId)) { if (dictPartitionLeaderProducersOfOneTopic.TryGetValue(partitionId, out oldProducer)) { if (oldProducer.Config.Brokers.Any() && oldProducer.Config.Brokers[0].Equals(producerConfig.Brokers[0])) { needRecreate = false; } } if (forceRecreateEvenHostPortSame) { needRecreate = true; } if (!needRecreate) { Logger.InfoFormat("CreateProducer == Add producer SKIP after got lock for topic {0} partition {1} since leader {2} not changed. Maybe created by other thread. END.", topic, partitionId, producerConfig.Brokers[0]); return; } bool removeOldProducer = false; if (oldProducer != null) { oldProducer.Dispose(); removeOldProducer = dictPartitionLeaderProducersOfOneTopic.TryRemove(partitionId, out oldProducer); Logger.InfoFormat("CreateProducer == Remove producer for topic {0} partition {1} leader {2} removeOldProducer:{3} ", topic, partitionId, oldProducer.Config.Brokers[0], removeOldProducer); } Producer <TKey, TData> producer = new Producer <TKey, TData>(producerConfig); bool addNewProducer = dictPartitionLeaderProducersOfOneTopic.TryAdd(partitionId, producer); Logger.InfoFormat("CreateProducer == Add producer for topic {0} partition {1} leader {2} SyncProducerOfOneBroker:{3} removeOldProducer:{4} addNewProducer:{5} END.", topic, partitionId, broker, producerConfig.SyncProducerOfOneBroker, removeOldProducer, addNewProducer); } } else { Logger.InfoFormat("CreateProducer == Add producer SKIP for topic {0} partition {1} since leader {2} not changed. END.", topic, partitionId, producerConfig.Brokers[0]); } }
public new void ProcessConfig(BrokerConfiguration currentConfiguration) { _managedByBrowserUrls.AddRange(currentConfiguration.StringArray[SectionName.ArrayManagedByBrowserUrl]?.Select(u => u.ToLower()) ?? new string[0]); }
/// <summary> /// Creates Store /// </summary> public Store(BrokerConfiguration configuration) { _configuration = configuration; Init(); }
//[System.Runtime.InteropServices.DllImport("kernel32.dll")] //private static extern bool AllocConsole(); //[System.Runtime.InteropServices.DllImport("kernel32.dll")] //private static extern bool AttachConsole(int pid); //[System.Runtime.InteropServices.DllImport("kernel32.dll", SetLastError = true)] //private static extern bool FreeConsole(); #endif /// <summary> /// Many runtime configuration declared, dump object model of it if required /// </summary> public void exportConfiguration() { File.WriteAllBytes("main.json", BrokerConfiguration.ExtractFromBroker(broker).SerialiseJson()); File.WriteAllBytes("modules.json", BrokerConfiguration.ExtractModulesFromBroker(broker).SerialiseJson()); File.WriteAllBytes("assemblies.json", BrokerConfiguration.ExtractAssemblysFromBroker(broker).SerialiseJson()); }
public void Connect(BrokerConfiguration configuration) { CreateConnection(configuration); }
public void ValidatorResolution() { var container = CreateContainer(); var meConfig = new SimpleMappingEngineConfiguration(container); meConfig.Configure(); var repository = new Mock<IRepository>(); container.RegisterInstance(repository.Object); var config = new BrokerConfiguration(container); config.Configure(); var validator = container.Resolve<IValidator<Broker>>("broker"); // Assert Assert.IsNotNull(validator, "Validator resolution failed"); }
public void ProcessConfig(BrokerConfiguration currentConfiguration) { }