public BrokerConfigurationBuilder()
{
this.brokerConfiguration = new BrokerConfiguration
{
Topics = new List <string>()
};
}
//public KafkaDispatcherMS(ProducerConfigManager producerConfigManager)
//{
// this._producerConfigManager = producerConfigManager;
//}
public virtual async Task UpsertDocument <TDocument>(IUpsertDocumentContext <TDocument> context) where TDocument : class
{
try
{
var topicName = context.Document.GetType().Name;
var brokerConfig = new BrokerConfiguration()
{
BrokerId = 0,
Host = "localhost",
Port = 9092
};
var config = new ProducerConfiguration(new List <BrokerConfiguration> {
brokerConfig
});
//var serializer = new BinarySerializer<TDocument>();
var json = this._serializer.Serialize(context.Document);
var content = Encoding.Default.GetBytes(json);
var msg = new Message(content);
var kafkaProducer = new Producer(config);
var batch = new ProducerData <string, Message>(topicName, msg);
kafkaProducer.Send(batch);
}
catch (Exception e)
{
}
}
private static void MicrosoftKafkaTest()
{
var brokerConfig = new BrokerConfiguration()
{
Host = "0.tcp.ngrok.io",
Port = 15069
};
var config = new ProducerConfiguration(new List <BrokerConfiguration> {
brokerConfig
});
var producer = new Kafka.Client.Producers.Producer(config);
var payload = Encoding.ASCII.GetBytes(@"
{
'method': 'POST',
'timeStamp': 1508164715,
'foo':
{
'id': 123
}
}");
var message = new Kafka.Client.Messages.Message(payload);
var data = new ProducerData <string, Kafka.Client.Messages.Message>("fun-topic", message);
try
{
producer.Send(data);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
private IList <BrokerConfiguration> GetBrokerConfigurations()
{
List <BrokerConfiguration> brokerConfigurations = new List <BrokerConfiguration>();
string brokers = ConfigurationManager.AppSettings["Brokers"];
string[] brokerArray = brokers.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries);
int port = String.IsNullOrEmpty(ConfigurationManager.AppSettings["BrokerPort"]) ? 8080 : Convert.ToInt32(ConfigurationManager.AppSettings["BrokerPort"]);
if (brokerArray.Count() <= 0)
{
Console.WriteLine("No brokers found!");
}
var broker = default(BrokerConfiguration);
for (int i = 0; i < brokerArray.Length; i++)
{
broker = new BrokerConfiguration()
{
//BrokerId = i + 1,
Host = brokerArray[i].Trim(),
Port = port
};
brokerConfigurations.Add(broker);
}
return(brokerConfigurations);
}
public TradeUtils(BrokerConfiguration brokerConfig, RuleConfiguration ruleConfig)
{
this.BrokerConfig = brokerConfig;
TradeVolumeCalculator = new TradeVolumeCalculator(ruleConfig);
OrderSpacingCalculator = new OrderSpacingCalculator(ruleConfig);
}
// public void CreateVHost(BrokerConfiguration configuration, Assembly callerAssembly)
// {
// AssemblyBuildInfoAttribute buildInfo = BuildHelper.GetBuildInfo(callerAssembly);
// if (buildInfo != null)
// {
// string vHostTestName = $@"{configuration.Vhost}_{buildInfo.BuildDate}_{buildInfo.BranchName.Replace('/', '_')}_{buildInfo.BuildNumber}";
// #if DEBUG
// vHostTestName = $@"{configuration.Vhost}_{Environment.MachineName}_{Environment.UserName}";
// #endif
// configuration.Vhost = CreateVHost(configuration, vHostTestName);
// }
// }
private T HTTPCall <T>(BrokerConfiguration cfg, Func <T> httpAction)
{
_httpConnector.Credentials = new NetworkCredential {
UserName = cfg.UserName, Password = cfg.Password
};
_httpConnector.Uri = new Uri($"http://{cfg.HostName}:1{cfg.Port}/");
return(httpAction());
}
private void CallPutRabbitMqApi(BrokerConfiguration cfg, string call, object body = null)
{
HTTPCall(cfg, () =>
{
var content = new StringContent(body != null ? _jsonSerializer.SerializeObject(body) : "", Encoding.UTF8, "application/json");
return(_httpConnector.Put <string>(call, content));
});
}
public void AddHost(BrokerConfiguration configuration, string vHostName)
{
if (!_vhosts.ContainsKey(configuration))
{
_vhosts[configuration] = new HashSet <string>();
}
_vhosts[configuration].Add(vHostName);
}
private void DeleteVHost(BrokerConfiguration configuration, string vHostName)
{
var vhosts = GetVHost(configuration);
string httpVHost = vHostName.Replace(@"/", @"%2f");
VHost vhost = vhosts.FirstOrDefault(vh => vh.Name == vHostName);
if (vhost == null)
{
return;
}
CallDeleteRabbitMqApi(configuration, $"api/vhosts/{httpVHost}");
}
private void CreateConnection(BrokerConfiguration configuration)
{
try
{
if (!IsConnect)
{
if (_factory == null)
{
_factory = new ConnectionFactory
{
HostName = configuration.HostName,
Port = configuration.Port,
UserName = configuration.UserName,
Password = configuration.Password,
VirtualHost = configuration.Vhost,
};
#if DEBUG
// Force to use Ipv4 instead of trying ipv6 and try catch exception in his code.
_factory.SocketFactory = family => ConnectionFactory.DefaultSocketFactory(AddressFamily.InterNetwork);
#endif
}
RegisteredConnectors.Clear();
_connection = _factory.CreateConnection($"{Environment.MachineName}:{Environment.UserName}");
_logger.LogInformation($"Connected on broker {_factory.HostName} - vHost: {_factory.VirtualHost}");
using (IModel channel = _connection.CreateModel())
{
CreateDeadLetter(channel);
}
_connection.ConnectionShutdown += OnConnectionShutdown;
_configuration = configuration;
}
if (IsConnect)
{
OnConnect?.Invoke(this, EventArgs.Empty);
}
}
catch (Exception e)
{
_logger.LogError(e, "Connection failed.");
}
}
public static void Main(string[] args)
{
var section = (BrokerConfigurationSection)ConfigurationManager.GetSection("brodBroker");
var configuration = BrokerConfiguration.FromConfigurationSection(section);
Console.WriteLine("Brod Broker, v0.0.0.0.1");
Console.WriteLine("-----------------------");
Console.WriteLine(" Storage Directory: {0}", configuration.StorageDirectory);
Console.WriteLine(" Default number of partitions: {0}", configuration.NumberOfPartitions);
//var socketServer = new SocketServer(configuration.ConsumerPort, configuration.ProducerPort, 1);
//socketServer.Startup();
var server = new Broker(configuration);
server.Start();
}
private string CreateVHost(BrokerConfiguration configuration, string vHostName)
{
var vhosts = GetVHost(configuration);
string httpVHost = vHostName.Replace(@"/", @"%2f");
VHost vhost = vhosts.FirstOrDefault(vh => vh.Name == vHostName);
if (vhost == null)
{
CallPutRabbitMqApi(configuration, $"api/vhosts/{httpVHost}");
CallPutRabbitMqApi(configuration, $"api/permissions/{httpVHost}/{configuration.UserName}", new Permission());
if (!_vhosts.ContainsKey(configuration))
{
_vhosts[configuration] = new HashSet <string>();
}
_vhosts[configuration].Add(vHostName);
}
return(vHostName);
}
public void ValidatorResolution()
{
var container = CreateContainer();
var meConfig = new SimpleMappingEngineConfiguration(container);
meConfig.Configure();
var repository = new Mock <IRepository>();
container.RegisterInstance(repository.Object);
var config = new BrokerConfiguration(container);
config.Configure();
var validator = container.Resolve <IValidator <Broker> >("broker");
// Assert
Assert.IsNotNull(validator, "Validator resolution failed");
}
/// <summary>
/// Convert a broker string into a BrokerConfiguration.
/// </summary>
/// <param name="brokerAddr">Broker string must follow the format of {Broker-ID},{Host}:{Port}</param>
/// <returns></returns>
public static BrokerConfiguration ToBrokerConfig(string brokerAddr)
{
if (string.IsNullOrEmpty(brokerAddr))
{
return(null);
}
var brokerParams = brokerAddr.Split(',');
if (brokerParams == null || brokerParams.Length != 2)
{
return(null);
}
var brokerId = -1;
if (!int.TryParse(brokerParams[0], out brokerId))
{
return(null);
}
var hostParams = brokerParams[1].Split(':');
if (hostParams == null || hostParams.Length != 2)
{
return(null);
}
var portNumber = -1;
if (!int.TryParse(hostParams[1], out portNumber))
{
return(null);
}
var broker = new BrokerConfiguration {
BrokerId = brokerId, Host = hostParams[0], Port = portNumber
};
return(broker);
}
public override bool LoadResources()
{
var brokerConfig = new BrokerConfiguration()
{
BrokerId = BrokerId,
Host = Host,
Port = Port
};
var config = new ProducerConfiguration(new List <BrokerConfiguration>()
{
brokerConfig
})
{
PartitionerClass = typeof(ProducerKeyPatition).AssemblyQualifiedName
};
_producer = new ProducerDevId(config);
return(true);
}
public BrokerFactory(BrokerConfiguration brokerConfiguration)
{
this.brokerConfiguration = brokerConfiguration;
}
private IEnumerable <VHost> GetVHost(BrokerConfiguration cfg) => CallGetRabbitMqApi <IEnumerable <VHost> >(cfg, "api/vhosts").Result;
private void CallDeleteRabbitMqApi(BrokerConfiguration cfg, string call) => HTTPCall(cfg, () => _httpConnector.Delete <string>(call));
public static PullResponse PullMessage(
ConsumerConfiguration consumerConfig,
BrokerConfiguration leaderBrokerConfig,
int correlationID,
string topic,
int partitionIndex,
long fetchOffset,
KafkaClientHelperConfiguration helperConfiguration,
out long offsetNew)
{
offsetNew = -1;
PullResponse result = null;
int payloadCount = 0;
// at least retry once
int maxRetry = 1;
int retryCount = 0;
string s = string.Empty;
bool success = false;
while (!success && retryCount < maxRetry)
{
try
{
var requestMap = new Dictionary <string, List <PartitionFetchInfo> >();
requestMap.Add(
topic,
new List <PartitionFetchInfo>()
{
new PartitionFetchInfo(
partitionIndex,
fetchOffset,
helperConfiguration.FetchSize)
});
using (Consumer consumer = new Consumer(consumerConfig, leaderBrokerConfig.Host, leaderBrokerConfig.Port))
{
var response = consumer.Fetch(new FetchRequest(
correlationID, //random.Next(int.MinValue, int.MaxValue), // correlation id
Assembly.GetExecutingAssembly().ManifestModule.ToString(), // client id
helperConfiguration.MaxWaitTime,
helperConfiguration.MinWaitBytes,
requestMap));
if (response == null)
{
throw new KeyNotFoundException(string.Format("FetchRequest returned null response,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, leaderBrokerConfig, topic, partitionIndex));
}
var partitionData = response.PartitionData(topic, partitionIndex);
if (partitionData == null)
{
throw new KeyNotFoundException(string.Format("PartitionData is null,fetchOffset={0},leader={1},topic={2},partition={3}",
fetchOffset, leaderBrokerConfig, topic, partitionIndex));
}
if (partitionData.Error == ErrorMapping.OffsetOutOfRangeCode)
{
s = "PullMessage OffsetOutOfRangeCode,change to Latest,topic={0},leader={1},partition={2},FetchOffset={3},retryCount={4},maxRetry={5}";
Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
return(null);
}
if (partitionData.Error != ErrorMapping.NoError)
{
s = "PullMessage ErrorCode={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.ErrorFormat(s, partitionData.Error, topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
return(null);
}
var messages = partitionData.MessageSet.Messages;
s = "PullMessage AfterFetch,resultMessageCount={0},topic={1},leader={2},partition={3},FetchOffset={4},retryCount={5},maxRetry={6}";
Logger.DebugFormat(s, null == messages ? "(null)" : messages.Count().ToString(), topic, leaderBrokerConfig, partitionIndex, fetchOffset, retryCount, maxRetry);
success = true;
result = new PullResponse(partitionData);
if (null != messages && messages.Count() > 0)
{
payloadCount = messages.Count();
long lastOffset = messages.Last().Offset;
if ((payloadCount + fetchOffset) != (lastOffset + 1))
{
s = "PullMessage offset payloadCount out-of-sync,topic={0},leader={1},partition={2},payloadCount={3},FetchOffset={4},lastOffset={5},retryCount={6},maxRetry={7}";
Logger.ErrorFormat(s, topic, leaderBrokerConfig, partitionIndex, payloadCount, fetchOffset, lastOffset, retryCount, maxRetry);
}
offsetNew = messages.Last().Offset + 1;
}
return(result);
}
}
catch (Exception)
{
if (retryCount >= maxRetry)
{
throw;
}
}
finally
{
retryCount++;
}
} // end of while loop
return(result);
}
private async Task <T> CallGetRabbitMqApi <T>(BrokerConfiguration cfg, string call)
where T : class
{
return(await HTTPCall(cfg, async() => await _httpConnector.Get <T>(call)));
}
private void CreateProducerOfOnePartition(string topic, int partitionId, BrokerConfiguration broker, ProducerConfiguration producerConfigTemplate, bool forceRecreateEvenHostPortSame)
{
Logger.InfoFormat("CreateProducer == enter: Topic:{0} partitionId:{1} broker:{2} forceRecreateEvenHostPortSame:{3} ", topic, partitionId, broker, forceRecreateEvenHostPortSame);
//Explicitly set partitionID
ProducerConfiguration producerConfig = new ProducerConfiguration(producerConfigTemplate, new List <BrokerConfiguration> {
broker
}, partitionId)
{
ForceToPartition = partitionId
};
if (!this.TopicPartitionsLeaderProducers.ContainsKey(topic))
{
this.TopicPartitionsLeaderProducers.TryAdd(topic, new ConcurrentDictionary <int, Producer <TKey, TData> >());
}
ConcurrentDictionary <int, Producer <TKey, TData> > dictPartitionLeaderProducersOfOneTopic = this.TopicPartitionsLeaderProducers[topic];
bool needRecreate = true;
Producer <TKey, TData> oldProducer = null;
if (dictPartitionLeaderProducersOfOneTopic.TryGetValue(partitionId, out oldProducer))
{
if (oldProducer.Config.Brokers.Any() &&
oldProducer.Config.Brokers[0].Equals(producerConfig.Brokers[0]))
{
needRecreate = false;
}
}
if (forceRecreateEvenHostPortSame)
{
needRecreate = true;
}
if (needRecreate)
{
lock (GetProduceLockOfTopic(topic, partitionId))
{
if (dictPartitionLeaderProducersOfOneTopic.TryGetValue(partitionId, out oldProducer))
{
if (oldProducer.Config.Brokers.Any() &&
oldProducer.Config.Brokers[0].Equals(producerConfig.Brokers[0]))
{
needRecreate = false;
}
}
if (forceRecreateEvenHostPortSame)
{
needRecreate = true;
}
if (!needRecreate)
{
Logger.InfoFormat("CreateProducer == Add producer SKIP after got lock for topic {0} partition {1} since leader {2} not changed. Maybe created by other thread. END.", topic, partitionId, producerConfig.Brokers[0]);
return;
}
bool removeOldProducer = false;
if (oldProducer != null)
{
oldProducer.Dispose();
removeOldProducer = dictPartitionLeaderProducersOfOneTopic.TryRemove(partitionId, out oldProducer);
Logger.InfoFormat("CreateProducer == Remove producer for topic {0} partition {1} leader {2} removeOldProducer:{3} ", topic, partitionId, oldProducer.Config.Brokers[0], removeOldProducer);
}
Producer <TKey, TData> producer = new Producer <TKey, TData>(producerConfig);
bool addNewProducer = dictPartitionLeaderProducersOfOneTopic.TryAdd(partitionId, producer);
Logger.InfoFormat("CreateProducer == Add producer for topic {0} partition {1} leader {2} SyncProducerOfOneBroker:{3} removeOldProducer:{4} addNewProducer:{5} END.", topic, partitionId, broker, producerConfig.SyncProducerOfOneBroker, removeOldProducer, addNewProducer);
}
}
else
{
Logger.InfoFormat("CreateProducer == Add producer SKIP for topic {0} partition {1} since leader {2} not changed. END.", topic, partitionId, producerConfig.Brokers[0]);
}
}
public new void ProcessConfig(BrokerConfiguration currentConfiguration)
{
_managedByBrowserUrls.AddRange(currentConfiguration.StringArray[SectionName.ArrayManagedByBrowserUrl]?.Select(u => u.ToLower()) ?? new string[0]);
}
/// <summary>
/// Creates Store
/// </summary>
public Store(BrokerConfiguration configuration)
{
_configuration = configuration;
Init();
}
//[System.Runtime.InteropServices.DllImport("kernel32.dll")]
//private static extern bool AllocConsole();
//[System.Runtime.InteropServices.DllImport("kernel32.dll")]
//private static extern bool AttachConsole(int pid);
//[System.Runtime.InteropServices.DllImport("kernel32.dll", SetLastError = true)]
//private static extern bool FreeConsole();
#endif
/// <summary>
/// Many runtime configuration declared, dump object model of it if required
/// </summary>
public void exportConfiguration()
{
File.WriteAllBytes("main.json", BrokerConfiguration.ExtractFromBroker(broker).SerialiseJson());
File.WriteAllBytes("modules.json", BrokerConfiguration.ExtractModulesFromBroker(broker).SerialiseJson());
File.WriteAllBytes("assemblies.json", BrokerConfiguration.ExtractAssemblysFromBroker(broker).SerialiseJson());
}
public void Connect(BrokerConfiguration configuration)
{
CreateConnection(configuration);
}
public void ValidatorResolution()
{
var container = CreateContainer();
var meConfig = new SimpleMappingEngineConfiguration(container);
meConfig.Configure();
var repository = new Mock<IRepository>();
container.RegisterInstance(repository.Object);
var config = new BrokerConfiguration(container);
config.Configure();
var validator = container.Resolve<IValidator<Broker>>("broker");
// Assert
Assert.IsNotNull(validator, "Validator resolution failed");
}
public void ProcessConfig(BrokerConfiguration currentConfiguration)
{
}
