void t_Elapsed()
{
NetMQContext ctx = NetMQContext.Create();
NetMQSocket socket = ctx.CreateSubscriberSocket();
// TODO: change socket connection ip, port
socket.Connect("tcp://172.16.12.188:5555");
socket.Subscribe(_tagID.ToString("000"));
socket.ReceiveReady += (s, a) =>
{
string topic = socket.ReceiveString();
string message = socket.ReceiveString();
msg m = JsonConvert.DeserializeObject <msg>(message);
WriteText(m);
//Application.DoEvents();
};
Poller poller = new Poller();
poller.AddSocket(socket);
poller.Start();
}
/// <summary>
/// Handling replies to a data push, a historical data request, or an available data request
/// </summary>
private void _dealerSocket_ReceiveReady(object sender, NetMQSocketEventArgs e)
{
lock (_dealerSocketLock)
{
//1st message part: what kind of stuff we're receiving
string type = _dealerSocket.ReceiveString();
switch (type)
{
case "PUSHREP":
HandleDataPushReply();
break;
case "HISTREQREP":
HandleHistoricalDataRequestReply();
break;
case "AVAILABLEDATAREP":
HandleAvailabledataReply();
break;
case "ERROR":
HandleErrorReply();
break;
}
}
}
static void OnReceived(NetMQSocket subscriber_socket)
{
while (true)
{
string song = subscriber_socket.ReceiveString();
string notes = subscriber_socket.ReceiveString();
Blowed(song, notes);
Console.WriteLine("[subcriber_socket] Received: song: {0}, notes: {1}", song, notes);
}
}
public static string SendRequest(string request, int timeout = 5000)
{
string joshuaHost = "10.30.16.16";
string joshuaPort = "2804";
//logHandler.WriteLog(string.Format("Sent request: {0}", request));
using (NetMQContext context = NetMQContext.Create())
{
using (NetMQSocket clientSocket = context.CreateRequestSocket())
{
try
{
clientSocket.Connect(string.Format("tcp://{0}:{1}", joshuaHost, joshuaPort));
clientSocket.Options.ReceiveTimeout = TimeSpan.FromMilliseconds(timeout);
clientSocket.Send(request);
//logHandler.WriteLog(string.Format("Returning {0} bytes", bytes.Length));
return(clientSocket.ReceiveString());
}
catch (Exception e)
{
Console.WriteLine(string.Format("Exception getting data: {0}", e));
return(null);
}
}
}
}
public void DoWork()
{
using (var context = NetMQContext.Create())
{
using (NetMQSocket receiver = context.CreatePullSocket(),
sender = context.CreatePushSocket())
{
receiver.Connect(_receiverAddress);
sender.Connect(_senderAddress);
// Allows us to remotely kill the process
while (!Token.IsCancellationRequested)
{
string taskString = receiver.ReceiveString();
var tasks = taskString.Split(',');
var job = tasks[0]; // First index is Job id
var result = new JobResult(job);
// Our 'work'
foreach (var word in tasks.Skip(1).Distinct())
{
result.Data[word] = tasks.Count(p => p == word);
}
// Send 'result' to the sink
sender.Send(JsonConvert.SerializeObject(result));
}
}
}
}
public void Start()
{
this.Log().Info("启动状态计算服务");
_context = NetMQContext.Create();
_socket = _context.CreateSocket(ZmqSocketType.Rep);
_socket.Connect(ConfigurationManager.AppSettings["broker"]);
_running = true;
while (_running)
{
this.Log().Info("开始接收消息");
var message = _socket.ReceiveString();
this.Log().Info("接收消息：{" + message + "}");
var taskMesg = JsonConvert.DeserializeObject <IndexTask>(message);
var objType = taskMesg.type;
var objCode = taskMesg.code;
var cycleType = taskMesg.cycle;// (TechCycle)Enum.Parse(typeof(TechCycle), taskMesg.cycle, true);
var tech = new TechCalculate(objType, cycleType, objCode, null);
tech.CalculateState();
this.Log().Info("处理完成：" + message);
_socket.Send("处理完成：{" + message + "}");
}
}
public void BindToLocal()
{
var validAliasesForLocalHost = new[] { "127.0.0.1", "localhost", System.Net.Dns.GetHostName() };
foreach (var alias in validAliasesForLocalHost)
{
using (NetMQContext context = NetMQContext.Create())
{
using (NetMQSocket localDealer = context.CreateDealerSocket())
{
localDealer.Bind("tcp://*:5002");
using (NetMQSocket connectingDealer = context.CreateDealerSocket())
{
connectingDealer.Connect("tcp://" + alias + ":5002");
localDealer.Send("test");
Assert.AreEqual("test", connectingDealer.ReceiveString());
Console.WriteLine(alias + " connected ");
}
}
}
}
}
static string Client(NetMQContext context, string endpoint, string requestText)
{
string answer = "";
using (NetMQSocket clientSocket = context.CreateRequestSocket())
{
clientSocket.Connect(endpoint);
// while (true)
// {
// Console.WriteLine("Please enter your message:");
clientSocket.Send(requestText);
answer = clientSocket.ReceiveString();
// Console.WriteLine("Answer from server: {0}", answer);
// if (message == "exit")
// {
// break;
// }
// }
}
return(answer);
}
public void Execute(IJobExecutionContext context)
{
LoggingExtensions.Logging.Log.InitializeWith <LoggingExtensions.log4net.Log4NetLog>();
using (NetMQContext nmContext = NetMQContext.Create())
using (NetMQSocket socket = nmContext.CreateSocket(ZmqSocketType.Rep))
{
socket.Connect("tcp://localhost:5566");
while (true)
{
var message = socket.ReceiveString();
var taskMesg = JsonConvert.DeserializeObject <IndexTask>(message);
var objType = taskMesg.type;
var objCode = taskMesg.code;
var cycleType = taskMesg.cycle;// (TechCycle)Enum.Parse(typeof(TechCycle), taskMesg.cycle, true);
IIndexService indexService = new IndexService();
//获取数据
var priceList = indexService.GetObjectData(objType.ToString(), cycleType, objCode);
//计算指数结果
//存储指数到MangoDB
//更新状态数据到MangoDB
var tech = new TechCalculate(objType, cycleType, objCode, priceList);
tech.Run();
}
}
}
void _reqSocket_ReceiveReady(object sender, NetMQSocketEventArgs e)
{
string requestType = _reqSocket.ReceiveString();
if (requestType == null)
{
return;
}
//handle ping requests
if (requestType == "PING")
{
_reqSocket.Send("PONG");
return;
}
//Handle real time data requests
if (requestType == "RTD") //Two part message: first, "RTD" string. Then the RealTimeDataRequest object.
{
HandleRTDataRequest();
}
//manage cancellation requests
//two part message: first: "CANCEL". Second: the instrument
if (requestType == "CANCEL")
{
HandleRTDataCancelRequest();
}
}
public void TechTask()
{
using (NetMQContext _context = NetMQContext.Create())
{
using (NetMQSocket _socket = _context.CreateSocket(ZmqSocketType.Req))
{
_socket.Connect(ConfigurationManager.AppSettings["broker"]);
while (true)
{
IndexTask task = null;
lock (this.TaskQueue)
{
if (this.TaskQueue.Count > 0)
{
task = this.TaskQueue.Dequeue();
this.Log().Info("remain:" + this.TaskQueue.Count);
}
}
if (task == null)
{
Thread.Sleep(5000);
break;
}
string msg = JsonConvert.SerializeObject(task);
byte[] msgBytes = Encoding.UTF8.GetBytes(msg);
_socket.Send(msg);
this.Log().Info("send:{" + msg + "}");
string message = _socket.ReceiveString();
this.Log().Info("rcv:" + message);
}
}
}
}
public static void Client(NetMQContext context)
{
using (NetMQSocket clientSocket = context.CreateRequestSocket())
{
clientSocket.Connect("tcp://127.0.0.1:5555");
JavaScriptSerializer seriallizer = null;
while (true)
{
seriallizer = new JavaScriptSerializer();
Console.WriteLine("Please enter your message:");
string msg = Console.ReadLine();
NetMQMessage message = new NetMQMessage();
var msgBytes = Common.ByteConvertHelp.Object2Bytes(msg);
clientSocket.Send(msgBytes);
string answer = clientSocket.ReceiveString();
Console.WriteLine("Answer from server: {0}", answer);
if (msg == "exit")
{
break;
}
}
}
}
public void Start()
{
this.Log().Info("启动计算服务");
_context = NetMQContext.Create();
_socket = _context.CreateSocket(ZmqSocketType.Rep);
_socket.Connect(ConfigurationManager.AppSettings["broker"]);
_running = true;
while (_running)
{
this.Log().Info("开始接收消息");
var message = _socket.ReceiveString();
this.Log().Info("接收消息：{" + message + "}");
var taskMesg = JsonConvert.DeserializeObject <IndexTask>(message);
var objType = taskMesg.type;
var objCode = taskMesg.code;
var cycleType = taskMesg.cycle;// (TechCycle)Enum.Parse(typeof(TechCycle), taskMesg.cycle, true);
//var date = taskMesg.date;
IIndexService indexService = new IndexService();
//获取数据
var priceList = indexService.GetObjectData(objType.ToString(), cycleType, objCode);
//计算指数结果
//存储指数到MangoDB
//更新状态数据到MangoDB
if (priceList.Count > 0)
{
var tech = new TechCalculate(objType, cycleType, objCode, priceList);
tech.Run();
}
this.Log().Info("处理完成");
_socket.Send("处理完成：{" + message + "}");
}
}
protected override Message SendInternal(Message message)
{
var json = message.ToJsonString();
socket.Send(json);
var response = socket.ReceiveString();
return(Message.FromJson(response));
}
public async Task ReceiveMessage()
{
Task task = new Task(() =>
{
var result = clientSocket.ReceiveString();
Console.WriteLine("REPLY " + result);
});
task.Start(scheduler);
await task;
}
/// <summary>
/// Listens the specified on message received.
/// </summary>
/// <param name="onMessageReceived">The on message received.</param>
/// <param name="token">The tokenSource.</param>
internal void Listen(Action <Message> onMessageReceived, CancellationTokenSource token)
{
socket.ReceiveReady += (sender, args) =>
{
var inbound = socket.ReceiveString();
var message = Message.FromJson(inbound);
onMessageReceived(message);
};
while (!token.IsCancellationRequested)
{
socket.Poll();
}
}
public static void Blow(string song, string notes)
{
new Thread(() => {
using (NetMQSocket request_socket = context.CreateRequestSocket()) {
request_socket.Connect(string.Format("tcp://{0}:{1}", Address, RequestPort));
request_socket.SendMore("blow").SendMore(song).Send(notes);
string response = request_socket.ReceiveString();
Console.WriteLine("[request_socket] Response: {0}", response);
request_socket.Close();
request_socket.Dispose();
}
}).Start();
}
static void Main(string[] args)
{
using (NetMQContext context = NetMQContext.Create())
{
using (NetMQSocket response = context.CreateResponseSocket())
{
string address = GetComputerLanIP();
if (PORT_NUMBER > 0)
{
if (!string.IsNullOrEmpty(address))
{
Console.WriteLine("Binding tcp://{0}:{1}", address, PORT_NUMBER);
response.Bind(string.Format("tcp://{0}:{1}", address, PORT_NUMBER));
while (true)
{
bool hasMore = true;
string msg = response.ReceiveString(out hasMore);
if (string.IsNullOrEmpty(msg))
{
Console.WriteLine("No msg received.");
break;
}
Console.WriteLine("Msg received! {0}", msg);
response.Send(msg, false, hasMore);
Thread.Sleep(1000);
}
response.Options.Linger = TimeSpan.Zero;
}
else
{
Console.WriteLine("Wrong IP address");
}
}
else
{
Console.WriteLine("The port number should be greater than 0");
}
Console.WriteLine("Press ENTER to exit...");
Console.ReadLine();
}
}
}
private void Send(string objCode, data.TechCycle cycle)
{
string msg = JsonConvert.SerializeObject(new IndexTask
{
type = ObjectType.Object,
code = objCode,
cycle = cycle
});
byte[] msgBytes = Encoding.UTF8.GetBytes(msg);
_socket.Send(msg);
this.Log().Info("发送:{" + msg + "}");
string message = _socket.ReceiveString();
this.Log().Info("接收:" + message);
}
public void Ipv6ToIpv4()
{
using (NetMQContext context = NetMQContext.Create())
{
using (NetMQSocket localDealer = context.CreateDealerSocket())
{
localDealer.Options.IPv4Only = false;
localDealer.Bind(string.Format("tcp://*:5002"));
using (NetMQSocket connectingDealer = context.CreateDealerSocket())
{
connectingDealer.Connect("tcp://" + IPAddress.Loopback.ToString() + ":5002");
connectingDealer.Send("test");
Assert.AreEqual("test", localDealer.ReceiveString());
}
}
}
}
public void BindRandom()
{
using (NetMQContext context = NetMQContext.Create())
{
using (NetMQSocket randomDealer = context.CreateDealerSocket())
{
int port = randomDealer.BindRandomPort("tcp://*");
using (NetMQSocket connectingDealer = context.CreateDealerSocket())
{
connectingDealer.Connect("tcp://127.0.0.1:" + port);
randomDealer.Send("test");
Assert.AreEqual("test", connectingDealer.ReceiveString());
}
}
}
}
public void HasInTest()
{
using (NetMQContext context = NetMQContext.Create())
{
using (NetMQSocket server = context.CreateRouterSocket())
{
server.Bind("tcp://*:5557");
// no one sent a message so it should be fasle
Assert.IsFalse(server.HasIn);
using (NetMQSocket client = context.CreateDealerSocket())
{
client.Connect("tcp://localhost:5557");
// wait for the client to connect
Thread.Sleep(100);
// now we have one client connected but didn't send a message yet
Assert.IsFalse(server.HasIn);
client.Send("1");
// wait for the message to arrive
Thread.Sleep(100);
// the has in should indicate a message is ready
Assert.IsTrue(server.HasIn);
byte[] identity = server.Receive();
string message = server.ReceiveString();
Assert.AreEqual(message, "1");
// we read the message, it should false again
Assert.IsFalse(server.HasIn);
}
}
}
}
private static void StartServerNetMq(NetMQContext context)
{
using (NetMQSocket serverSocket = context.CreateResponseSocket())
{
serverSocket.Options.SendTimeout = TimeSpan.FromMilliseconds(60000);
serverSocket.Bind(string.Format("tcp://*:{0}", 2804));
string message = string.Empty;
string retMsg = string.Empty;
Console.WriteLine("Server started;");
while (true)
{
//log.WriteLog("Received command {0}, {1} bytes returned.", message, results.Length);
try
{
message = serverSocket.ReceiveString();
//Console.WriteLine("Message: {0} received.", message);
if (message == "Who's there?")
{
TimeSpan timeSinceLast = DateTime.Now - lastVerification;
//if (verificationState == VerificationState.Verifying && timeSinceLast.TotalSeconds > 15)
// name = string.Empty;
retMsg = name;
}
serverSocket.Send(retMsg);
}
catch (Exception e)
{
Console.WriteLine("Error sending data; message: {0}, error: {1}", message, e);
serverSocket.Send(string.Empty);
}
}
}
}
/// <summary>
/// Add an instrument to QDMS.
/// </summary>
/// <param name="instrument"></param>
/// <returns>The instrument with its ID set if successful, null otherwise.</returns>
public Instrument AddInstrument(Instrument instrument)
{
if (!Connected)
{
RaiseEvent(Error, this, new ErrorArgs(-1, "Could not add instrument - not connected."));
return(null);
}
if (instrument == null)
{
RaiseEvent(Error, this, new ErrorArgs(-1, "Could not add instrument - instrument is null."));
return(null);
}
using (NetMQSocket s = _context.CreateSocket(ZmqSocketType.Req))
{
s.Connect(string.Format("tcp://{0}:{1}", _host, _instrumentServerPort));
var ms = new MemoryStream();
s.SendMore("ADD"); //first we send an "ADD" request
//then we need to serialize and send the instrument
s.Send(MyUtils.ProtoBufSerialize(instrument, ms));
//then get the reply
string result = s.ReceiveString(TimeSpan.FromSeconds(1));
if (result != "SUCCESS")
{
RaiseEvent(Error, this, new ErrorArgs(-1, "Instrument addition failed: received no reply."));
return(null);
}
//Addition was successful, receive the instrument and return it
byte[] serializedInstrument = s.Receive();
return(MyUtils.ProtoBufDeserialize <Instrument>(serializedInstrument, ms));
}
}
public void Start()
{
using (NetMQSocket response_socket = context.CreateResponseSocket())
using (NetMQSocket publisher_socket = context.CreateXPublisherSocket()) {
string response_address = string.Format("tcp://{0}:{1}", Address, ResponsePort);
string publisher_address = string.Format("tcp://{0}:{1}", Address, PublisherPort);
response_socket.Bind(response_address);
publisher_socket.Bind(publisher_address);
Console.WriteLine("[response_socket] Bound on {0}", response_address);
Console.WriteLine("[publisher_socket] Bound on {0}", publisher_address);
using (Poller poller = new Poller(response_socket, publisher_socket)) {
response_socket.ReceiveReady += delegate(object sender, NetMQSocketEventArgs args) {
string message = response_socket.ReceiveString();
if (message.StartsWith("blow"))
{
string song = "";
string notes = "";
try {
song = response_socket.ReceiveString();
notes = response_socket.ReceiveString();
if (song.Length > 64)
{
song = song.Substring(0, 64);
}
if (notes.Length > 64)
{
notes = notes.Substring(0, 64);
}
cache [song] = notes;
Console.WriteLine("[response_socket] Received: song: {0}, notes: {1}", song, notes);
} catch (Exception e) {
Console.WriteLine("[response_socket] Invalid request: {0}", e.Message);
}
response_socket.Send("OK");
Console.WriteLine("[response_socket] Sent: OK");
publisher_socket.SendMore(song).Send(notes);
Console.WriteLine("[publisher_socket] Sent: song: {0}, notes: {1}", song, notes);
return;
}
if (message.Equals("ping"))
{
Console.WriteLine("[response_socket] Received: {0}", message);
int timestamp = (int)DateTime.UtcNow.Subtract(new DateTime(1970, 1, 1)).TotalSeconds;
response_socket.Send(timestamp.ToString());
Console.WriteLine("[response_socket] Sent: {0}", timestamp);
return;
}
Console.WriteLine("[response_socket] Invalid request: {0}", message);
args.Socket.Send("Meow?");
};
// Send cached notes to new subscribers
publisher_socket.ReceiveReady += delegate(object sender, NetMQSocketEventArgs args) {
NetMQMessage message = publisher_socket.ReceiveMessage();
// Subscribe == 1, Unsubscibe == 0
if (message.First.Buffer [0] != 1)
{
return;
}
string song = message.First.ConvertToString().Substring(1);
string cached_notes = (string)cache [song];
if (cached_notes != null)
{
publisher_socket.SendMore(song).Send(cached_notes);
}
};
poller.Start();
}
}
}
void _socket_ReceiveReady(object sender, NetMQSocketEventArgs e)
{
var ms = new MemoryStream();
List <Instrument> instruments;
bool hasMore;
string request = _socket.ReceiveString(SendReceiveOptions.DontWait, out hasMore);
if (request == null)
{
return;
}
//if the request is for a search, receive the instrument w/ the search parameters and pass it to the searcher
if (request == "SEARCH" && hasMore)
{
byte[] buffer = _socket.Receive();
var searchInstrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms);
Log(LogLevel.Info, string.Format("Instruments Server: Received search request: {0}",
searchInstrument));
try
{
instruments = _instrumentManager.FindInstruments(null, searchInstrument);
}
catch (Exception ex)
{
Log(LogLevel.Error, string.Format("Instruments Server: Instrument search error: {0}",
ex.Message));
instruments = new List <Instrument>();
}
}
else if (request == "ALL") //if the request is for all the instruments, we don't need to receive anything else
{
Log(LogLevel.Info, "Instruments Server: received request for list of all instruments.");
instruments = _instrumentManager.FindInstruments();
}
else if (request == "ADD" && hasMore) //request to add instrument
{
byte[] buffer = _socket.Receive();
var instrument = MyUtils.ProtoBufDeserialize <Instrument>(buffer, ms);
Log(LogLevel.Info, string.Format("Instruments Server: Received instrument addition request. Instrument: {0}",
instrument));
Instrument addedInstrument;
try
{
addedInstrument = _instrumentManager.AddInstrument(instrument);
}
catch (Exception ex)
{
addedInstrument = null;
Log(LogLevel.Error, string.Format("Instruments Server: Instrument addition error: {0}",
ex.Message));
}
_socket.SendMore(addedInstrument != null ? "SUCCESS" : "FAILURE");
_socket.Send(MyUtils.ProtoBufSerialize(addedInstrument, ms));
return;
}
else //no request = loop again
{
return;
}
byte[] uncompressed = MyUtils.ProtoBufSerialize(instruments, ms); //serialize the list of instruments
ms.Read(uncompressed, 0, (int)ms.Length); //get the uncompressed data
byte[] result = LZ4Codec.Encode(uncompressed, 0, (int)ms.Length); //compress it
//before we send the result we must send the length of the uncompressed array, because it's needed for decompression
_socket.SendMore(BitConverter.GetBytes(uncompressed.Length));
//then finally send the results
_socket.Send(result);
}
public void OnDataReady()
{
switch (m_state)
{
case WebSocketClientState.Closed:
m_state = WebSocketClientState.Handshake;
string clientHandshake = m_streamSocket.ReceiveString();
string[] lines = clientHandshake.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
string key;
if (ValidateClientHandshake(lines, out key))
{
string acceptKey = GenerateAcceptKey(key);
try
{
m_streamSocket.SendMore(Identity, Identity.Length, true);
m_streamSocket.Send("HTTP/1.1 101 Switching Protocols\r\n" +
"Upgrade: websocket\r\n" +
"Connection: Upgrade\r\n" +
"Sec-WebSocket-Accept: " + acceptKey + "\r\n" +
"Sec-WebSocket-Protocol: WSNetMQ\r\n\r\n");
m_decoder = new Decoder();
m_decoder.Message += OnMessage;
m_state = WebSocketClientState.Ready;
}
catch (NetMQException)
{
m_state = WebSocketClientState.Closed;
}
}
else
{
m_state = WebSocketClientState.Closed;
try
{
m_streamSocket.SendMore(Identity, Identity.Length, true);
m_streamSocket.Send("HTTP/1.1 400 Bad Request\r\nSec-WebSocket-Version: 13\r\n");
// invalid request, close the socket and raise closed event
m_streamSocket.SendMore(Identity, Identity.Length, true);
m_streamSocket.Send("");
}
catch (NetMQException ex)
{
}
}
break;
case WebSocketClientState.Ready:
byte[] message = m_streamSocket.Receive();
m_decoder.Process(message);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
private static IList<BehaviorInfo> GetBehaviorModules2(NetMQSocket socket,
IList<BehaviorInfo> behaviorList)
{
var ret = new List<BehaviorInfo>();
var dict = new Dictionary<string, BehaviorInfo>();
//behavior_modules
if (socket != null && behaviorList != null && behaviorList.Count > 0)
{
socket.Send("behavior_modules");
var resp = socket.ReceiveString(new TimeSpan(0, 0, 0, 0, RecvTimeout));
if (!string.IsNullOrEmpty(resp))
{
var modules = JArray.Parse(resp);
if (modules != null && modules.Count > 0)
{
//Console.WriteLine(resp);
foreach (var module in modules)
{
var moduleName = module.Value<string>("name");
var responder = module.SelectToken("$.responder");
string host = string.Empty;
int port = 0;
if (responder != null)
{
host = responder.Value<string>("Host");
port = responder.Value<int>("Port");
}
var behaviors = module.SelectToken("$.behaviors");
foreach (var behavior in behaviors)
{
string name = behavior.Value<string>("name");
string functionName = behavior.Value<string>("function_name");
var parameters = new Dictionary<string, object>();
var args = behavior.SelectToken("$.arg");
foreach (var arg in args)
{
parameters.Add(arg.Value<string>("name"), new Dictionary<string, object>
{
{"value", arg.Value<string>("value")},
{"place_holder", arg.Value<string>("place_holder")},
{"type", arg.Value<string>("type")}
});
}
if (!dict.ContainsKey(name))
{
dict.Add(name, new BehaviorInfo
{
BehaviorName = name,
FunctionName = functionName,
Ip = host,
Port = port,
Parameters = parameters
});
}
}
}
}
}
if (dict.Count > 0)
{
// ReSharper disable once LoopCanBeConvertedToQuery
foreach (var item in behaviorList)
{
Log.InfoFormat("Before : {0}",item.ToString());
if (dict.ContainsKey(item.BehaviorName))
{
var temp = dict[item.BehaviorName].Clone() as BehaviorInfo;
if (temp != null)
{
foreach (var parameter in item.Parameters)
{
if (!temp.Parameters.ContainsKey(parameter.Key))
{
temp.Parameters.Add(parameter);
}
else
{
temp.Parameters[parameter.Key] = parameter.Value;
}
}
Log.InfoFormat("After : {0}", temp.ToString());
ret.Add(temp);
}
//ret.Add(dict[item.BehaviorName]);
// Update the values of the parameter list with the once originally parsed from the xml file
//foreach (var parameter in item.Parameters)
//{
// if (!dict[item.BehaviorName].Parameters.ContainsKey(parameter.Key))
// {
// dict[item.BehaviorName].Parameters.Add(parameter);
// }
// else
// {
// dict[item.BehaviorName].Parameters[parameter.Key] = parameter.Value;
// }
//}
//ret.Add(dict[item.BehaviorName]);
}
}
}
}
return ret;
}
private static IList<BehaviorInfo> GetBehaviorModules(NetMQSocket socket,
IList<BehaviorInfo> behaviorList)
{
var ret = new List<BehaviorInfo>();
var dict = new Dictionary<string, BehaviorInfo>();
//behavior_modules
if (socket != null && behaviorList != null && behaviorList.Count > 0)
{
ret.AddRange(behaviorList);
socket.Send("behavior_modules");
var resp = socket.ReceiveString(new TimeSpan(0, 0, 0, 0, RecvTimeout));
if (!string.IsNullOrEmpty(resp))
{
var modules = JArray.Parse(resp);
if (modules != null && modules.Count > 0)
{
//Console.WriteLine(resp);
foreach (var module in modules)
{
var moduleName = module.Value<string>("name");
var responder = module.SelectToken("$.responder");
string host = string.Empty;
int port = 0;
if (responder != null)
{
host = responder.Value<string>("Host");
port = responder.Value<int>("Port");
}
var behaviors = module.SelectToken("$.behaviors");
foreach (var behavior in behaviors)
{
string name = behavior.Value<string>("name");
string functionName = behavior.Value<string>("function_name");
var args = behavior.SelectToken("$.arg");
var parameters = new Dictionary<string, object>();
foreach (var arg in args)
{
parameters.Add(arg.Value<string>("name"), new Dictionary<string, object>
{
{"value", arg.Value<string>("value")},
{"place_holder", arg.Value<string>("place_holder")},
{"type", arg.Value<string>("type")}
});
}
var matchingBehaviors = ret.Where(s => s.BehaviorName == name).ToList();
foreach (var matchingBehavior in matchingBehaviors)
{
matchingBehavior.ModuleName = moduleName;
matchingBehavior.FunctionName = functionName;
matchingBehavior.Ip = host;
matchingBehavior.Port = port;
foreach (var parameter in parameters)
{
if (!matchingBehavior.Parameters.ContainsKey(parameter.Key))
{
matchingBehavior.Parameters.Add(parameter);
}
}
}
}
}
}
}
}
return ret;
}
private void ProcessDataFromWorkers(BPlusTree <string, int> map)
{
using (var context = NetMQContext.Create())
{
using (NetMQSocket receiver = context.CreatePullSocket())
{
Task.Run(() =>
{
// The Sink listens to the Ventilator for news about jobs
using (NetMQSocket sink = context.CreatePullSocket())
{
sink.Connect(_ventilatorReceiveAddress);
while (_writingData)
{
string message = "";
try
{
message = sink.ReceiveString();
}
catch (ObjectDisposedException e)
{
// We're done. Exit the loop.
break;
}
// A new job was pushed
if (message != "done")
{
_knownJobs.Add(message);
}
// We're done!
else
{
Task.Run(() =>
{
// While the count isn't the same, wait.
// Todo: Bake better status monitoring to prevent deadlocking
// if things are ever dropped.
while (_processedJobs.Count != _knownJobs.Count)
{
Thread.Sleep(10);
}
receiver.Close();
sink.Close();
_writingData = false;
});
}
}
}
});
receiver.Bind(_receiverAddress);
var dict = new Dictionary <string, int>();
while (_writingData)
{
string message = "";
try
{
message = receiver.ReceiveString();
}
catch (ObjectDisposedException e)
{
// We know there isn't more work, so we break this loop.
break;
}
var jobResult = JsonConvert.DeserializeObject <JobResult>(message);
foreach (var word in jobResult.Data)
{
// Update existing values
if (dict.ContainsKey(word.Key))
{
dict[word.Key] = word.Value + dict[word.Key];
}
else // Add new value
{
dict[word.Key] = word.Value;
}
// This is around the max that we can hold in 4gb safely.
// Assuming our strings aren't suuuper long.
if (dict.Count > 3020000)
{
Console.WriteLine("Flushing " + jobResult.JobId + "... ");
var stopwatch = new Stopwatch();
stopwatch.Start();
FlushToDisk(map, dict);
dict = new Dictionary <string, int>();
stopwatch.Stop();
Console.WriteLine("Done flushing " + jobResult.JobId + ". Time: " + stopwatch.ElapsedMilliseconds + "ms");
}
}
_processedJobs.Add(jobResult.JobId);
}
Console.WriteLine("Flattening data to TSV file.");
// Save our changes to the ondisk db
if (map.Any())
{
Console.WriteLine("Pushing data to BPlusTree.");
FlushToDisk(map, dict);
Console.WriteLine(map.Count);
WriteToTSVFile(map, "Output.tsv");
}
// Everything is in memory so just use that.
else
{
WriteToTSVFile(dict, "Output.tsv");
}
// We're done!
WaitingOnWork = false;
}
}
}
