protected override void ServerTerminate( )
{
Logr.Log("~A RTDServer.ServerTerminate");
// Clear down any running timers...
CronManager.Instance( ).Clear( );
s_Instances.Remove(this);
}
public String GetQueryURL(String qtype, String qkey)
{
// We're looking for a row that has 'quandl' or 'tiingo' in the first cell,
// query in the second, and then qkey in the third.
int row = FindRow(qtype, "query", qkey);
if (row == -1)
{
Logr.Log(String.Format("GetQueryURL: couldn't find {0}.{1}", qtype, qkey));
return("");
}
int col = 3;
string name;
object val;
Dictionary <string, object> qterms = new Dictionary <string, object>( );
do
{
name = GetCellAsString(row, col);
val = GetCell(row, col + 1);
if (name != null && name != "")
{
qterms.Add(name, val);
}
col += 2;
} while (name != null && name != "");
if (qtype == "quandl")
{
return(BuildQuandlQuery(qterms));
}
return(BuildTiingoQuery(qterms));
}
public RTDServer( )
{
Logr.Log("~A RTDServer created");
m_Subscriptions = new Dictionary <Topic, string>( );
m_TopicMap = new Dictionary <string, Topic>( );
s_Instances.Add(this);
}
public Dictionary <String, String> GetTiingoWebSock(String wskey)
{
// We're looking for a row that has 'twebsock' in the first cell, tiingo in the second,
// and then wskey in the third.
int row = FindRow("twebsock", "tiingo", wskey);
if (row == -1)
{
Logr.Log(String.Format("GetTiingoWebSock: couldn't find {0}", wskey));
return(null);
}
// Now we've found the right row we expect to find the url in col D
// and the auth_token on a config row. The auth_token is mandatory
// for a tiingo websock.
string url = GetCellAsString(row, 3);
string auth_token = GetQueryConfig("tiingo", "auth_token");
if (url == null || auth_token == null)
{
Logr.Log(String.Format("GetTiingoWebSock: bad url or auth_token wskey({0})", wskey));
return(null);
}
var req = new Dictionary <string, string>( )
{
{ "type", "twebsock" }, { "key", wskey }, { "url", url }, { "auth_token", auth_token }
};
// Now let's deal with optional elements of a tiingo request: proxy config may be supplied if we have to
// connect via a proxy.
GetProxyConfig("tiingo", req);
return(req);
}
public String GetWebSock(String wskey)
{
// We're looking for a row that has 'websock' in the first cell, url in the second,
// and then wskey in the third.
int row = FindRow("websock", "url", wskey);
if (row == -1)
{
Logr.Log(String.Format("GetWebSock: couldn't find {0}", wskey));
return(null);
}
// Now we've found the right row we expect to find three columns to make up a
// URL: host, port, path in D, E & F
string host = GetCellAsString(row, 3);
string port = GetCellAsString(row, 4);
string path = GetCellAsString(row, 5);
if (host == null || port == null || path == null)
{
Logr.Log(String.Format("GetWebSock: bad host, port or path wskey({0})", wskey));
return(null);
}
string url = String.Format("ws://{0}:{1}/{2}", host, port, path);
return(url);
}
void MessageReceived(object sender, MessageReceivedEventArgs e)
{
Logr.Log(String.Format("TWSCallback.MessageReceived: {0}", e.Message));
var msg = JsonConvert.DeserializeObject <IDictionary <string, object> >(e.Message, s_JsonSettings);
m_RTMHandler.MessageReceived(msg); // may cause callbacks to HeartBeat or MktDataTick below
}
public void MktDataTick(IList <object> tick)
{
if (tick.Count < m_MktDataRecord.Length)
{
Logr.Log(String.Format("TWSCallback.MktDataTick: fld count under {0}! {1}",
m_MktDataRecord.Length, tick.ToString( )));
return;
}
string ticker = tick[m_TickerIndex].ToString();
lock (m_Client) {
if (!m_Subscriptions.ContainsKey(ticker))
{
return;
}
SortedSet <string> fldset = m_Subscriptions[ticker];
for (int inx = 0; inx < m_MktDataRecord.Length; inx++)
{
string fld = m_MktDataRecord[inx];
object val = tick[inx];
if (fldset.Contains(fld) && val != null)
{
UpdateRTD(m_Key, String.Format("{0}_{1}", ticker, fld), tick[inx].ToString( ));
}
}
}
// TODO: add TWSCache to s_Cache so that we only need an RTD sub to one field in a record, for
// instance bid, and then the rest can be pulled from the cache...
// s_Cache.UpdateWSCache( m_Key, updates );
}
public bool AddCron(string ckey, string cronex, DateTime?start, DateTime?end)
{
// no locking here as we won't touch any objects that are shared with another thread
string[] cronflds = new string[6];
Logr.Log(String.Format("AddCron: cronex({0}) start({1}) end({2})", cronex, start, end));
try {
if (m_CronMap.ContainsKey(ckey))
{
// If there's already a timer with ckey it may be that an Excel users has triggered
// another invocation of s2cron( ) by editting the s2cfg sheet, or with a sh-ctrl-alt-F9.
// Either way, we need to remove the old timer, and create a new one, but only if the
// new one is different.
CronTimer oldTimer = m_CronMap[ckey];
if (oldTimer.Cronex == cronex && oldTimer.Start == start && oldTimer.End == end)
{
// no change, so we won't overwrite the entry for ckey
return(true);
}
m_CronMap.Remove(ckey);
oldTimer.Close( );
}
m_CronMap[ckey] = new CronTimer(ckey, cronex, start, end);
return(true);
}
catch (Exception ex) {
Logr.Log(String.Format("AddCron: {0}", ex.Message));
return(false);
}
}
protected void ConfigureProxy(Dictionary <string, string> work, WebClient wc)
{
// If the dictionary has proxy config, then set it up...
if (!work.ContainsKey("http_proxy_host"))
{
return;
}
int port = 80;
string host = work["http_proxy_host"];
if (work.ContainsKey("http_proxy_port"))
{
if (!Int32.TryParse(work["http_proxy_port"], out port))
{
port = 80;
}
}
WebProxy proxy = new WebProxy(String.Format("{0}:{1}", host, port), true);
string user = "", pass = "";
if (work.ContainsKey("http_proxy_user") && work.ContainsKey("http_proxy_password"))
{
user = work["http_proxy_user"];
pass = work["http_proxy_password"];
proxy.Credentials = new NetworkCredential(user, pass);
}
wc.Proxy = proxy;
Logr.Log(String.Format("ConfigureProxy host({0}) port({1}) user({2}) pass({3})", host, port, user, pass));
}
void Closed(object sender, EventArgs e)
{
Logr.Log(String.Format("Closed: wskey({0})", m_Key));
if (m_ClosedCB != null)
{
m_ClosedCB(m_Key);
}
UpdateRTD(m_Key, "status", "closed");
}
protected override void DisconnectData(Topic topic)
{
lock (m_TopicMap) {
string stopic = m_Subscriptions[topic];
Logr.Log(String.Format("~A DisconnectData: {0}", stopic));
m_Subscriptions.Remove(topic);
m_TopicMap.Remove(stopic);
}
}
protected void OnTimerEvent(object o, ElapsedEventArgs e)
{
m_Count++;
ScheduleTimer( );
Logr.Log(String.Format("OnTimerEvent count({0}) last({1}) next({2})",
m_Count, m_LastEventTime, m_NextEventTime));
UpdateRTD(m_Key, "count", Convert.ToString(m_Count));
UpdateRTD(m_Key, "last", m_LastEventTime);
UpdateRTD(m_Key, "next", m_NextEventTime);
}
public static object s2sub(
[ExcelArgument(Name = "SubCache", Description = "[quandl|tiingo|cron|websock]")] string subcache,
[ExcelArgument(Name = "CacheKey", Description = "Row key from s2cfg")] string ckey,
[ExcelArgument(Name = "Property", Description = "[status|count|next|last|mX_Y_Z]")] string prop)
{
string[] arrey = { subcache, ckey, prop };
string stopic = String.Join(".", arrey);
Logr.Log(String.Format("s2sub: {0}", stopic));
return(XlCall.RTD("SSAddin.RTDServer", null, stopic));
}
protected override object ConnectData(Topic topic, System.Collections.Generic.IList <string> topicInfo, ref bool newValues)
{
lock (m_TopicMap) {
string stopic = topicInfo[0];
int topicId = GetTopicId(topic);
Logr.Log(String.Format("~A ConnectData: {0} - {1}", topicId, stopic));
m_Subscriptions.Add(topic, stopic);
m_TopicMap.Add(stopic, topic);
return(ExcelErrorUtil.ToComError(ExcelError.ExcelErrorNA));
}
}
public bool ScheduleTimer( )
{
if (m_Closed)
{
Logr.Log(String.Format("ScheduleTimer: {0} is closed", m_Key));
return(false);
}
// NB this method is called by the Excel thread in the first instance. Subsequent calls are
// on a pool thread, as .Net dispatches timer callbacks on pool threads, unless we specify otherwise.
// I'm not bothering with a lock because the first timer event isn't scheduled until we do
// m_Timer.Enabled = true below, and we don't touch the iterator after that, so there's no
// chance that we'll have two threads touching m_Iterator at the same time.
// What if the next time we got from m_Iterator is already in the past?
// If it is keep moving fwd til we get a time in the future. Bear in mind there's
// an error condition where Current can appear to be in the future when it's not.
// If DateTime.Now==2015-06-03T20:03:04.9998700, and Current==2015-06-03T20:03:05
// then CompareTo will tell us that Current is in the future, when for our
// purposes it's not. If ticks is -ve then Current is in the past. If Current is
// a small +/-ve then it's the same as Now since our unit of granularity in the
// cron sys is 1 sec. There are 10,000 ticks to the millisec, 10,000,000 to the sec.
// So we'll look for Current to be 1,000,000 ticks later than Now before scheduling.
// Which is +1,000,000. If ticks is -ve then Current is in the past. This check
// should also prevent interval==0 below!
long ticks = m_Iterator.Current.Ticks - DateTime.Now.Ticks;
while (ticks < 1000000)
{
if (!m_Iterator.MoveNext( ))
{
Logr.Log(String.Format("ScheduleTimer: {0} exhausted", m_Key));
return(false);
}
ticks = m_Iterator.Current.Ticks - DateTime.Now.Ticks;
}
m_LastEventTime = DateTime.Now.ToString( );
m_NextEventTime = m_Iterator.Current.ToString( );
// Ticks is number of 100 nanos since 0001-01-01T00:00:00. Diff between now
// and next event time 10K is the number of millisecs until the next cron event
// for ckey. https://msdn.microsoft.com/en-us/library/system.datetime.ticks%28v=vs.100%29.aspx
// 10,000 ticks in 1 millisec
long interval = Math.Abs(ticks / 10000);
if (interval == 0)
{
// Given the code above, this should not happen!
Logr.Log(String.Format("ScheduleTimer: ZERO interval! ckey({0}) Current({1}) Now({2})",
m_Key, m_Iterator.Current, DateTime.Now));
return(false);
}
m_Timer.Interval = interval;
Logr.Log(String.Format("ScheduleTimer: ckey({0}) Current({1}) Now({2})", m_Key, m_Iterator.Current, DateTime.Now));
return(true);
}
public Tuple <String, DateTime?, DateTime?> GetCronTab(String ctabkey)
{
// We're looking for a row that has 'cron' in the first cell, tab in the second,
// and then ctabkey in the third.
int row = FindRow("cron", "tab", ctabkey);
if (row == -1)
{
Logr.Log(String.Format("GetCronTab: couldn't find {0}", ctabkey));
return(null);
}
// Now we've found the right row we expect to find six columns to make up a
// crontab entry in D, E, F, G, H, I, J, and then two more columns for start
// & end in K & L
string[] flds = new string[6];
int col = 0;
for ( ; col < 6; col++)
{
flds[col] = GetCellAsString(row, col + 3);
}
string cronex = String.Join(" ", flds);
double dstart, dend;
DateTime?start = null; // return nulls if K & L cols are empty
DateTime?end = null;
// new DateTime( start.Year, start.Month, start.Day, 23, 59, 59 );
// If the start and end cells on the cron tab entry on the s2cfg page are time of
// day eg 09:30:00 and not full date times then they yield DateTime doubles that
// are < 1.0 as they encode no date/day info. But the Interval arithmetic for the
// next event in CronManager uses DateTime.Now as a baseline, and that includes
// date/day info. So we must baseline off the date/day for today too.
DateTime sod = new DateTime(DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, 0, 0, 0);
double dsod = sod.ToOADate( );
string sstart = GetCellAsString(row, 3 + col++);
string send = GetCellAsString(row, 3 + col++);
if (Double.TryParse(sstart, out dstart))
{
if (dstart < 1.0)
{
dstart += dsod;
}
start = DateTime.FromOADate(dstart);
}
if (Double.TryParse(send, out dend))
{
if (dend < 1.0)
{
dend += dsod;
}
end = DateTime.FromOADate(dend);
}
return(new Tuple <String, DateTime?, DateTime?>(cronex, start, end));
}
public TWSCallback(Dictionary <string, string> work, ClosedCB ccb)
{
// No need to bother locking in the ctor. We are on the background
// worker thread here, but we won't get methods fired on the pool
// threads until this method exits.
m_Key = work["key"];
m_URL = work["url"];
work.TryGetValue("auth_token", out m_AuthToken);
m_ClosedCB = ccb;
try {
m_Client = new WebSocket(m_URL);
m_RTMHandler = new TiingoRealTimeMessageHandler(m_Client, MktDataTick, HeartBeat, SetSubID);
m_Client.Opened += new EventHandler(Opened);
m_Client.Error += new EventHandler <SuperSocket.ClientEngine.ErrorEventArgs>(Error);
m_Client.Closed += new EventHandler(Closed);
m_Client.MessageReceived += new EventHandler <MessageReceivedEventArgs>(MessageReceived);
m_Client.DataReceived += new EventHandler <WebSocket4Net.DataReceivedEventArgs>(DataReceived);
// Do we need to set up a proxy?
if (work.TryGetValue("http_proxy_host", out m_ProxyHost))
{
IPHostEntry he = Dns.GetHostEntry(m_ProxyHost);
int port = 80;
if (work.TryGetValue("http_proxy_host", out m_ProxyPort))
{
if (!Int32.TryParse(m_ProxyPort, out port))
{
port = 80;
}
}
var proxy = new HttpConnectProxy(new IPEndPoint(he.AddressList[0], port));
// Do we need to supply authentication to the proxy?
if (work.TryGetValue("http_proxy_user", out m_ProxyUser))
{
work.TryGetValue("http_proxy_password", out m_ProxyPassword);
// encode user:password as base64 and supply as 'Proxy-Authorization: Basic dXNlbWU6dGVzdA=='
string upass = String.Format("{0}:{1}", m_ProxyUser, m_ProxyPassword);
var plainTextBytes = System.Text.Encoding.UTF8.GetBytes(upass);
string b64 = System.Convert.ToBase64String(plainTextBytes);
proxy.Authorization = String.Format("Basic {0}", b64);
}
m_Client.Proxy = proxy;
}
Logr.Log(String.Format("TWSCallback.ctor: key({0}) url({1}) auth_token({2}) http_proxy_host({3}) http_proxy_port({4}) http_proxy_user({5}) http_proxy_password({6})",
m_Key, m_URL, m_AuthToken, m_ProxyHost, m_ProxyPort, m_ProxyUser, m_ProxyPassword));
m_Client.Open( );
}
catch (System.ArgumentException ae) {
Logr.Log(String.Format("TWSCallback.ctor: {0}", ae.Message));
}
catch (System.FormatException fe) {
Logr.Log(String.Format("TWSCallback.ctor: format error fmt({0}) auth({1}) err({2})", s_SubscribeMessageFormat, m_AuthToken, fe.Message));
}
}
protected bool DoTiingoQuery(Dictionary <string, string> work)
{
string qkey = work["key"];
string url = work["url"];
string auth_token = work["auth_token"];
string line = "";
string lineCount = "0";
try {
// Set up the web client to HTTP GET
var client = new WebClient( );
ConfigureProxy(work, client);
client.Headers.Set("Content-Type", "application/json");
client.Headers.Set("Authorization", String.Format("Token {0}", auth_token));
Stream data = client.OpenRead(url);
var reader = new StreamReader(data);
// Local file to dump result
int pid = Process.GetCurrentProcess( ).Id;
string jsnfname = String.Format("{0}\\{1}_{2}.jsn", m_TempDir, qkey, pid);
Logr.Log(String.Format("running tiingo qkey({0}) {1} persisted at {2}", qkey, url, jsnfname));
var jsnf = new StreamWriter(jsnfname);
UpdateRTD("tiingo", qkey, "status", "starting");
// Clear any previous result from the cache so we don't append repeated data
s_Cache.ClearTiingo(qkey);
StringBuilder sb = new StringBuilder( );
while (reader.Peek( ) >= 0)
{
// For each CSV line returned by quandl, dump to localFS, add to in mem cache, and
// send a line count update to any RTD subscriber
line = reader.ReadLine( );
jsnf.WriteLine(line);
sb.AppendLine(line);
}
jsnf.Close( );
data.Close( );
reader.Close( );
UpdateRTD("tiingo", qkey, "status", "complete");
UpdateRTD("tiingo", "all", "count", String.Format("{0}", m_TiingoCount++));
Logr.Log(String.Format("tiingo qkey({0}) complete count({1})", qkey, lineCount));
List <SSTiingoHistPrice> updates = JsonConvert.DeserializeObject <List <SSTiingoHistPrice> >(sb.ToString( ));
s_Cache.UpdateTHPCache(qkey, updates);
UpdateRTD("tiingo", qkey, "count", String.Format("{0}", updates.Count));
return(true);
}
catch (System.IO.IOException ex) {
Logr.Log(String.Format("tiingo qkey({0}) url({1}) {2}", qkey, url, ex));
}
catch (System.Net.WebException ex) {
Logr.Log(String.Format("tiingo qkey({0}) url({1}) {2}", qkey, url, ex));
}
return(false);
}
public void BackgroundWork( )
{
// We're running on the background thread. Loop until we're told to exit...
Logr.Log(String.Format("~A BackgroundWork thread started"));
// Main loop for worker thread. It will briefly hold the m_InFlight lock when
// it removes entries, and also the m_InputQueue lock in GetWork( ) as it
// removes entries. DoQuandlQuery( ) will grab the cache lock when it's
// adding cache entries. Obviously, no lock should be held permanently! JOS 2015-04-31
while (true)
{
// Wait for a signal from the other thread to say there's some work.
m_Event.WaitOne( );
String[] work = GetWork( );
while (work != null)
{
if (work[0] == "stop")
{
// exit req from excel thread
Logr.Log(String.Format("~A BackgroundWork thread exiting"));
return;
}
string fkey = String.Format("{0}.{1}", work[0], work[1]);
Logr.Log(String.Format("~A BackgroundWork new request fkey({0})", fkey));
if (work[0] == "quandl")
{
bool ok = DoQuandlQuery(work[1], work[2]);
lock (m_InFlight) {
m_InFlight.Remove(fkey);
}
}
else if (work[0] == "tiingo")
{
bool ok = DoTiingoQuery(work[1], work[2], work[3]);
lock (m_InFlight) {
m_InFlight.Remove(fkey);
}
}
else if (work[0] == "websock")
{
WSCallback wscb = new WSCallback(work[1], work[2], this.WSCallbackClosed);
lock (m_InFlight) {
m_WSCallbacks.Add(fkey, wscb);
}
}
work = GetWork( );
}
// We've exhausted the queued work, so reset the event so that we wait in the
// WaitOne( ) invocation above until another thread signals that there's some
// more work.
m_Event.Reset( );
}
}
public String GetQueryConfig(String qtype, String ckey)
{
// We're looking for a row that has qtype [quandl|tiingo] in the first cell, config in the second,
// and then ckey in the third.
int row = FindRow(qtype, "config", ckey);
if (row == -1)
{
Logr.Log(String.Format("GetQueryConfig: couldn't find {0}.{1}", qtype, ckey));
return("");
}
return(GetCellAsString(row, 3));
}
public String GetQuandlConfig(String ckey)
{
// We're looking for a row that has 'quandl' in the first cell, config in the second,
// and then ckey in the third.
int row = FindRow("quandl", "config", ckey);
if (row == -1)
{
Logr.Log(String.Format("GetQuandlConfig: couldn't find {0}", ckey));
return("");
}
return(GetCellAsString(row, 3));
}
void Closed(object sender, EventArgs e)
{
Logr.Log(String.Format("TWSCallback.Closed: wskey({0})", m_Key));
if (m_ClosedCB != null)
{
m_ClosedCB(String.Format("twebsock.{0}", m_Key));
}
lock (m_Client) {
// Socket has closed, and will need to be reopened. The reopen will trigger
// another initial subscription, and then a new sub ID.
m_SubID = null;
}
UpdateRTD(m_Key, "status", "closed");
}
protected void UpdateRTD(string qkey, string subelem, string value)
{
// The RTD server doesn't necessarily exist. If no cell calls
// s2sub( ) it won't be instanced by Excel.
RTDServer rtd = RTDServer.GetInstance( );
if (rtd == null)
{
Logr.Log(String.Format("UpdateRTD: no RTD server!"));
return;
}
string stopic = String.Format("cron.{0}.{1}", qkey, subelem);
rtd.CacheUpdate(stopic, value);
}
public void CacheUpdate(string stopic, string value)
{
lock (m_TopicMap) {
Logr.Log(String.Format("~A CacheUpdate topic({0}) val({1})", stopic, value));
if (m_TopicMap.ContainsKey(stopic))
{
Topic topic = m_TopicMap[stopic];
topic.UpdateValue(value);
}
else
{
Logr.Log(String.Format("~A CacheUpdate UNKNOWN topic({0}) val({1})", stopic, value));
}
}
}
public void MessageReceived(IDictionary <string, object> msg)
{
if (msg == null)
{
Logr.Log(String.Format("TiingoRealTimeMessageHandler.MessageReceived: null msg!"));
return;
}
if (!msg.ContainsKey("messageType"))
{
Logr.Log(String.Format("TiingoRealTimeMessageHandler.MessageReceived: missing messageType field!"));
return;
}
// for example messages https://api.tiingo.com/docs/iex/realtime#priceData
string mt = msg["messageType"].ToString( );
switch (mt)
{
case "I": // Informational
if (msg.ContainsKey("data"))
{
var dd = (IDictionary <string, object>)msg["data"];
if (dd.ContainsKey("subscriptionId"))
{
m_SetSubID(dd["subscriptionId"].ToString( ));
}
}
break;
case "H": // Heartbeat
m_HB(++m_HBCount);
break;
case "A": // Market data
if (msg.ContainsKey("data"))
{
m_Tick((IList <object>)msg["data"]);
}
break;
case "E": // Error
break;
default:
Logr.Log(String.Format("TiingoRealTimeMessageHandler.MessageReceived: unexpected messageType({0})!", mt));
break;
}
}
public void CacheUpdateBatch(string stroot, List <SSWebCell> updates)
{
lock (m_TopicMap) {
Logr.Log(String.Format("~A CacheUpdateBatch {0} {1}", stroot, updates.Count));
foreach (SSWebCell wc in updates)
{
String stopic = String.Format("{0}.{1}", stroot, wc.id);
if (m_TopicMap.ContainsKey(stopic))
{
Topic topic = m_TopicMap[stopic];
topic.UpdateValue(wc.body);
Logr.Log(String.Format("RTDServer.CacheUpdateBatch: topic({0}) value({1})", stopic, wc.body));
}
}
}
}
void MessageReceived(object sender, MessageReceivedEventArgs e)
{
List <SSWebCell> updates = JsonConvert.DeserializeObject <List <SSWebCell> >(e.Message);
Logr.Log(String.Format("MessageReceived: updates.Count({0})", updates.Count));
if (updates.Count == 0)
{
return;
}
RTDServer rtd = RTDServer.GetInstance( );
if (rtd != null)
{
rtd.CacheUpdateBatch(String.Format("websock.{0}", m_Key), updates);
}
s_Cache.UpdateWSCache(m_Key, updates);
}
protected bool DoQuandlQuery(Dictionary <string, string> work)
{
string qkey = work["key"];
string url = work["url"];
string line = "";
string lineCount = "0";
try {
// Set up the web client to HTTP GET
var client = new WebClient( );
ConfigureProxy(work, client);
Stream data = client.OpenRead(url);
var reader = new StreamReader(data);
// Local file to dump result
int pid = Process.GetCurrentProcess( ).Id;
string csvfname = String.Format("{0}\\{1}_{2}.csv", m_TempDir, qkey, pid);
Logr.Log(String.Format("running quandl qkey({0}) {1} persisted at {2}", qkey, url, csvfname));
var csvf = new StreamWriter(csvfname);
UpdateRTD("quandl", qkey, "status", "starting");
// Clear any previous result from the cache so we don't append repeated data
s_Cache.ClearQuandl(qkey);
while (reader.Peek( ) >= 0)
{
// For each CSV line returned by quandl, dump to localFS, add to in mem cache, and
// send a line count update to any RTD subscriber
line = reader.ReadLine( );
csvf.WriteLine(line);
lineCount = String.Format("{0}", s_Cache.AddQuandlLine(qkey, line.Split(csvDelimiterChars)));
UpdateRTD("quandl", qkey, "count", lineCount);
}
csvf.Close( );
data.Close();
reader.Close();
UpdateRTD("quandl", qkey, "status", "complete");
UpdateRTD("quandl", "all", "count", String.Format("{0}", m_QuandlCount++));
Logr.Log(String.Format("quandl qkey({0}) complete count({1})", qkey, lineCount));
return(true);
}
catch (System.IO.IOException ex) {
Logr.Log(String.Format("quandl qkey({0}) url({1}) {2}", qkey, url, ex));
}
catch (System.Net.WebException ex) {
Logr.Log(String.Format("quandl qkey({0}) url({1}) {2}", qkey, url, ex));
}
return(false);
}
void DispatchSubscriptions( )
{
// We could be called by either thread, so lock as we'll potentially change object state
// and send stuff down the socker while another thread wants to do the same.
lock (m_Client) {
if (m_Client.State == WebSocketState.Open)
{
// The socket is open, so the Opened( ) callback below must have already fired.
// An open socket isn't enough. We also need a subID, which we only have after
// we've processed the response to the initial subscription.
StringBuilder sb = new StringBuilder( );
int inx = 0;
foreach (string sub in m_PendingSubs)
{
if (inx > 0)
{
sb.Append(",");
}
sb.Append(String.Format("\"{0}\"", sub));
inx++;
}
string sublist = sb.ToString( );
string ed;
if (m_SubID != null)
{
// We've got a subID, so only send a message if we've got tickers to add.
if (sublist.Length == 0)
{
return;
}
ed = String.Format(s_EventDataSubIdFormat, m_SubID, sublist);
}
else
{
// We don't have a subID, so compose an initial message and send whether or not
// we have a ticker list.
ed = String.Format(s_EventDataFormat, sublist);
}
string submsg = String.Format(s_SubscribeMessageFormat, m_AuthToken, ed);
Logr.Log(String.Format("TWSCallback.DispatchSubscriptions: subscribe message({0})", submsg));
m_Client.Send(submsg);
m_PendingSubs.Clear();
}
}
}
public bool AddRequest(Dictionary <string, string> req)
{
// Every request *must* have type, key, url. There may be other optionals like
// auth_token, https_proxy_host...
string type = req["type"];
string key = req["key"];
string fkey = String.Format("{0}.{1}", req["type"], req["key"]);
bool isWebQuery = (type == "quandl" || type == "tiingo");
// Is this job pending or in progress?
lock (m_InFlight) {
if (m_InFlight.Contains(fkey)) // Queued or running...
{
if (isWebQuery)
{
Logr.Log(String.Format("~A AddRequest: {0} is already inflight", fkey));
}
return(false); // so bail
}
// Nested locking - look out! We're on the Excel thread here as we're invoked by
// worksheet functions. The background worker thread does use this lock too, but
// not at the same time as m_InFlight, so we should be OK.
lock (m_InputQueue) {
// Running on the main Excel thread here. Q the work, and
// signal the background thread to wake up...
if (isWebQuery)
{
Logr.Log(String.Format("~A AddRequest adding {0} {1}", type, key));
}
m_InputQueue.Enqueue(req);
}
// NB some fkeys are only ever added to m_InFlight, and are never removed. For
// instance the s2sub notifications. We only need an s2sub notification to go to
// the background thread once to create websock subscriptions. And if it's an
// hbcount for instance, there's no work on the background thread, so we'll
// let it through once, then block subsequent notifies. We may want to revisit
// this logic in future if we need to resubscribe to websock topics. But for
// the time being let's work around by starting and stopping the sheet.
// JOS 2016-05-26
m_InFlight.Add(fkey);
m_Event.Set( ); // signal worker thread to wake
}
return(true);
}