private void DownloadQuotes()
{
try
{
while (!_cancelled)
{
int remaining = 0;
string symbol = null;
lock (_pending)
{
if (_pending.Count == 0)
{
lock (_retry)
{
foreach (var item in _retry)
{
_completed--;
_pending.Add(item);
}
_retry.Clear();
}
}
if (_pending.Count > 0)
{
symbol = _pending.FirstOrDefault();
_pending.Remove(symbol);
remaining = _pending.Count;
}
}
if (symbol == null)
{
// done!
break;
}
// even if it fails, we consider the job complete (from a progress point of view).
_completed++;
// weed out any securities that have no symbol or have a
if (string.IsNullOrEmpty(symbol))
{
// skip securities that have no symbol.
}
else if (symbol.IndexOfAny(illegalUrlChars) >= 0)
{
// since we are passing the symbol on an HTTP URI line, we can't pass Uri illegal characters...
OnError(string.Format(Walkabout.Properties.Resources.SkippingSecurityIllegalSymbol, symbol));
OnSymbolNotFound(symbol);
}
else
{
try
{
// this service doesn't want too many calls per second.
int ms = _throttle.GetSleep();
while (ms > 0)
{
if (ms > 1000)
{
int seconds = ms / 1000;
OnError("AlphaVantage quote service needs to sleep for " + seconds + " seconds");
}
else
{
OnError("AlphaVantage quote service needs to sleep for " + ms.ToString() + " ms");
}
OnSuspended(true);
while (!_cancelled && ms > 0)
{
Thread.Sleep(1000);
ms -= 1000;
}
OnSuspended(false);
ms = _throttle.GetSleep();
}
if (_cancelled)
{
break;
}
string uri = string.Format(address, symbol, _settings.ApiKey);
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri);
req.UserAgent = "USER_AGENT=Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;)";
req.Method = "GET";
req.Timeout = 10000;
req.UseDefaultCredentials = false;
_current = req;
Debug.WriteLine("AlphaVantage fetching quote " + symbol);
WebResponse resp = req.GetResponse();
_throttle.RecordCall();
using (Stream stm = resp.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stm, Encoding.UTF8))
{
string json = sr.ReadToEnd();
JObject o = JObject.Parse(json);
StockQuote quote = ParseStockQuote(o);
if (quote == null || quote.Symbol == null)
{
OnError(string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbol));
OnSymbolNotFound(symbol);
}
else if (string.Compare(quote.Symbol, symbol, StringComparison.OrdinalIgnoreCase) != 0)
{
// todo: show appropriate error...
}
else
{
OnQuoteAvailable(quote);
}
}
}
OnError(string.Format(Walkabout.Properties.Resources.FetchedStockQuotes, symbol));
}
catch (System.Net.WebException we)
{
if (we.Status != WebExceptionStatus.RequestCanceled)
{
OnError(string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbol) + "\r\n" + we.Message);
}
else
{
// we cancelled, so bail.
_cancelled = true;
break;
}
HttpWebResponse http = we.Response as HttpWebResponse;
if (http != null)
{
// certain http error codes are fatal.
switch (http.StatusCode)
{
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.InternalServerError:
case HttpStatusCode.Unauthorized:
OnError(http.StatusDescription);
_cancelled = true;
break;
}
}
}
catch (Exception e)
{
// continue
string message = string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbol) + "\r\n" + e.Message;
if (message.Contains("Please visit https://www.alphavantage.co/premium/"))
{
lock (_retry)
{
_retry.Add(symbol);
}
_throttle.CallsThisMinute += this._settings.ApiRequestsPerMinuteLimit;
}
OnComplete(PendingCount == 0, message);
}
Thread.Sleep(1000); // this is so we don't starve out the download service.
}
}
}
catch
{
}
_completed = 0;
if (PendingCount == 0)
{
OnComplete(true, "AlphaVantage download complete");
}
else
{
OnComplete(false, "AlphaVantage download cancelled");
}
_downloadThread = null;
_current = null;
}
private void DownloadQuotes()
{
try
{
// This is on a background thread
int max_batch = 100;
List <string> batch = new List <string>();
int remaining = 0;
while (!_cancelled)
{
string symbol = null;
lock (_pending)
{
if (_pending.Count > 0)
{
symbol = _pending.FirstOrDefault();
_pending.Remove(symbol);
remaining = _pending.Count;
}
}
if (symbol == null)
{
// done!
break;
}
// weed out any securities that have no symbol or have a symbol that would be invalid.
if (string.IsNullOrEmpty(symbol))
{
// skip securities that have no symbol.
}
else if (symbol.IndexOfAny(illegalUrlChars) >= 0)
{
// since we are passing the symbol on an HTTP URI line, we can't pass Uri illegal characters...
OnSymbolNotFound(symbol);
OnError(string.Format(Walkabout.Properties.Resources.SkippingSecurityIllegalSymbol, symbol));
}
else
{
batch.Add(symbol);
}
if (batch.Count() == max_batch || remaining == 0)
{
// even it if tails we consider the job completed from a status point of view.
_completed += batch.Count;
string symbols = string.Join(",", batch);
try
{
// this service doesn't want too many calls per second.
int ms = _throttle.GetSleep();
while (ms > 0 && !_cancelled)
{
if (ms > 1000)
{
int seconds = ms / 1000;
OnError("IEXCloud service needs to sleep for " + seconds + " seconds");
}
else
{
OnError("IEXCloud service needs to sleep for " + ms.ToString() + " ms");
}
OnSuspended(true);
while (!_cancelled && ms > 0)
{
Thread.Sleep(1000);
ms -= 1000;
}
OnSuspended(false);
ms = _throttle.GetSleep();
}
if (_cancelled)
{
break;
}
string uri = string.Format(address, symbols, _settings.ApiKey);
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri);
req.UserAgent = "USER_AGENT=Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;)";
req.Method = "GET";
req.Timeout = 10000;
req.UseDefaultCredentials = false;
_current = req;
WebResponse resp = req.GetResponse();
_throttle.RecordCall();
using (Stream stm = resp.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stm, Encoding.UTF8))
{
string json = sr.ReadToEnd();
JObject o = JObject.Parse(json);
List <StockQuote> result = ParseStockQuotes(o);
// make sure they are all returned, and report errors for any that are not.
foreach (string s in batch)
{
StockQuote q = (from i in result where string.Compare(i.Symbol, s, StringComparison.OrdinalIgnoreCase) == 0 select i).FirstOrDefault();
if (q == null)
{
OnError(string.Format("No quote returned for symbol {0}", s));
OnSymbolNotFound(s);
}
else
{
OnQuoteAvailable(q);
}
}
}
}
Thread.Sleep(1000); // there is also a minimum sleep between requests that we must enforce.
symbols = string.Join(", ", batch);
OnError(string.Format(Walkabout.Properties.Resources.FetchedStockQuotes, symbols));
}
catch (System.Net.WebException we)
{
if (we.Status != WebExceptionStatus.RequestCanceled)
{
OnError(string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbols) + "\r\n" + we.Message);
}
else
{
// we cancelled, so bail.
_cancelled = true;
break;
}
HttpWebResponse http = we.Response as HttpWebResponse;
if (http != null)
{
// certain http error codes are fatal.
switch (http.StatusCode)
{
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.InternalServerError:
case HttpStatusCode.Unauthorized:
OnError(http.StatusDescription);
_cancelled = true;
break;
}
}
}
catch (Exception e)
{
// continue
OnError(string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbols) + "\r\n" + e.Message);
}
batch.Clear();
}
_current = null;
}
}
catch
{
}
if (PendingCount == 0)
{
OnComplete(true, "IEXCloud download complete");
}
else
{
OnComplete(false, "IEXCloud download cancelled");
}
_downloadThread = null;
_current = null;
_completed = 0;
}
