internal bool RemoveDuplicates()
{
if (this.History != null)
{
StockQuote previous = null;
List <StockQuote> duplicates = new List <StockQuote>();
for (int i = 0; i < this.History.Count; i++)
{
StockQuote quote = this.History[i];
if (quote.Date == DateTime.MinValue)
{
duplicates.Add(quote);
}
else if (previous != null)
{
if (previous.Date.Date == quote.Date.Date)
{
duplicates.Add(previous);
}
}
previous = quote;
}
foreach (StockQuote dup in duplicates)
{
this.History.Remove(dup);
}
return(duplicates.Count > 0);
}
return(false);
}
void ProcessResult(StockQuote quote)
{
string symbol = quote.Symbol;
decimal price = quote.Close;
if (price != 0)
{
// we want to stop this from adding new Security objects by passing false
// because the Security objects should already exist as given to Enqueue
// and we shouldn't even fetch anything of those Security objects don't already
// have a 'Symbol' to lookup.
Security s = this.myMoney.Securities.FindSymbol(symbol, false);
if (s == null || s.IsDeleted)
{
return;
}
// Check to see if the security name has changed and update if needed
string securityName = quote.Name;
if (!string.IsNullOrEmpty(securityName) && (string.IsNullOrEmpty(s.Name) || s.Name == symbol))
{
s.Name = securityName;
}
s.Price = price;
s.PriceDate = quote.Date;
}
}
private void OnQuoteAvailable(StockQuote quote)
{
if (QuoteAvailable != null)
{
QuoteAvailable(this, quote);
}
}
private static StockQuote ParseStockQuote(JObject o)
{
StockQuote result = null;
Newtonsoft.Json.Linq.JToken value;
if (o.TryGetValue("Note", StringComparison.Ordinal, out value))
{
string message = (string)value;
throw new Exception(message);
}
if (o.TryGetValue("Error Message", StringComparison.Ordinal, out value))
{
string message = (string)value;
throw new Exception(message);
}
if (o.TryGetValue("Global Quote", StringComparison.Ordinal, out value))
{
result = new StockQuote()
{
Downloaded = DateTime.Now
};
if (value.Type == JTokenType.Object)
{
JObject child = (JObject)value;
if (child.TryGetValue("01. symbol", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
result.Symbol = (string)value;
}
if (child.TryGetValue("02. open", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
result.Open = (decimal)value;
}
if (child.TryGetValue("03. high", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
result.High = (decimal)value;
}
if (child.TryGetValue("04. low", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
result.Low = (decimal)value;
}
if (child.TryGetValue("08. previous close", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
result.Close = (decimal)value;
}
if (child.TryGetValue("06. volume", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
result.Volume = (decimal)value;
}
if (child.TryGetValue("07. latest trading day", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
result.Date = (DateTime)value;
}
}
}
return(result);
}
public void OnQuoteAvailable(StockQuote quote)
{
// record downloaded quotes so they can be merged with quote histories.
lock (_downloadedQuotes)
{
_downloadedQuotes[quote.Symbol] = quote;
}
}
private void OnServiceQuoteAvailable(object sender, StockQuote e)
{
Tuple <int, int> progress = GetProgress();
status.ShowProgress(e.Name, 0, progress.Item1, progress.Item2);
lock (fetched)
{
fetched.Add(e.Symbol);
}
this._downloadLog.OnQuoteAvailable(e);
lock (_batch)
{
_batch.Add(e);
}
}
public bool AddQuote(StockQuote quote, bool replace = true)
{
if (History == null)
{
History = new List <StockQuote>();
}
quote.Date = quote.Date.Date;
if (!string.IsNullOrEmpty(quote.Name))
{
this.Name = quote.Name;
quote.Name = null;
}
int len = History.Count;
for (int i = 0; i < len; i++)
{
var h = History[i];
if (h.Date == quote.Date)
{
// already have this one
if (replace)
{
h.Downloaded = quote.Downloaded;
h.Open = quote.Open;
h.Close = quote.Close;
h.High = quote.High;
h.Low = quote.Low;
h.Volume = quote.Volume;
}
return(true);
}
if (h.Date > quote.Date)
{
// keep it sorted by date
History.Insert(i, quote);
return(true);
}
}
History.Add(quote);
return(true);
}
internal bool IsComplete()
{
if (!this.Complete || this.History.Count == 0)
{
return(false);
}
int missing = 0; // in the last 3 months
var holidays = new UsHolidays();
DateTime workDay = holidays.GetPreviousWorkDay(DateTime.Today.AddDays(1));
DateTime stopDate = workDay.AddMonths(-3);
int count = 0; // work days
for (int i = this.History.Count - 1; i >= 0; i--)
{
count++;
StockQuote quote = this.History[i];
DateTime date = quote.Date.Date;
if (date > workDay)
{
continue; // might have duplicates?
}
if (workDay < stopDate)
{
break;
}
if (date < workDay)
{
if (!knownClosures.Contains(workDay))
{
missing++;
}
i++;
}
workDay = holidays.GetPreviousWorkDay(workDay);
}
// There are some random stock market closures which we can't keep track of easily, so
// make sure we are not missing more than 1% of the history.
return(missing < count * 0.01);
}
private StockQuoteHistory ParseTimeSeries(JObject o)
{
StockQuoteHistory history = new StockQuoteHistory();
history.History = new List <StockQuote>();
history.Complete = true; // this is a complete history.
Newtonsoft.Json.Linq.JToken value;
if (o.TryGetValue("Note", StringComparison.Ordinal, out value))
{
string message = (string)value;
throw new Exception(message);
}
if (o.TryGetValue("Error Message", StringComparison.Ordinal, out value))
{
string message = (string)value;
throw new Exception(message);
}
if (o.TryGetValue("Meta Data", StringComparison.Ordinal, out value))
{
if (value.Type == JTokenType.Object)
{
JObject child = (JObject)value;
if (child.TryGetValue("2. Symbol", StringComparison.Ordinal, out value))
{
history.Symbol = (string)value;
}
}
}
else
{
throw new Exception("Time series data schema has changed");
}
if (o.TryGetValue("Time Series (Daily)", StringComparison.Ordinal, out value))
{
if (value.Type == JTokenType.Object)
{
JObject series = (JObject)value;
foreach (var p in series.Properties().Reverse())
{
DateTime date;
if (DateTime.TryParse(p.Name, out date))
{
value = series.GetValue(p.Name);
if (value.Type == JTokenType.Object)
{
StockQuote quote = new StockQuote()
{
Date = date, Downloaded = DateTime.Now
};
JObject child = (JObject)value;
if (child.TryGetValue("1. open", StringComparison.Ordinal, out value))
{
quote.Open = (decimal)value;
}
if (child.TryGetValue("4. close", StringComparison.Ordinal, out value))
{
quote.Close = (decimal)value;
}
if (child.TryGetValue("2. high", StringComparison.Ordinal, out value))
{
quote.High = (decimal)value;
}
if (child.TryGetValue("3. low", StringComparison.Ordinal, out value))
{
quote.Low = (decimal)value;
}
if (child.TryGetValue("5. volume", StringComparison.Ordinal, out value))
{
quote.Volume = (decimal)value;
}
history.History.Add(quote);
}
}
}
}
}
else
{
throw new Exception("Time series data schema has changed");
}
return(history);
}
private void DownloadQuotes()
{
try
{
while (!_cancelled)
{
int remaining = 0;
string symbol = null;
lock (_pending)
{
if (_pending.Count == 0)
{
lock (_retry)
{
foreach (var item in _retry)
{
_completed--;
_pending.Add(item);
}
_retry.Clear();
}
}
if (_pending.Count > 0)
{
symbol = _pending.FirstOrDefault();
_pending.Remove(symbol);
remaining = _pending.Count;
}
}
if (symbol == null)
{
// done!
break;
}
// even if it fails, we consider the job complete (from a progress point of view).
_completed++;
// weed out any securities that have no symbol or have a
if (string.IsNullOrEmpty(symbol))
{
// skip securities that have no symbol.
}
else if (symbol.IndexOfAny(illegalUrlChars) >= 0)
{
// since we are passing the symbol on an HTTP URI line, we can't pass Uri illegal characters...
OnError(string.Format(Walkabout.Properties.Resources.SkippingSecurityIllegalSymbol, symbol));
OnSymbolNotFound(symbol);
}
else
{
try
{
// this service doesn't want too many calls per second.
int ms = _throttle.GetSleep();
while (ms > 0)
{
if (ms > 1000)
{
int seconds = ms / 1000;
OnError("AlphaVantage quote service needs to sleep for " + seconds + " seconds");
}
else
{
OnError("AlphaVantage quote service needs to sleep for " + ms.ToString() + " ms");
}
OnSuspended(true);
while (!_cancelled && ms > 0)
{
Thread.Sleep(1000);
ms -= 1000;
}
OnSuspended(false);
ms = _throttle.GetSleep();
}
if (_cancelled)
{
break;
}
string uri = string.Format(address, symbol, _settings.ApiKey);
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri);
req.UserAgent = "USER_AGENT=Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;)";
req.Method = "GET";
req.Timeout = 10000;
req.UseDefaultCredentials = false;
_current = req;
Debug.WriteLine("AlphaVantage fetching quote " + symbol);
WebResponse resp = req.GetResponse();
_throttle.RecordCall();
using (Stream stm = resp.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stm, Encoding.UTF8))
{
string json = sr.ReadToEnd();
JObject o = JObject.Parse(json);
StockQuote quote = ParseStockQuote(o);
if (quote == null || quote.Symbol == null)
{
OnError(string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbol));
OnSymbolNotFound(symbol);
}
else if (string.Compare(quote.Symbol, symbol, StringComparison.OrdinalIgnoreCase) != 0)
{
// todo: show appropriate error...
}
else
{
OnQuoteAvailable(quote);
}
}
}
OnError(string.Format(Walkabout.Properties.Resources.FetchedStockQuotes, symbol));
}
catch (System.Net.WebException we)
{
if (we.Status != WebExceptionStatus.RequestCanceled)
{
OnError(string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbol) + "\r\n" + we.Message);
}
else
{
// we cancelled, so bail.
_cancelled = true;
break;
}
HttpWebResponse http = we.Response as HttpWebResponse;
if (http != null)
{
// certain http error codes are fatal.
switch (http.StatusCode)
{
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.InternalServerError:
case HttpStatusCode.Unauthorized:
OnError(http.StatusDescription);
_cancelled = true;
break;
}
}
}
catch (Exception e)
{
// continue
string message = string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbol) + "\r\n" + e.Message;
if (message.Contains("Please visit https://www.alphavantage.co/premium/"))
{
lock (_retry)
{
_retry.Add(symbol);
}
_throttle.CallsThisMinute += this._settings.ApiRequestsPerMinuteLimit;
}
OnComplete(PendingCount == 0, message);
}
Thread.Sleep(1000); // this is so we don't starve out the download service.
}
}
}
catch
{
}
_completed = 0;
if (PendingCount == 0)
{
OnComplete(true, "AlphaVantage download complete");
}
else
{
OnComplete(false, "AlphaVantage download cancelled");
}
_downloadThread = null;
_current = null;
}
private static List <StockQuote> ParseStockQuotes(JObject o)
{
List <StockQuote> result = new List <StockQuote>();
// See https://iexcloud.io/docs/api/#quote, lots more info available than what we are extracting here.
foreach (var pair in o)
{
// KeyValuePair<string, JToken>
string name = pair.Key;
JToken token = pair.Value;
if (token.Type == JTokenType.Object)
{
var quote = new StockQuote()
{
Downloaded = DateTime.Now
};
result.Add(quote);
JObject child = (JObject)token;
JToken value;
if (child.TryGetValue("quote", StringComparison.Ordinal, out value) && value.Type == JTokenType.Object)
{
child = (JObject)value;
if (child.TryGetValue("symbol", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
quote.Symbol = (string)value;
}
if (child.TryGetValue("companyName", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
quote.Name = (string)value;
}
if (child.TryGetValue("open", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
quote.Open = (decimal)value;
}
if (child.TryGetValue("close", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
quote.Close = (decimal)value;
}
if (child.TryGetValue("high", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
quote.High = (decimal)value;
}
if (child.TryGetValue("low", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
quote.Low = (decimal)value;
}
if (child.TryGetValue("latestVolume", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
quote.Volume = (decimal)value;
}
if (child.TryGetValue("closeTime", StringComparison.Ordinal, out value) && value.Type != JTokenType.Null)
{
long ticks = (long)value;
quote.Date = DateTimeOffset.FromUnixTimeMilliseconds(ticks).LocalDateTime;
}
}
}
}
return(result);
}
private void DownloadQuotes()
{
try
{
// This is on a background thread
int max_batch = 100;
List <string> batch = new List <string>();
int remaining = 0;
while (!_cancelled)
{
string symbol = null;
lock (_pending)
{
if (_pending.Count > 0)
{
symbol = _pending.FirstOrDefault();
_pending.Remove(symbol);
remaining = _pending.Count;
}
}
if (symbol == null)
{
// done!
break;
}
// weed out any securities that have no symbol or have a symbol that would be invalid.
if (string.IsNullOrEmpty(symbol))
{
// skip securities that have no symbol.
}
else if (symbol.IndexOfAny(illegalUrlChars) >= 0)
{
// since we are passing the symbol on an HTTP URI line, we can't pass Uri illegal characters...
OnSymbolNotFound(symbol);
OnError(string.Format(Walkabout.Properties.Resources.SkippingSecurityIllegalSymbol, symbol));
}
else
{
batch.Add(symbol);
}
if (batch.Count() == max_batch || remaining == 0)
{
// even it if tails we consider the job completed from a status point of view.
_completed += batch.Count;
string symbols = string.Join(",", batch);
try
{
// this service doesn't want too many calls per second.
int ms = _throttle.GetSleep();
while (ms > 0 && !_cancelled)
{
if (ms > 1000)
{
int seconds = ms / 1000;
OnError("IEXCloud service needs to sleep for " + seconds + " seconds");
}
else
{
OnError("IEXCloud service needs to sleep for " + ms.ToString() + " ms");
}
OnSuspended(true);
while (!_cancelled && ms > 0)
{
Thread.Sleep(1000);
ms -= 1000;
}
OnSuspended(false);
ms = _throttle.GetSleep();
}
if (_cancelled)
{
break;
}
string uri = string.Format(address, symbols, _settings.ApiKey);
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(uri);
req.UserAgent = "USER_AGENT=Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;)";
req.Method = "GET";
req.Timeout = 10000;
req.UseDefaultCredentials = false;
_current = req;
WebResponse resp = req.GetResponse();
_throttle.RecordCall();
using (Stream stm = resp.GetResponseStream())
{
using (StreamReader sr = new StreamReader(stm, Encoding.UTF8))
{
string json = sr.ReadToEnd();
JObject o = JObject.Parse(json);
List <StockQuote> result = ParseStockQuotes(o);
// make sure they are all returned, and report errors for any that are not.
foreach (string s in batch)
{
StockQuote q = (from i in result where string.Compare(i.Symbol, s, StringComparison.OrdinalIgnoreCase) == 0 select i).FirstOrDefault();
if (q == null)
{
OnError(string.Format("No quote returned for symbol {0}", s));
OnSymbolNotFound(s);
}
else
{
OnQuoteAvailable(q);
}
}
}
}
Thread.Sleep(1000); // there is also a minimum sleep between requests that we must enforce.
symbols = string.Join(", ", batch);
OnError(string.Format(Walkabout.Properties.Resources.FetchedStockQuotes, symbols));
}
catch (System.Net.WebException we)
{
if (we.Status != WebExceptionStatus.RequestCanceled)
{
OnError(string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbols) + "\r\n" + we.Message);
}
else
{
// we cancelled, so bail.
_cancelled = true;
break;
}
HttpWebResponse http = we.Response as HttpWebResponse;
if (http != null)
{
// certain http error codes are fatal.
switch (http.StatusCode)
{
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.InternalServerError:
case HttpStatusCode.Unauthorized:
OnError(http.StatusDescription);
_cancelled = true;
break;
}
}
}
catch (Exception e)
{
// continue
OnError(string.Format(Walkabout.Properties.Resources.ErrorFetchingSymbols, symbols) + "\r\n" + e.Message);
}
batch.Clear();
}
_current = null;
}
}
catch
{
}
if (PendingCount == 0)
{
OnComplete(true, "IEXCloud download complete");
}
else
{
OnComplete(false, "IEXCloud download cancelled");
}
_downloadThread = null;
_current = null;
_completed = 0;
}
