public void Init()
{
_dataEachDay.Clear();
using (var db = new MarketContext())
{
using (var connection = db.GetConnection())
{
connection.Open();
if (_log)
{
Console.WriteLine("Reading all eod entries");
}
var allDays = db.GetAllEodEntries(connection);
if (_log)
{
Console.WriteLine("organizing eod entries into days");
}
var byDay = allDays.GroupBy(x => x.Date).ToDictionary(x => x.Key);
var days = allDays.Select(x => x.Date).Distinct().OrderBy(x => x).ToList();
var count = 0;
foreach (var day in days)
{
// exclude holidays
if (byDay[day].Any(x => x.Vol > 0))
{
_dataEachDay[count++] = byDay[day].Where(x => x.Close > 0 && x.Vol > 0).ToList();
}
}
_allTickers = allDays.Select(x => x.Ticker).Distinct().ToList();
if (_log)
{
Console.WriteLine("Gathering related tickers per ticker");
}
foreach (var ticker in _allTickers)
{
var relationships = db.GetRelations(ticker, connection);
_parentToRelations[ticker] = new HashSet <string>(relationships.Select(x => x.RelatedTicker));
}
}
}
}
private static void Download(string exchange, DateTime startDate, DateTime endDate, bool initialRun)
{
var regexKey = new Regex("&k=([a-z0-9]*)&");
const string templateUrl = "http://eoddata.com/data/filedownload.aspx?e={0}&sd={1}&ed={2}&d=1&k={3}&o=d&ea=1&p=0";
var session = new AgentSession();
var action = new AgentAction("http://eoddata.com/", false);
var document = AgentHandler.Instance.PerformAction(session, action);
var forms = FormContainer.ParseAll(document.ResponseString, document.Uri);
var loginForm = forms.FirstOrDefault(x => x.Elements.Any(y => y.Name.EndsWith("txtEmail")) &&
x.Elements.Any(y => y.Name.EndsWith("txtPassword")) &&
x.Elements.Any(y => y.Name.EndsWith("btnLogin")));
if (loginForm == null)
{
Console.WriteLine("Can't log in");
return;
}
loginForm.Elements.First(x => x.Name.EndsWith("txtEmail")).Value = "*****@*****.**";
loginForm.Elements.First(x => x.Name.EndsWith("txtPassword")).Value = "3b4f5w8i";
action = loginForm.CreateLocalAction();
action.WebURL = "http://eoddata.com/";
var loggedIn = AgentHandler.Instance.PerformAction(session, action);
if (!loggedIn.ResponseString.Contains("chris stafford"))
{
throw new Exception("Can't log in");
}
var downloadAction = new AgentAction("http://eoddata.com/download.aspx", false);
var downloadResult = AgentHandler.Instance.PerformAction(session, downloadAction);
var key = regexKey.Match(downloadResult.ResponseString).Result("$1");
var downloadUrl = string.Format(
templateUrl,
exchange,
startDate.ToString("yyyyMMdd"),
endDate.ToString("yyyyMMdd"),
key);
Console.WriteLine($"Downloading {exchange}");
var downloadDataAction = new AgentAction(downloadUrl, false);
var data = AgentHandler.Instance.PerformAction(session, downloadDataAction);
Console.WriteLine("Done. Saving.");
var parseDate = new Func <string, DateTime>(datetimestr => new DateTime(
int.Parse(datetimestr.Substring(0, 4)),
int.Parse(datetimestr.Substring(4, 2)),
int.Parse(datetimestr.Substring(6, 2))));
var lastTicker = string.Empty;
using (var db = new MarketContext())
{
using (var connection = db.GetConnection())
{
connection.Open();
using (var transaction = connection.BeginTransaction())
{
try
{
data.ResponseString.Split('\n').ToList().ForEach(line =>
{
var items = line.Split(',');
if (items.Length < 8)
{
return;
}
var eoddata = new EodEntry
{
Ticker = exchange + ":" + items[0],
Per = items[1],
Date = parseDate(items[2]),
Open = decimal.Parse(items[3]),
High = decimal.Parse(items[4]),
Low = decimal.Parse(items[5]),
Close = decimal.Parse(items[6]),
Vol = double.Parse(items[7]),
};
if (lastTicker != eoddata.Ticker)
{
lastTicker = eoddata.Ticker;
Console.WriteLine(eoddata.Ticker);
}
db.Insert(eoddata, connection, initialRun);
});
}
finally
{
transaction.Commit();
}
}
}
}
Console.WriteLine("Done");
}
public static void Run()
{
var regex = new Regex("streaming:\\[.*?\\]", RegexOptions.Singleline);
using (var db = new MarketContext())
{
using (var connection = db.GetConnection())
{
connection.Open();
var tickers = db.GetMarketplaceTickers(connection);
const string financeUrl = "https://www.google.com/finance?q={0}";
var session = new AgentSession();
var count = 0;
foreach (var ticker in tickers)
{
var transaction = connection.BeginTransaction();
try
{
Console.Write($"{++count} of {tickers.Count}: {ticker} - ");
var url = string.Format(financeUrl, ticker.Replace(":", "%3A"));
var action = new AgentAction(url, false);
var document = AgentHandler.Instance.PerformAction(session, action);
var match = regex.Match(document.ResponseString);
if (!match.Success)
{
Console.WriteLine("No relations!");
continue;
}
var json = "{" + match + "}";
var relatedItems = ((JObject)JsonConvert.DeserializeObject(json)).First?.First?.Children().AsJEnumerable()
.ToList();
if (relatedItems == null)
{
Console.WriteLine("\nERROR!");
continue;
}
foreach (var relatedItem in relatedItems)
{
var values = relatedItem.Children().Select(x => (JProperty)x).ToDictionary(x => x.Name);
var relatedTicker = values["s"].Value;
var relatedExchange = values["e"].Value;
if (ticker.EndsWith($":{relatedTicker}"))
{
continue;
}
Console.Write($"{relatedTicker} ");
if (tickers.Contains($"{relatedExchange}:{relatedTicker}"))
{
db.AddRelationship(ticker, $"{relatedExchange}:{relatedTicker}", connection);
continue;
}
var probableTicker = tickers.FirstOrDefault(x => x.EndsWith($":{relatedTicker}"));
if (probableTicker != null)
{
db.AddRelationship(ticker, probableTicker, connection);
Console.Write($"({probableTicker}) ");
}
else
{
Console.Write("! ");
}
}
Console.WriteLine();
}
finally
{
transaction.Commit();
}
}
}
}
Console.WriteLine("Done.");
Console.ReadLine();
}
