static void Main(string[] args)
{
try
{
logger.Debug("Starting Agent Process");
session = new AgentSession();
}
catch (Exception ex)
{
logger.Error(ex, $"Cannot start Agent Session");
}
try
{
server = new WebSocketServer("ws://0.0.0.0:8181");
server.RestartAfterListenError = true;
server.Start(ConfigureEvents);
}
catch (Exception ex)
{
logger.Fatal(ex, "Cannot start the WebSocket listener");
return;
}
logger.Debug($"Service has been started on {server.Location}:{server.Port}");
Console.Write("Press any key to shutdown the server...");
Console.Read();
}
protected override string OnInitState(IEventMessage message)
{
var eventMessage = TypeCast(message);
if (eventMessage == null) return StateType.INIT;
var agentSession = new AgentSession(eventMessage.Handler)
{
ConferenceId = eventMessage.ConferenceId,
AgentName = eventMessage.AgentName,
PublisherUrl = eventMessage.PresentationUrl,
EndpointAddress = eventMessage.EndpointAddress,
IsPublisher = true
};
sessionContext.Register(agentSession);
agentSession.Process(new ResponseEvent
{
AgentName = sessionContext.SessionOfAgent.AgentName,
ConferenceId = sessionContext.SessionOfAgent.ConferenceId,
PresentationUrl = eventMessage.PresentationUrl,
SessionId = sessionContext.SessionId,
ResponseStatus = ResponseStatus.Ok
});
return StateType.READY;
}
public DispatcherBase(AgentSession session, double interval)
{
Session = session;
_jobScheduler = new Timer();
_jobScheduler.Elapsed += PerformWork;
_jobScheduler.AutoReset = true;
Interval = interval;
}
public NodeStatusDispatcher(AgentSession session, double interval) : base(session, interval)
{
statusProbes = new List <StatusProbeBase>();
statusProbes.Add(new NodeLogStatusProbe());
statusProbes.Add(new NodeOperationStatusProbe());
statusProbes.Add(new NodeDeploymentStatusProbe());
statusProbes.Add(new NodeProcessStatusProbe());
}
private void ScrapePage(string pageName)
{
Console.WriteLine("Doing page " + pageName);
string url = @"http://www.nyse.com/about/listed/lc_ny_name_`.js".Replace("`", pageName);
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document = AgentHandler.Instance.PerformAction(session, action);
string[] segments = document.ResponseString.Split(']');
foreach (string segment in segments)
{
int startIdx = segment.IndexOf("\"") + 1;
if (startIdx == 0)
{
continue;
}
int endIdx = segment.IndexOf("\"", startIdx);
string stockTicker = segment.Substring(startIdx, endIdx - startIdx);
startIdx = segment.IndexOf("\"", endIdx + 1) + 1;
endIdx = segment.IndexOf("\"", startIdx);
string companyName = segment.Substring(startIdx, endIdx - startIdx);
startIdx = segment.IndexOf("\"", endIdx + 1) + 1;
endIdx = segment.IndexOf("\"", startIdx);
string country = segment.Substring(startIdx, endIdx - startIdx);
;
StockName name = new StockName
{
Stock = stockTicker,
Company_Name = companyName,
Country = country,
Exchange = "NYSE"
};
if (!Madness.StockNames.Any(x => x.Stock == stockTicker))
{
Console.WriteLine("Found new stock: " + stockTicker);
Madness.AddToStockNames(name);
}
}
}
private static void Download(string exchange, DateTime startDate, DateTime endDate, bool initialRun)
{
var regexKey = new Regex("&k=([a-z0-9]*)&");
const string templateUrl = "http://eoddata.com/data/filedownload.aspx?e={0}&sd={1}&ed={2}&d=1&k={3}&o=d&ea=1&p=0";
var session = new AgentSession();
var action = new AgentAction("http://eoddata.com/", false);
var document = AgentHandler.Instance.PerformAction(session, action);
var forms = FormContainer.ParseAll(document.ResponseString, document.Uri);
var loginForm = forms.FirstOrDefault(x => x.Elements.Any(y => y.Name.EndsWith("txtEmail")) &&
x.Elements.Any(y => y.Name.EndsWith("txtPassword")) &&
x.Elements.Any(y => y.Name.EndsWith("btnLogin")));
if (loginForm == null)
{
Console.WriteLine("Can't log in");
return;
}
loginForm.Elements.First(x => x.Name.EndsWith("txtEmail")).Value = "*****@*****.**";
loginForm.Elements.First(x => x.Name.EndsWith("txtPassword")).Value = "3b4f5w8i";
action = loginForm.CreateLocalAction();
action.WebURL = "http://eoddata.com/";
var loggedIn = AgentHandler.Instance.PerformAction(session, action);
if (!loggedIn.ResponseString.Contains("chris stafford"))
{
throw new Exception("Can't log in");
}
var downloadAction = new AgentAction("http://eoddata.com/download.aspx", false);
var downloadResult = AgentHandler.Instance.PerformAction(session, downloadAction);
var key = regexKey.Match(downloadResult.ResponseString).Result("$1");
var downloadUrl = string.Format(
templateUrl,
exchange,
startDate.ToString("yyyyMMdd"),
endDate.ToString("yyyyMMdd"),
key);
Console.WriteLine($"Downloading {exchange}");
var downloadDataAction = new AgentAction(downloadUrl, false);
var data = AgentHandler.Instance.PerformAction(session, downloadDataAction);
Console.WriteLine("Done. Saving.");
var parseDate = new Func <string, DateTime>(datetimestr => new DateTime(
int.Parse(datetimestr.Substring(0, 4)),
int.Parse(datetimestr.Substring(4, 2)),
int.Parse(datetimestr.Substring(6, 2))));
var lastTicker = string.Empty;
using (var db = new MarketContext())
{
using (var connection = db.GetConnection())
{
connection.Open();
using (var transaction = connection.BeginTransaction())
{
try
{
data.ResponseString.Split('\n').ToList().ForEach(line =>
{
var items = line.Split(',');
if (items.Length < 8)
{
return;
}
var eoddata = new EodEntry
{
Ticker = exchange + ":" + items[0],
Per = items[1],
Date = parseDate(items[2]),
Open = decimal.Parse(items[3]),
High = decimal.Parse(items[4]),
Low = decimal.Parse(items[5]),
Close = decimal.Parse(items[6]),
Vol = double.Parse(items[7]),
};
if (lastTicker != eoddata.Ticker)
{
lastTicker = eoddata.Ticker;
Console.WriteLine(eoddata.Ticker);
}
db.Insert(eoddata, connection, initialRun);
});
}
finally
{
transaction.Commit();
}
}
}
}
Console.WriteLine("Done");
}
public AgentHealthStatusProbe(AgentSession session)
{
this.session = session;
}
public AgentHealthcheckDispatcher(AgentSession session, double interval) : base(session, interval)
{
agentHealthStatusProbe = new AgentHealthStatusProbe(session);
}
public static void PullQuotes(DateTime start, DateTime stop, string stockName)
{
ModelMadness madness = new ModelMadness();
StockName stock = (from q in madness.StockNames where q.Stock == stockName select q).First();
Console.WriteLine("Working on " + stock.Stock);
//http://ichart.finance.yahoo.com/table.csv?s=TKR&a=00&b=1&c=2000&d=01&e=18&f=2010&g=d&ignore=.csv
string url = @"http://ichart.finance.yahoo.com/table.csv?s=" +
stock.Stock + "&a=" + TwoDig(start.Month - 1) + "&b=" + start.Day + "&c=" + start.Year +
"&d=" + TwoDig(stop.Month - 1) + "&e=" + stop.Day + "&f=" + stop.Year +
"&g=d&ignore=.csv";
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
bool found = false;
AgentDocument document = null;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
found = true;
}
catch (Exception ex)
{
Console.WriteLine("ERROR: " + url + " - " + ex.Message);
}
if (!found)
{
return;
}
//Date,Open,High,Low,Close,Volume,Adj Close
string[] rows = document.ResponseString.Split('\n');
if (rows.Length < 2)
{
return;
}
for (int i = 1; i < rows.Length; i++)
{
string row = rows[i];
string[] fields = row.Split(',');
if (fields.Length < 7)
{
Console.WriteLine((i - 2) + " records stored");
continue;
}
quote q = new quote
{
day = (DateTime) new DateTimeConverter().ConvertFrom(fields[0]),
open = decimal.Parse(fields[1]),
high = decimal.Parse(fields[2]),
low = decimal.Parse(fields[3]),
close = decimal.Parse(fields[4]),
volume = long.Parse(fields[5]),
adjusted_close = decimal.Parse(fields[6]),
StockName = stock
};
madness.AddToquotes(q);
}
madness.SaveChanges();
}
public static void Run()
{
var regex = new Regex("streaming:\\[.*?\\]", RegexOptions.Singleline);
using (var db = new MarketContext())
{
using (var connection = db.GetConnection())
{
connection.Open();
var tickers = db.GetMarketplaceTickers(connection);
const string financeUrl = "https://www.google.com/finance?q={0}";
var session = new AgentSession();
var count = 0;
foreach (var ticker in tickers)
{
var transaction = connection.BeginTransaction();
try
{
Console.Write($"{++count} of {tickers.Count}: {ticker} - ");
var url = string.Format(financeUrl, ticker.Replace(":", "%3A"));
var action = new AgentAction(url, false);
var document = AgentHandler.Instance.PerformAction(session, action);
var match = regex.Match(document.ResponseString);
if (!match.Success)
{
Console.WriteLine("No relations!");
continue;
}
var json = "{" + match + "}";
var relatedItems = ((JObject)JsonConvert.DeserializeObject(json)).First?.First?.Children().AsJEnumerable()
.ToList();
if (relatedItems == null)
{
Console.WriteLine("\nERROR!");
continue;
}
foreach (var relatedItem in relatedItems)
{
var values = relatedItem.Children().Select(x => (JProperty)x).ToDictionary(x => x.Name);
var relatedTicker = values["s"].Value;
var relatedExchange = values["e"].Value;
if (ticker.EndsWith($":{relatedTicker}"))
{
continue;
}
Console.Write($"{relatedTicker} ");
if (tickers.Contains($"{relatedExchange}:{relatedTicker}"))
{
db.AddRelationship(ticker, $"{relatedExchange}:{relatedTicker}", connection);
continue;
}
var probableTicker = tickers.FirstOrDefault(x => x.EndsWith($":{relatedTicker}"));
if (probableTicker != null)
{
db.AddRelationship(ticker, probableTicker, connection);
Console.Write($"({probableTicker}) ");
}
else
{
Console.Write("! ");
}
}
Console.WriteLine();
}
finally
{
transaction.Commit();
}
}
}
}
Console.WriteLine("Done.");
Console.ReadLine();
}
public AgentDocument PerformAction(AgentSession session, AgentAction action, int timeout)
{
AgentDocument document;
// these resources must be released if any exceptions occur
HttpWebResponse actionResponse = null;
Stream responseStream = null;
try
{
// set up the request and its headers
var actionRequest = (HttpWebRequest)WebRequest.Create(new Uri(action.WebURL));
if (!string.IsNullOrEmpty(action.Referer))
{
actionRequest.Referer = action.Referer;
}
if (action.IsPost)
{
actionRequest.Method = "POST";
}
else
{
actionRequest.Method = "GET";
}
actionRequest.ContentType = "application/x-www-form-urlencoded";
if (action.UserAgent == null)
{
actionRequest.UserAgent =
"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0";
}
else
{
actionRequest.UserAgent = action.UserAgent;
}
actionRequest.Accept =
"text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,video/x-mng,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1";
actionRequest.AllowAutoRedirect = action.AllowRedirect;
actionRequest.KeepAlive = false;
if (!string.IsNullOrEmpty(action.ProxyAddress))
{
var parts = action.ProxyAddress.Split(':');
if (parts.Length == 2)
{
actionRequest.Proxy = new WebProxy(parts[0], int.Parse(parts[1]));
}
else
{
actionRequest.Proxy = new WebProxy(parts[0]);
}
}
if (timeout > 0)
{
actionRequest.Timeout = timeout;
}
actionRequest.Headers.Add("Accept-Language", "en-us,en;q=0.5");
actionRequest.ProtocolVersion = new Version(action.Version);
actionRequest.CookieContainer = session.Cookies;
if (action.IsPost)
{
if (action.Attachment == null)
{
WriteContent(action, actionRequest);
}
else
{
WriteFileAttachmentContent(action, actionRequest);
}
}
document = new AgentDocument();
actionRequest.ServicePoint.Expect100Continue = false;
actionResponse = (HttpWebResponse)actionRequest.GetResponse();
document.RedirectUri = action.AllowRedirect
? actionResponse.ResponseUri.ToString()
: actionResponse.GetResponseHeader("Location");
responseStream = actionResponse.GetResponseStream();
Debug.Assert(responseStream != null, "responseStream != null");
var responseBuilder = new StringWriter();
var buffer = new byte[1024];
int n;
do
{
n = responseStream.Read(buffer, 0, buffer.Length);
var charBuffer = new char[buffer.Length];
for (var i = 0; i < buffer.Length; i++)
{
charBuffer[i] = (char)buffer[i];
}
responseBuilder.Write(charBuffer, 0, n);
} while (n > 0);
document.ResponseString = responseBuilder.GetStringBuilder().ToString();
responseBuilder.Close();
responseBuilder.Dispose();
document.Uri = actionResponse.ResponseUri.ToString();
}
finally
{
try
{
responseStream?.Close();
}
catch
{
// ignored
}
try
{
actionResponse?.Close();
}
catch
{
// ignored
}
}
return(document);
}
// *** Public API ***
public AgentDocument PerformAction(AgentSession session, AgentAction action)
{
return(PerformAction(session, action, 0));
}
private static void GrabProfile(string symbol)
{
Console.Write("Grabbing Profile for " + symbol);
string url = @"http://finance.yahoo.com/q/pr?s=" + symbol;
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
}
catch
{
return;
}
string doc = document.ResponseString.ToLower();
var extractor = new ProgressiveDataExtractor(doc, ">details<");
if (extractor.NotFound)
{
Console.WriteLine(" - not found");
return;
}
Console.WriteLine(" - found");
ModelMadness madness = new ModelMadness();
string sector = extractor.ExtractString("sector");
Sector dbSector = null;
Industry dbIndustry = null;
if (sector != null && sector != "n/a")
{
sector = sector.Replace("&", "&");
string industry = extractor.ExtractString("industry").Replace("&", "&");
var obj = (from q in madness.Sectors where q.Sector1 == sector select q).ToArray();
if (obj.Length == 1)
{
dbSector = obj[0];
}
else
{
dbSector = new Sector {
Sector1 = sector
};
madness.AddToSectors(dbSector);
}
var obj2 = (from q in madness.Industries
where q.Sector.Sector1 == sector && q.Iindustry == industry
select q).ToArray();
if (obj2.Length == 1)
{
dbIndustry = obj2[0];
}
else
{
dbIndustry = new Industry {
Iindustry = industry, Sector = dbSector
};
madness.AddToIndustries(dbIndustry);
}
}
StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First();
CompanyProfile profile = new CompanyProfile
{
Sector = dbSector,
Industry = dbIndustry,
num_employees = extractor.ExtractInt("full time employees"),
StockName = stock,
scrape_day = DateTime.Now.Date
};
profile.summary = extractor.ExtractString("business summary");
profile.cgq = extractor.ExtractDecimal("corporate governance", "<b");
extractor.ExtractString("key executives");
int totalAge = 0;
int numAges = 0;
long totalPay = 0;
int numPays = 0;
int? curAge;
long?curPay;
do
{
curAge = extractor.ExtractInt("yfnc_tabledata1", "</b");
if (curAge > 111)
{
curAge = null;
}
if (curAge != null)
{
numAges++;
totalAge += (int)curAge;
curPay = extractor.ExtractLong("yfnc_tabledata1", "nowrap");
if (curPay != null)
{
numPays++;
totalPay += (long)curPay;
}
}
} while (curAge != null);
profile.avg_executive_age = totalAge == 0 ? null : (int?)totalAge / numAges;
profile.avg_executive_pay = totalPay == 0 ? null : (long?)totalPay / numPays;
madness.AddToCompanyProfiles(profile);
madness.SaveChanges();
}
private static void GrabStats(string symbol)
{
Console.Write("Grabbing Key Stats for " + symbol);
string url = @"http://finance.yahoo.com/q/ks?s=" + symbol;
AgentSession session = new AgentSession();
AgentAction action = new AgentAction(url, false);
AgentDocument document;
try
{
document = AgentHandler.Instance.PerformAction(session, action);
}
catch
{
return;
}
var doc = document.ResponseString;
var extractor = new ProgressiveDataExtractor(doc, "Valuation Measures");
if (extractor.NotFound)
{
Console.WriteLine(" - Not found");
return;
}
Console.WriteLine(" - found");
ModelMadness madness = new ModelMadness();
StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First();
CompanyStat stats = new CompanyStat
{
market_cap = extractor.ExtractLong("Market Cap"),
enterprise_value = extractor.ExtractLong("Enterprise Value"),
trailing_pe = extractor.ExtractDecimal("Trailing P/E"),
forward_pe = extractor.ExtractDecimal("Forward P/E"),
peg_ratio = extractor.ExtractDecimal("PEG Ratio"),
price_sales = extractor.ExtractDecimal("Price/Sales"),
price_book = extractor.ExtractDecimal("Price/Book"),
ent_value_rev = extractor.ExtractLong("Enterprise Value/Revenue"),
ent_value_ebitda = extractor.ExtractDecimal("Enterprise Value/EBITDA"),
fiscal_year_ends = extractor.ExtractString("Fiscal Year Ends"),
most_recent_quarter = extractor.ExtractString("Most Recent Quarter"),
profit_margin = extractor.ExtractDecimal("Profit Margin"),
operating_margin = extractor.ExtractDecimal("Operating Margin"),
return_on_assets = extractor.ExtractDecimal("Return on Assets"),
return_on_equity = extractor.ExtractDecimal("Return on Equity"),
revenue = extractor.ExtractLong("Revenue"),
revenue_per_share = extractor.ExtractDecimal("Revenue Per Share"),
qrt_rev_growth = extractor.ExtractDecimal("Qtrly Revenue Growth"),
gross_profit = extractor.ExtractLong("Gross Profit"),
ebitda = extractor.ExtractLong("EBITDA"),
net_income_a_c = extractor.ExtractLong("Net Income Avl to Common"),
diluted_eps = extractor.ExtractDecimal("Diluted EPS"),
qrt_earnings_growth = extractor.ExtractDecimal("Qtrly Earnings Growth"),
total_cash = extractor.ExtractLong("Total Cash"),
cash_per_share = extractor.ExtractDecimal("Total Cash Per Share"),
total_debt = extractor.ExtractLong("Total Debt"),
total_debt_equit = extractor.ExtractDecimal("Total Debt/Equity"),
current_ratio = extractor.ExtractDecimal("Current Ratio"),
book_value_p_share = extractor.ExtractDecimal("Book Value Per Share"),
operating_cash_flow = extractor.ExtractLong("Operating Cash Flow"),
levered_free_cash_flow = extractor.ExtractLong("Levered Free Cash Flow"),
StockName = stock,
scrape_day = DateTime.Now.Date
};
madness.AddToCompanyStats(stats);
madness.SaveChanges();
}
public ResourceFromAgentDispatcher(AgentSession session, double interval) : base(session, interval)
{
resourceStreams = new Dictionary <Resource, FileStream>();
}
