static void Main(string[] args) { try { logger.Debug("Starting Agent Process"); session = new AgentSession(); } catch (Exception ex) { logger.Error(ex, $"Cannot start Agent Session"); } try { server = new WebSocketServer("ws://0.0.0.0:8181"); server.RestartAfterListenError = true; server.Start(ConfigureEvents); } catch (Exception ex) { logger.Fatal(ex, "Cannot start the WebSocket listener"); return; } logger.Debug($"Service has been started on {server.Location}:{server.Port}"); Console.Write("Press any key to shutdown the server..."); Console.Read(); }
protected override string OnInitState(IEventMessage message) { var eventMessage = TypeCast(message); if (eventMessage == null) return StateType.INIT; var agentSession = new AgentSession(eventMessage.Handler) { ConferenceId = eventMessage.ConferenceId, AgentName = eventMessage.AgentName, PublisherUrl = eventMessage.PresentationUrl, EndpointAddress = eventMessage.EndpointAddress, IsPublisher = true }; sessionContext.Register(agentSession); agentSession.Process(new ResponseEvent { AgentName = sessionContext.SessionOfAgent.AgentName, ConferenceId = sessionContext.SessionOfAgent.ConferenceId, PresentationUrl = eventMessage.PresentationUrl, SessionId = sessionContext.SessionId, ResponseStatus = ResponseStatus.Ok }); return StateType.READY; }
public DispatcherBase(AgentSession session, double interval) { Session = session; _jobScheduler = new Timer(); _jobScheduler.Elapsed += PerformWork; _jobScheduler.AutoReset = true; Interval = interval; }
public NodeStatusDispatcher(AgentSession session, double interval) : base(session, interval) { statusProbes = new List <StatusProbeBase>(); statusProbes.Add(new NodeLogStatusProbe()); statusProbes.Add(new NodeOperationStatusProbe()); statusProbes.Add(new NodeDeploymentStatusProbe()); statusProbes.Add(new NodeProcessStatusProbe()); }
private void ScrapePage(string pageName) { Console.WriteLine("Doing page " + pageName); string url = @"http://www.nyse.com/about/listed/lc_ny_name_`.js".Replace("`", pageName); AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); AgentDocument document = AgentHandler.Instance.PerformAction(session, action); string[] segments = document.ResponseString.Split(']'); foreach (string segment in segments) { int startIdx = segment.IndexOf("\"") + 1; if (startIdx == 0) { continue; } int endIdx = segment.IndexOf("\"", startIdx); string stockTicker = segment.Substring(startIdx, endIdx - startIdx); startIdx = segment.IndexOf("\"", endIdx + 1) + 1; endIdx = segment.IndexOf("\"", startIdx); string companyName = segment.Substring(startIdx, endIdx - startIdx); startIdx = segment.IndexOf("\"", endIdx + 1) + 1; endIdx = segment.IndexOf("\"", startIdx); string country = segment.Substring(startIdx, endIdx - startIdx); ; StockName name = new StockName { Stock = stockTicker, Company_Name = companyName, Country = country, Exchange = "NYSE" }; if (!Madness.StockNames.Any(x => x.Stock == stockTicker)) { Console.WriteLine("Found new stock: " + stockTicker); Madness.AddToStockNames(name); } } }
private static void Download(string exchange, DateTime startDate, DateTime endDate, bool initialRun) { var regexKey = new Regex("&k=([a-z0-9]*)&"); const string templateUrl = "http://eoddata.com/data/filedownload.aspx?e={0}&sd={1}&ed={2}&d=1&k={3}&o=d&ea=1&p=0"; var session = new AgentSession(); var action = new AgentAction("http://eoddata.com/", false); var document = AgentHandler.Instance.PerformAction(session, action); var forms = FormContainer.ParseAll(document.ResponseString, document.Uri); var loginForm = forms.FirstOrDefault(x => x.Elements.Any(y => y.Name.EndsWith("txtEmail")) && x.Elements.Any(y => y.Name.EndsWith("txtPassword")) && x.Elements.Any(y => y.Name.EndsWith("btnLogin"))); if (loginForm == null) { Console.WriteLine("Can't log in"); return; } loginForm.Elements.First(x => x.Name.EndsWith("txtEmail")).Value = "*****@*****.**"; loginForm.Elements.First(x => x.Name.EndsWith("txtPassword")).Value = "3b4f5w8i"; action = loginForm.CreateLocalAction(); action.WebURL = "http://eoddata.com/"; var loggedIn = AgentHandler.Instance.PerformAction(session, action); if (!loggedIn.ResponseString.Contains("chris stafford")) { throw new Exception("Can't log in"); } var downloadAction = new AgentAction("http://eoddata.com/download.aspx", false); var downloadResult = AgentHandler.Instance.PerformAction(session, downloadAction); var key = regexKey.Match(downloadResult.ResponseString).Result("$1"); var downloadUrl = string.Format( templateUrl, exchange, startDate.ToString("yyyyMMdd"), endDate.ToString("yyyyMMdd"), key); Console.WriteLine($"Downloading {exchange}"); var downloadDataAction = new AgentAction(downloadUrl, false); var data = AgentHandler.Instance.PerformAction(session, downloadDataAction); Console.WriteLine("Done. Saving."); var parseDate = new Func <string, DateTime>(datetimestr => new DateTime( int.Parse(datetimestr.Substring(0, 4)), int.Parse(datetimestr.Substring(4, 2)), int.Parse(datetimestr.Substring(6, 2)))); var lastTicker = string.Empty; using (var db = new MarketContext()) { using (var connection = db.GetConnection()) { connection.Open(); using (var transaction = connection.BeginTransaction()) { try { data.ResponseString.Split('\n').ToList().ForEach(line => { var items = line.Split(','); if (items.Length < 8) { return; } var eoddata = new EodEntry { Ticker = exchange + ":" + items[0], Per = items[1], Date = parseDate(items[2]), Open = decimal.Parse(items[3]), High = decimal.Parse(items[4]), Low = decimal.Parse(items[5]), Close = decimal.Parse(items[6]), Vol = double.Parse(items[7]), }; if (lastTicker != eoddata.Ticker) { lastTicker = eoddata.Ticker; Console.WriteLine(eoddata.Ticker); } db.Insert(eoddata, connection, initialRun); }); } finally { transaction.Commit(); } } } } Console.WriteLine("Done"); }
public AgentHealthStatusProbe(AgentSession session) { this.session = session; }
public AgentHealthcheckDispatcher(AgentSession session, double interval) : base(session, interval) { agentHealthStatusProbe = new AgentHealthStatusProbe(session); }
public static void PullQuotes(DateTime start, DateTime stop, string stockName) { ModelMadness madness = new ModelMadness(); StockName stock = (from q in madness.StockNames where q.Stock == stockName select q).First(); Console.WriteLine("Working on " + stock.Stock); //http://ichart.finance.yahoo.com/table.csv?s=TKR&a=00&b=1&c=2000&d=01&e=18&f=2010&g=d&ignore=.csv string url = @"http://ichart.finance.yahoo.com/table.csv?s=" + stock.Stock + "&a=" + TwoDig(start.Month - 1) + "&b=" + start.Day + "&c=" + start.Year + "&d=" + TwoDig(stop.Month - 1) + "&e=" + stop.Day + "&f=" + stop.Year + "&g=d&ignore=.csv"; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); bool found = false; AgentDocument document = null; try { document = AgentHandler.Instance.PerformAction(session, action); found = true; } catch (Exception ex) { Console.WriteLine("ERROR: " + url + " - " + ex.Message); } if (!found) { return; } //Date,Open,High,Low,Close,Volume,Adj Close string[] rows = document.ResponseString.Split('\n'); if (rows.Length < 2) { return; } for (int i = 1; i < rows.Length; i++) { string row = rows[i]; string[] fields = row.Split(','); if (fields.Length < 7) { Console.WriteLine((i - 2) + " records stored"); continue; } quote q = new quote { day = (DateTime) new DateTimeConverter().ConvertFrom(fields[0]), open = decimal.Parse(fields[1]), high = decimal.Parse(fields[2]), low = decimal.Parse(fields[3]), close = decimal.Parse(fields[4]), volume = long.Parse(fields[5]), adjusted_close = decimal.Parse(fields[6]), StockName = stock }; madness.AddToquotes(q); } madness.SaveChanges(); }
public static void Run() { var regex = new Regex("streaming:\\[.*?\\]", RegexOptions.Singleline); using (var db = new MarketContext()) { using (var connection = db.GetConnection()) { connection.Open(); var tickers = db.GetMarketplaceTickers(connection); const string financeUrl = "https://www.google.com/finance?q={0}"; var session = new AgentSession(); var count = 0; foreach (var ticker in tickers) { var transaction = connection.BeginTransaction(); try { Console.Write($"{++count} of {tickers.Count}: {ticker} - "); var url = string.Format(financeUrl, ticker.Replace(":", "%3A")); var action = new AgentAction(url, false); var document = AgentHandler.Instance.PerformAction(session, action); var match = regex.Match(document.ResponseString); if (!match.Success) { Console.WriteLine("No relations!"); continue; } var json = "{" + match + "}"; var relatedItems = ((JObject)JsonConvert.DeserializeObject(json)).First?.First?.Children().AsJEnumerable() .ToList(); if (relatedItems == null) { Console.WriteLine("\nERROR!"); continue; } foreach (var relatedItem in relatedItems) { var values = relatedItem.Children().Select(x => (JProperty)x).ToDictionary(x => x.Name); var relatedTicker = values["s"].Value; var relatedExchange = values["e"].Value; if (ticker.EndsWith($":{relatedTicker}")) { continue; } Console.Write($"{relatedTicker} "); if (tickers.Contains($"{relatedExchange}:{relatedTicker}")) { db.AddRelationship(ticker, $"{relatedExchange}:{relatedTicker}", connection); continue; } var probableTicker = tickers.FirstOrDefault(x => x.EndsWith($":{relatedTicker}")); if (probableTicker != null) { db.AddRelationship(ticker, probableTicker, connection); Console.Write($"({probableTicker}) "); } else { Console.Write("! "); } } Console.WriteLine(); } finally { transaction.Commit(); } } } } Console.WriteLine("Done."); Console.ReadLine(); }
public AgentDocument PerformAction(AgentSession session, AgentAction action, int timeout) { AgentDocument document; // these resources must be released if any exceptions occur HttpWebResponse actionResponse = null; Stream responseStream = null; try { // set up the request and its headers var actionRequest = (HttpWebRequest)WebRequest.Create(new Uri(action.WebURL)); if (!string.IsNullOrEmpty(action.Referer)) { actionRequest.Referer = action.Referer; } if (action.IsPost) { actionRequest.Method = "POST"; } else { actionRequest.Method = "GET"; } actionRequest.ContentType = "application/x-www-form-urlencoded"; if (action.UserAgent == null) { actionRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:29.0) Gecko/20100101 Firefox/29.0"; } else { actionRequest.UserAgent = action.UserAgent; } actionRequest.Accept = "text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,video/x-mng,image/png,image/jpeg,image/gif;q=0.2,*/*;q=0.1"; actionRequest.AllowAutoRedirect = action.AllowRedirect; actionRequest.KeepAlive = false; if (!string.IsNullOrEmpty(action.ProxyAddress)) { var parts = action.ProxyAddress.Split(':'); if (parts.Length == 2) { actionRequest.Proxy = new WebProxy(parts[0], int.Parse(parts[1])); } else { actionRequest.Proxy = new WebProxy(parts[0]); } } if (timeout > 0) { actionRequest.Timeout = timeout; } actionRequest.Headers.Add("Accept-Language", "en-us,en;q=0.5"); actionRequest.ProtocolVersion = new Version(action.Version); actionRequest.CookieContainer = session.Cookies; if (action.IsPost) { if (action.Attachment == null) { WriteContent(action, actionRequest); } else { WriteFileAttachmentContent(action, actionRequest); } } document = new AgentDocument(); actionRequest.ServicePoint.Expect100Continue = false; actionResponse = (HttpWebResponse)actionRequest.GetResponse(); document.RedirectUri = action.AllowRedirect ? actionResponse.ResponseUri.ToString() : actionResponse.GetResponseHeader("Location"); responseStream = actionResponse.GetResponseStream(); Debug.Assert(responseStream != null, "responseStream != null"); var responseBuilder = new StringWriter(); var buffer = new byte[1024]; int n; do { n = responseStream.Read(buffer, 0, buffer.Length); var charBuffer = new char[buffer.Length]; for (var i = 0; i < buffer.Length; i++) { charBuffer[i] = (char)buffer[i]; } responseBuilder.Write(charBuffer, 0, n); } while (n > 0); document.ResponseString = responseBuilder.GetStringBuilder().ToString(); responseBuilder.Close(); responseBuilder.Dispose(); document.Uri = actionResponse.ResponseUri.ToString(); } finally { try { responseStream?.Close(); } catch { // ignored } try { actionResponse?.Close(); } catch { // ignored } } return(document); }
// *** Public API *** public AgentDocument PerformAction(AgentSession session, AgentAction action) { return(PerformAction(session, action, 0)); }
private static void GrabProfile(string symbol) { Console.Write("Grabbing Profile for " + symbol); string url = @"http://finance.yahoo.com/q/pr?s=" + symbol; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); AgentDocument document; try { document = AgentHandler.Instance.PerformAction(session, action); } catch { return; } string doc = document.ResponseString.ToLower(); var extractor = new ProgressiveDataExtractor(doc, ">details<"); if (extractor.NotFound) { Console.WriteLine(" - not found"); return; } Console.WriteLine(" - found"); ModelMadness madness = new ModelMadness(); string sector = extractor.ExtractString("sector"); Sector dbSector = null; Industry dbIndustry = null; if (sector != null && sector != "n/a") { sector = sector.Replace("&", "&"); string industry = extractor.ExtractString("industry").Replace("&", "&"); var obj = (from q in madness.Sectors where q.Sector1 == sector select q).ToArray(); if (obj.Length == 1) { dbSector = obj[0]; } else { dbSector = new Sector { Sector1 = sector }; madness.AddToSectors(dbSector); } var obj2 = (from q in madness.Industries where q.Sector.Sector1 == sector && q.Iindustry == industry select q).ToArray(); if (obj2.Length == 1) { dbIndustry = obj2[0]; } else { dbIndustry = new Industry { Iindustry = industry, Sector = dbSector }; madness.AddToIndustries(dbIndustry); } } StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First(); CompanyProfile profile = new CompanyProfile { Sector = dbSector, Industry = dbIndustry, num_employees = extractor.ExtractInt("full time employees"), StockName = stock, scrape_day = DateTime.Now.Date }; profile.summary = extractor.ExtractString("business summary"); profile.cgq = extractor.ExtractDecimal("corporate governance", "<b"); extractor.ExtractString("key executives"); int totalAge = 0; int numAges = 0; long totalPay = 0; int numPays = 0; int? curAge; long?curPay; do { curAge = extractor.ExtractInt("yfnc_tabledata1", "</b"); if (curAge > 111) { curAge = null; } if (curAge != null) { numAges++; totalAge += (int)curAge; curPay = extractor.ExtractLong("yfnc_tabledata1", "nowrap"); if (curPay != null) { numPays++; totalPay += (long)curPay; } } } while (curAge != null); profile.avg_executive_age = totalAge == 0 ? null : (int?)totalAge / numAges; profile.avg_executive_pay = totalPay == 0 ? null : (long?)totalPay / numPays; madness.AddToCompanyProfiles(profile); madness.SaveChanges(); }
private static void GrabStats(string symbol) { Console.Write("Grabbing Key Stats for " + symbol); string url = @"http://finance.yahoo.com/q/ks?s=" + symbol; AgentSession session = new AgentSession(); AgentAction action = new AgentAction(url, false); AgentDocument document; try { document = AgentHandler.Instance.PerformAction(session, action); } catch { return; } var doc = document.ResponseString; var extractor = new ProgressiveDataExtractor(doc, "Valuation Measures"); if (extractor.NotFound) { Console.WriteLine(" - Not found"); return; } Console.WriteLine(" - found"); ModelMadness madness = new ModelMadness(); StockName stock = (from q in madness.StockNames where q.Stock == symbol select q).First(); CompanyStat stats = new CompanyStat { market_cap = extractor.ExtractLong("Market Cap"), enterprise_value = extractor.ExtractLong("Enterprise Value"), trailing_pe = extractor.ExtractDecimal("Trailing P/E"), forward_pe = extractor.ExtractDecimal("Forward P/E"), peg_ratio = extractor.ExtractDecimal("PEG Ratio"), price_sales = extractor.ExtractDecimal("Price/Sales"), price_book = extractor.ExtractDecimal("Price/Book"), ent_value_rev = extractor.ExtractLong("Enterprise Value/Revenue"), ent_value_ebitda = extractor.ExtractDecimal("Enterprise Value/EBITDA"), fiscal_year_ends = extractor.ExtractString("Fiscal Year Ends"), most_recent_quarter = extractor.ExtractString("Most Recent Quarter"), profit_margin = extractor.ExtractDecimal("Profit Margin"), operating_margin = extractor.ExtractDecimal("Operating Margin"), return_on_assets = extractor.ExtractDecimal("Return on Assets"), return_on_equity = extractor.ExtractDecimal("Return on Equity"), revenue = extractor.ExtractLong("Revenue"), revenue_per_share = extractor.ExtractDecimal("Revenue Per Share"), qrt_rev_growth = extractor.ExtractDecimal("Qtrly Revenue Growth"), gross_profit = extractor.ExtractLong("Gross Profit"), ebitda = extractor.ExtractLong("EBITDA"), net_income_a_c = extractor.ExtractLong("Net Income Avl to Common"), diluted_eps = extractor.ExtractDecimal("Diluted EPS"), qrt_earnings_growth = extractor.ExtractDecimal("Qtrly Earnings Growth"), total_cash = extractor.ExtractLong("Total Cash"), cash_per_share = extractor.ExtractDecimal("Total Cash Per Share"), total_debt = extractor.ExtractLong("Total Debt"), total_debt_equit = extractor.ExtractDecimal("Total Debt/Equity"), current_ratio = extractor.ExtractDecimal("Current Ratio"), book_value_p_share = extractor.ExtractDecimal("Book Value Per Share"), operating_cash_flow = extractor.ExtractLong("Operating Cash Flow"), levered_free_cash_flow = extractor.ExtractLong("Levered Free Cash Flow"), StockName = stock, scrape_day = DateTime.Now.Date }; madness.AddToCompanyStats(stats); madness.SaveChanges(); }
public ResourceFromAgentDispatcher(AgentSession session, double interval) : base(session, interval) { resourceStreams = new Dictionary <Resource, FileStream>(); }