static public void AccessEnglishWeb() { InstrumentContext ctx = new InstrumentContext(); String queryPath = @"http://www.londonstockexchange.com/exchange/prices-and-markets/stocks/summary/company-summary/"; List <Target> londonStocks = AccessFiles.getList(StockName.London); foreach (Target item in londonStocks) { string xquery = ".//div[@class='commonTable table-responsive']/table//td"; string path = queryPath + item.urlIdentifier + ".html"; Console.WriteLine("Getting {0}", path); var myArray = GetTable(path, xquery); if (myArray == null) { continue; } Instrument myInstrument = new Instrument(); { string changes = myArray[3].InnerText.Replace("\r", string.Empty).Replace("\n", string.Empty).Replace(")", string.Empty).Replace("(", string.Empty); String[] m = Regex.Split(changes, "%"); myInstrument.PctChange(m[0]); myInstrument.Change(m[1]); myInstrument.LastPrice(myArray[1].InnerText); myInstrument.DayHigh(myArray[5].InnerText); myInstrument.DayLow(myArray[7].InnerText); myInstrument.Volume(myArray[9].InnerText); string isolatePrevClose = myArray[11].InnerText.Replace("\r", string.Empty).Replace("\n", string.Empty); m = Regex.Split(isolatePrevClose, "on"); myInstrument.PrevClose(m[0]); myInstrument.Bid(myArray[13].InnerText); myInstrument.Ask(myArray[15].InnerText); myInstrument.ticker = item.shortIdentifier; myInstrument.timestamp = System.DateTime.Now; myInstrument.stockExchangeName = StockName.London; myInstrument.url = path; } // accessing the mongoDB // In order to not get banned we delay individul request from the london website //Getting the data from Xignite GlobalQuote tempQuery = AccessXIgnite.getXigniteData("XLON", item.shortIdentifier); var serializedParent = JsonConvert.SerializeObject(tempQuery); GlobalQuoteExtend query = JsonConvert.DeserializeObject <GlobalQuoteExtend>(serializedParent); query.DBtimestamp = myInstrument.timestamp; ctx.IgniteInstruments.InsertOne(query); var differences = myInstrument.compareInstrument(query); ctx.Instruments.InsertOne(myInstrument); if (differences.different == true) { ctx.mismatchInstruments.InsertOne(differences); Console.WriteLine("Found a mismatch! " + differences.ticker + " " + differences.comments); } System.Threading.Thread.Sleep(1000); } }
static void AccessHongKong() { //some psuedo random sample of stocks string[] stockList = new string[] { "8091", "136", "771", "299", "8358", "1656", "3639", "21", "8316", "1663", "2927", "1637", "646", "8327", "1231", "8195", "8256", "8416", "362", "1462" }; foreach (string stock in stockList) { string Url = @"https://www.hkex.com.hk/eng/invest/company/quote_page_e.asp?WidCoID=" + stock + @"&WidCoAbbName=&Month=1&langcode=e"; using (WebClient client = new WebClient()) { client.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36"; client.UseDefaultCredentials = true; HtmlDocument htmlDoc = new HtmlDocument(); htmlDoc.LoadHtml(client.DownloadString(Url)); Instrument myInstrument = new Instrument(); { myInstrument.stockExchangeName = StockName.HongKong; var a = htmlDoc.DocumentNode.SelectNodes("//td[@bgcolor='#CCCCCC']"); HtmlNode[] map = new HtmlNode[a.Count]; a.CopyTo(map, 0); myInstrument.pctChange = 0; if (map[1].InnerHtml.Contains("down")) { myInstrument.Change("-" + map[1].InnerText); myInstrument.PctChange("-" + map[2].InnerText); } else { myInstrument.Change(map[1].InnerText); myInstrument.PctChange(map[2].InnerText); } myInstrument.LastPrice(map[0].InnerText); myInstrument.Bid(map[3].InnerText); myInstrument.Ask(map[4].InnerText); myInstrument.DayHigh(map[6].InnerText); myInstrument.DayLow(map[7].InnerText); myInstrument.ticker = stock; myInstrument.PrevClose(RemoveExtraText(map[8].InnerText)); myInstrument.Volume(map[11].InnerText); myInstrument.timestamp = System.DateTime.Now; } GlobalQuote mytempQuery = AccessXIgnite.getXigniteData("XHGK", myInstrument.ticker); var serializedParent = JsonConvert.SerializeObject(mytempQuery); GlobalQuoteExtend query = JsonConvert.DeserializeObject <GlobalQuoteExtend>(serializedParent); var differences = myInstrument.compareInstrument(query); } } }
static void AccessNasdaq() { List <Target> stockList = AccessFiles.getList(StockName.Nasdaq); foreach (Target stock in stockList) { string xquery = ".//div[@class='genTable thin']//td"; HtmlNode[] myArray = GetTable(stock.urlIdentifier, xquery); String[] m = Regex.Split(myArray[17].InnerText, ";"); string[] innerText = new string[myArray.Length]; for (int i = 0; i < myArray.Length; i++) { innerText[i] = myArray[i].InnerText.Replace("&", string.Empty).Replace("#", string.Empty).Replace("%", string.Empty).Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("nbsp", string.Empty).Replace("$", string.Empty).Replace(";", string.Empty); } for (int i = 0; i < m.Length; i++) { m[i].Replace("&", string.Empty).Replace("#", string.Empty).Replace("%", string.Empty).Replace("\r", string.Empty).Replace("\n", string.Empty).Replace("nbsp", string.Empty).Replace("$", string.Empty).Replace(";", string.Empty); } Instrument myInstrument = new Instrument(); { // This is a combination of change and pct change and we thus separate them if (m[1] == "9660") { myInstrument.PctChange("-" + m[3]); myInstrument.Change("-" + m[0]); } // New: Nasdaq started marking unchanged in a funny way else if (m[0].Contains("unch")) { myInstrument.PctChange(0); myInstrument.Change(0); } else { myInstrument.PctChange(m[3]); myInstrument.Change(m[0]); } myInstrument.LastPrice(innerText[1]); myInstrument.DayHigh(innerText[5]); myInstrument.DayLow(innerText[21]); myInstrument.Volume(innerText[3]); myInstrument.PrevClose(innerText[19]); myInstrument.Bid(innerText[7]); myInstrument.Ask(innerText[23]); myInstrument.ticker = stock.shortIdentifier; myInstrument.timestamp = System.DateTime.Now; myInstrument.stockExchangeName = StockName.Nasdaq; myInstrument.url = stock.urlIdentifier; } InstrumentContext ctx = new InstrumentContext(); GlobalQuote mytempQuery = AccessXIgnite.getXigniteData("XNAS", myInstrument.ticker); var serializedParent = JsonConvert.SerializeObject(mytempQuery); GlobalQuoteExtend query = JsonConvert.DeserializeObject <GlobalQuoteExtend>(serializedParent); query.DBtimestamp = myInstrument.timestamp; ctx.IgniteInstruments.InsertOne(query); var differences = myInstrument.compareInstrument(query); ctx.Instruments.InsertOne(myInstrument); if (differences.different == true) { ctx.mismatchInstruments.InsertOne(differences); } ctx.Instruments.InsertOne(myInstrument); } }
public Instrument compareInstrument(GlobalQuoteExtend input) { different = false; Instrument val = new Instrument(); if (roundLondon(input.Last) != this.lastPrice && (lastPrice != 0)) { double check = roundLondon(input.Last); double check2 = this.lastPrice; different = true; val.lastPrice = this.lastPrice; val.comments += String.Format("Price x={0}, c={1} ", roundLondon(input.Last), this.lastPrice); } if (roundLondon(input.High) != dayHigh && (dayHigh != 0)) { different = true; val.dayHigh = this.dayHigh; val.comments += String.Format("dayHigh x={0}, c={1} ", roundLondon(input.High), this.dayHigh); } if (roundLondon(input.Low) != dayLow && (dayLow != 0)) { different = true; val.dayLow = this.dayLow; val.comments += String.Format("dayLow x={0}, c={1} ", roundLondon(input.Low), this.dayLow); } if (input.Volume != volume && (volume != 0)) { different = true; val.volume = this.volume; val.comments += String.Format("volume x={0}, c={1} ", input.Volume, this.volume); } if (roundLondon(input.Ask) != ask && (ask != 0)) { different = true; val.ask = this.ask; val.comments += String.Format("ask x={0}, c={1} ", roundLondon(input.Ask), this.ask); } if (roundLondon(input.ChangeFromPreviousClose) != change && (change != 0)) { different = true; val.change = this.change; val.comments += String.Format("change x={0}, c={1} ", roundLondon(input.ChangeFromPreviousClose), this.change); } if (roundLondon(input.PreviousClose) != prevClose && (prevClose != 0)) { different = true; val.prevClose = this.prevClose; val.comments += String.Format("prevClose x={0}, c={1} ", roundLondon(input.PreviousClose), this.prevClose); } Console.WriteLine(this.ticker + " at time (UTC) " + DateTime.Now.ToUniversalTime().ToLongTimeString()); if (this.stockExchangeName == StockName.London) { Console.Write("\n Last Trade: Time x=" + input.Time + " Price of last trade in c= " + this.lastTradePrice + " "); } if (different == true) { val.different = true; val.timestamp = this.timestamp; val.stockExchangeName = this.stockExchangeName; val.ticker = this.ticker; Console.WriteLine(val.comments); } else { Console.WriteLine("Is the same"); } double averageOnBidAndAsk = (this.ask + this.bid) / 2; if (this.stockExchangeName == StockName.London) { Console.WriteLine("Average of offer and bid is " + averageOnBidAndAsk + "\n"); } return(val); }