static void Main(string[] args) { string filePath = "c:\\Temp\\HTML"; List <string> webPages = new List <string>(); webPages.Add("https://www.c-sharpcorner.com/"); webPages.Add("https://www.reddit.com/"); webPages.Add("https://www.packtpub.com/"); Console.WriteLine("Scraping the following webpages..."); foreach (var webPage in webPages) { Console.WriteLine(webPage); } Scrape scrape = new Scrape(); List <SiteData> webContents = scrape.GetSites(webPages); for (var i = 0; i < webPages.Count; i++) { var fileName = Path.Combine(filePath, $"HTML_{i}.html"); if (File.Exists(fileName)) { File.Delete(fileName); } using (FileStream fileStream = new FileStream(fileName, FileMode.CreateNew)) { var line = Encoding.UTF8.GetBytes($"<!--{webContents[i].URL} -->"); fileStream.Write(line); line = Encoding.UTF8.GetBytes(webContents[i].Body); fileStream.Write(line); } } }
} // ITunesScraper(ITunesSettings, HttpClient) #endregion #region Public Methods public async Task <Scrape> GetNewScrape(string upc, Guid scanId) { string getUrl = _PartnerUri + upc; Scrape scrape = new Scrape(scanId, (int)Partners.ITunes, false, getUrl, upc, "iTunes"); try { HttpResponseMessage response = await _Client.GetAsync(getUrl); string json = response.Content.ReadAsStringAsync().Result; dynamic upcResultCount = JsonConvert.DeserializeObject <dynamic>(json); if (response.IsSuccessStatusCode) { if (upcResultCount["resultCount"] > 0) { scrape.IsLive = true; } // if else { scrape.IsLive = false; } // else } // if } // try catch (Exception e) { Console.WriteLine($"Request to client was not successful. Exception message: {e.InnerException}"); } // catch return(scrape); } // GetNewScrape(string, Guid)
static void Main(string[] args) { Scrape myScrape = new Scrape(); string value = myScrape.ScrapeWebpage("http://msdn.microsoft.com"); Console.WriteLine(value); }
private void btnTestHttpCaptureText_Click(object sender, EventArgs e) { Scrape s = new Scrape(); this.btnTestHttpCaptureText.Enabled = false; this.toolStripStatusLabel1.Text = "Loading URL ..."; this.groupBox1.Text = string.Empty; this.txtCaptureRURL.Text = string.Empty; this.txtSourceTextREC.Text = string.Empty; this.txtURL.Enabled = false; this.txtCaptureRURL.Enabled = false; this.Refresh(); string capture = string.Empty; try { WebScrapeResponse r = s.HttpCapture(this.txtURL.Text, string.Empty, string.Empty, null); this.groupBox1.Text = string.Format("Done: ({0}) {1}", r.StatusCode, r.StatusText); this.toolStripStatusLabel1.Text = "Loading complete"; this.txtCaptureRURL.Text = (r.ErrorMessage == string.Empty ? r.Content : r.ErrorMessage); this.txtSourceTextREC.Text = this.txtCaptureRURL.Text; this.tabControl1.TabPages["tabRegExChunk"].Focus(); } catch (Exception ex) { this.groupBox1.Text = "Failed to retrieve URL"; this.toolStripStatusLabel1.Text = "Loading failed"; this.txtCaptureRURL.Text = DetailedException.WithUserContent(ref ex); } this.btnTestHttpCaptureText.Enabled = true; this.txtURL.Enabled = true; this.txtCaptureRURL.Enabled = true; }
public void GetAch_Pick_All() { string pickAll = @"<tr class=''><td><a href='/w/Pick_All' title='Pick All'>Pick All</a></td><td>Yes</td><td>Use a lockpick to open ALL of New Varrock's treasure chests.</td><td>Exploration</td><td>New Varrock</td><td>15<p><br></p></td></tr>"; HtmlNode achNode = HtmlNode.CreateNode(pickAll); AchievementObject achievementActual = Scrape.GetAchievementRow(achNode); AchievementObject achievementExpected = new AchievementObject { AName = "Pick All", AMembers = "Yes", ADescription = "Use a lockpick to open ALL of New Varrock's treasure chests.", ACategories = new List <string> { "Exploration" }, ASubcategories = new List <string> { "New Varrock" }, ARunescore = 15, ALink = "/w/Pick_All", AQuestReqs = new List <string> { "Dimension of Disaster: Shield of Arrav (partial)", "Dimension of Disaster: Demon Slayer (partial)" }, ASkillReqs = new List <string> { "70 Thieving" } }; AssertingAchObj(achievementExpected, achievementActual); }
public void Check_3Skill_Requirements() { AchievementObject aBridgeNotFar = new AchievementObject { AName = "A Bridge Not Far", ADescription = "Cross the River Lum using a crossbow.", ARunescore = 10, AMembers = "Yes", ALink = "/w/A_Bridge_Not_Far", ACategories = new List <string> { "Exploration" }, ASubcategories = new List <string> { "Desert" }, AQuestReqs = new List <string>(), ASkillReqs = new List <string>() }; aBridgeNotFar = Scrape.GetRequirements(aBridgeNotFar); Assert.Contains("8 Agility", aBridgeNotFar.ASkillReqs); Assert.Contains("19 Strength", aBridgeNotFar.ASkillReqs); Assert.Contains("37 Ranged", aBridgeNotFar.ASkillReqs); }
public void GetAch_Kill_Kril_Vol_2_VI() { string killKrilVol2VI = @"<tr class=''><td><a href='/w/Kill_K%27ril_Vol_2_VI' title='Kill K''ril Vol 2 VI'>Kill K'ril Vol 2 VI</a></td><td>Yes</td><td>Defeat K'ril Tsutsaroth in hard mode. (X/100)</td><td>Combat</td><td>Boss Kills</td><td>5<p><br></p></td></tr>"; HtmlNode achNode = HtmlNode.CreateNode(killKrilVol2VI); AchievementObject achievementActual = Scrape.GetAchievementRow(achNode); AchievementObject achievementExpected = new AchievementObject { AName = "Kill K'ril Vol 2 VI", AMembers = "Yes", ADescription = "Defeat K'ril Tsutsaroth in hard mode. (X/100)", ACategories = new List <string> { "Combat" }, ASubcategories = new List <string> { "Boss Kills" }, ARunescore = 5, AQuestReqs = new List <string> { "Troll Stronghold (partial)" }, ASkillReqs = new List <string> { "60 Strength or 60 Agility", "70 Constitution" }, ALink = "/w/Kill_K%27ril_Vol_2_VI" }; AssertingAchObj(achievementExpected, achievementActual); }
public void GetAch_No_Smoke_Without_Pyre() { string noSmokeWithoutPyre = @"<tr class=''><td><a href='/w/No_Smoke_Without_Pyre' title='No Smoke Without Pyre'>No Smoke Without Pyre</a></td><td>Yes</td><td>Make a pyre ship from magic logs.</td><td>Exploration</td><td>Fremennik</td><td>25<p><br></p></td></tr>"; HtmlNode achNode = HtmlNode.CreateNode(noSmokeWithoutPyre); AchievementObject achievementActual = Scrape.GetAchievementRow(achNode); AchievementObject achievementExpected = new AchievementObject { AName = "No Smoke Without Pyre", AMembers = "Yes", ADescription = "Make a pyre ship from magic logs.", ACategories = new List <string> { "Exploration" }, ASubcategories = new List <string> { "Fremennik" }, ARunescore = 25, ALink = "/w/No_Smoke_Without_Pyre", AQuestReqs = new List <string> { "Completed Firemaking section of Barbarian Training" }, ASkillReqs = new List <string> { "85 Crafting", "85 Firemaking" } }; AssertingAchObj(achievementExpected, achievementActual); }
public static void Demo() { Scrape myScrape = new Scrape(); string value = myScrape.ScrapeWebpage("https://en.wikipedia.org/wiki/John_Cleese"); Console.WriteLine(value); }
public void Execute_ScrapeHasNotFinished_ReturnsScrapeList() { var startedAt = DateTime.UtcNow; var source = new Source() { Id = 1, Name = "FreeBmdBirths", FriendlyName = "FreeBMD Births" }; var scrape = new Scrape() { Id = 1, Source = source, StartedAt = startedAt }; var scrapeList = new List <Scrape>() { scrape }; var mockRepository = new Mock <IScrapeRepository>(); mockRepository.Setup(q => q.GetAll()).Returns(scrapeList.AsQueryable()); var listQuery = new GetScrapeListQuery(mockRepository.Object); var result = listQuery.Execute(); Assert.NotNull(result); Assert.Equal(1, result.Count); Assert.Equal(1, result[0].Id); Assert.Equal("FreeBMD Births", result[0].SourceName); Assert.Equal(startedAt, result[0].StartedAt); Assert.False(result[0].TimeTaken.HasValue); Assert.Equal(0, result[0].NumberOfRecordsScraped); }
static void Main(string[] args) { Scrape myScrape = new Scrape(); String content = myScrape.ScrapeWepPage("http://wsb.pl", @"C:\Temp\web.txt"); try { //string content = File.ReadAllText(@"C:\Temp\web.txt"); Console.WriteLine(content); } catch (FileNotFoundException ex) { Console.WriteLine("Coś poszło nie tak ..."); Console.WriteLine($"Sprawdź czy istnieje plik {ex.FileName}."); } catch (DirectoryNotFoundException) { Console.WriteLine("Coś poszło nie tak ..."); Console.WriteLine("Sprawdź czy istnieje folder"); } catch (Exception ex) { Console.WriteLine("Coś poszło nie tak ..."); Console.WriteLine($"Wystąpił błąd {ex.Message}"); } finally { Console.WriteLine("Kończę działanie !"); } Console.Read(); }
public void Check_1Skill_1Quest_Requirements() { AchievementObject aCleanSlate = new AchievementObject { AName = "A Clean Slate", ADescription = "Cleanse the Corrupted Seren Stone with at least one crystal.", ARunescore = 15, AMembers = "Yes", ALink = "/w/A_Clean_Slate", ACategories = new List <string> { "Exploration" }, ASubcategories = new List <string> { "Tirannwn" }, AQuestReqs = new List <string>(), ASkillReqs = new List <string>() }; aCleanSlate = Scrape.GetRequirements(aCleanSlate); Assert.Contains("Plague's End", aCleanSlate.AQuestReqs); Assert.Contains("75 Prayer", aCleanSlate.ASkillReqs); }
public void GetAll_MultipleRecordsToReturn_ReturnsAllRecords() { var source = new Source() { Id = 1, Name = "name", FriendlyName = "FriendlyName" }; var scrape1 = new Scrape() { Id = 1, SourceId = 1, Source = source, Surname = "surname", StartedAt = DateTime.UtcNow }; var scrape2 = new Scrape() { Id = 2, SourceId = 1, Source = source, Surname = "surname2", StartedAt = DateTime.UtcNow }; var data = new List <Scrape> { scrape1, scrape2 }.AsQueryable(); var mockDbSet = GetMockDbSet(data); var mockContext = new Mock <IDatabaseContext>(); mockContext.Setup(c => c.Scrapes).Returns(mockDbSet.Object); var repository = new ScrapeRepository(mockContext.Object); var result = repository.GetAll(); Assert.NotNull(result); Assert.Equal(2, result.Count <Scrape>()); }
private void threadFunc(object arg) { try { List <string> data = arg as List <string>; int ThreadIndex = 0; int.TryParse(data[0], out ThreadIndex); List <string> param = new List <string>(); for (int i = 1; i < data.Count; i++) { param.Add(data[i]); } task = new Scrape(ThreadIndex, onWriteStatus, onSetProgress, param, delay, useProxy, proxylist); bool result = false; if (rbtnname.Checked == true) { result = task.doWorkByName(savepath); } else { result = task.doWorkBySeller(savepath); } } catch (Exception e) { string error = e.Message; } }
public void GetAch_Great_Responsibility() { string greatResponsibility = @"<tr class=''><td><a href='/w/Great_Responsibility' title='Great Responsibility'>Great Responsibility</a></td><td>Yes</td><td>Have over 490,000 charge<sup class='noprint fact'>[<span class='fact-text' title='The preceding quote has been reproduced verbatim and is not a transcription error.'>sic</span>]</sup> in a jumbo generator in the Invention Guild.</td><td>Skills</td><td>Invention</td><td>20<p><br></p></td></tr>"; HtmlNode achNode = HtmlNode.CreateNode(greatResponsibility); AchievementObject achievementActual = Scrape.GetAchievementRow(achNode); AchievementObject achievementExpected = new AchievementObject { AName = "Great Responsibility", AMembers = "Yes", ADescription = "Have over 490,000 charge[sic] in a jumbo generator in the Invention Guild.", ACategories = new List <string> { "Skills" }, ASubcategories = new List <string> { "Invention" }, ARunescore = 20, ALink = "/w/Great_Responsibility", AQuestReqs = new List <string> { "None" }, ASkillReqs = new List <string> { "102 Invention" } }; AssertingAchObj(achievementExpected, achievementActual); }
public void GetAch_A_Mini_Shipment() { string aMiniShipmentRow = @"<tr class=''><td><a href='/w/A_Mini_Shipment' title='A Mini Shipment'>A Mini Shipment</a></td><td>Yes</td><td>Loot the trunk in the Temple of Aminishi.</td><td>Combat<br>Completionist</td><td>Elite Dungeons<br>Master Quest Cape</td><td>5<p><br></p></td></tr>"; HtmlNode achNode = HtmlNode.CreateNode(aMiniShipmentRow); AchievementObject achievementActual = Scrape.GetAchievementRow(achNode); AchievementObject achievementExpected = new AchievementObject { AName = "A Mini Shipment", AMembers = "Yes", ADescription = "Loot the trunk in the Temple of Aminishi.", ACategories = new List <string> { "Combat", "Completionist" }, ASubcategories = new List <string> { "Elite Dungeons", "Master Quest Cape" }, ARunescore = 5, AQuestReqs = new List <string> { "Curse of the Black Stone (partial)" }, ASkillReqs = new List <string> { "None" }, ALink = "/w/A_Mini_Shipment" }; AssertingAchObj(achievementExpected, achievementActual); }
protected static async Task ScanListings() { string appid = ConfigurationManager.AppSettings["appID"]; int count = db.PostedListings.Count(); int i = 0; //foreach (PostedListing p in db.PostedListings.Where(r => r.ListedItemID == "223072961963")) foreach (PostedListing p in db.PostedListings) { try { string sourcrUrl = p.SourceUrl; var result = await Scrape.GetDetail(sourcrUrl); Console.WriteLine("processing " + (++i) + " of " + count.ToString()); if (!string.IsNullOrEmpty(result.availability)) { if (result.availability.Contains("Out of stock")) { var myListing = await scrapeAPI.ebayAPIs.GetSingleItem(p.ListedItemID, appid); if (myListing.Qty > 0) { Console.WriteLine("OUT OF STOCK " + p.Title); string reviseResult = scrapeAPI.ebayAPIs.ReviseQty(p.ListedItemID, qty: 0); dsutil.DSUtil.WriteFile(Log_File, p.ListedItemID + " " + p.Title); dsutil.DSUtil.WriteFile(Log_File, p.SourceUrl); dsutil.DSUtil.WriteFile(Log_File, reviseResult); string ret = await dsutil.DSUtil.SendMailProd("*****@*****.**", "OUT OF STO " + p.Title, "revise listing", "localhost"); if (!string.IsNullOrEmpty(ret)) { dsutil.DSUtil.WriteFile(Log_File, "prod email failed: " + ret); ret = dsutil.DSUtil.SendMailDev("*****@*****.**", "OUT OF STO " + p.Title, "revise listing"); if (!string.IsNullOrEmpty(ret)) { dsutil.DSUtil.WriteFile(Log_File, "dev email failed: " + ret); } } } else { dsutil.DSUtil.WriteFile(Log_File, p.ListedItemID + " " + p.Title); dsutil.DSUtil.WriteFile(Log_File, p.SourceUrl); dsutil.DSUtil.WriteFile(Log_File, "Listing quantity already set to 0 - no action taken."); } dsutil.DSUtil.WriteFile(Log_File, string.Empty); } } } catch (Exception exc) { dsutil.DSUtil.WriteFile(Log_File, "ERROR: " + p.ListedItemID + " " + exc.Message); } } dsutil.DSUtil.WriteFile(Log_File, string.Format("Processed {0} listings.", count)); }
static void Main(string[] args) { Scrape myScrape = new Scrape(); string outputString = myScrape.ScrapeWebPage("https://www.atom.io"); Console.WriteLine(outputString); Console.ReadLine(); }
public IList<Payment> GetPayments() { Scrape scrape = new Scrape(); scrape.user = user; scrape.pass = pass; scrape.Run(); return scrape.Payments; }
static void Main(string[] args) { Scrape myScrape = new Scrape(); string value = myScrape.ScrapeWebPage("https://duckduckgo.com/"); Console.WriteLine(value); Console.ReadLine(); }
static void Main(string[] args) { Console.WriteLine("Hello World!"); Scrape scrape = new Scrape(); string value = scrape.ScrapeWebpage("http://msdn.microsoft.com"); Console.WriteLine(value); }
static void Main(string[] args) { Scrape myScrape = new Scrape(); //references from MyCodeLibrary string value = myScrape.ScrapeWebPage("https://ois2.ttu.ee/uusois/uus_ois2.tud_leht"); Console.WriteLine(value); Console.ReadLine(); }
static void Main(string[] args) { Scrape myScrape = new Scrape(); string value = myScrape.ScrapeWebPage("http://google.com"); Console.WriteLine(value); Console.ReadLine(); }
//to access parts of the .net framework class library. //1. right click references in the solution explorer and add reference. //2. Nuget //----- //Make our own class library to reference, here we scraped a web page and wrote it to a file. static void Main(string[] args) { Scrape myScrape = new Scrape(); string result = myScrape.ScrapeWebpage("http://www.msdn.microsoft.com", @"C:\Users\asus\Desktop\Programming\CSharp\learncs\ReferenceToAssembly\WriteText.txt"); Console.WriteLine(result); Console.ReadLine(); }
static void Main(string[] args) { Scrape myScrape = new Scrape(); string reply = myScrape.ScrapeWebPage("https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/file-system/how-to-write-to-a-text-file"); Console.WriteLine(reply); }
private void btnRegExChunkTest_Click(object sender, EventArgs e) { Scrape s = new Scrape(); this.btnRegExChunkTest.Enabled = false; this.txtRegExStart.Enabled = false; this.chkStartIgnoreCase.Enabled = false; this.nudStartIteration.Enabled = false; this.txtRegExEnd.Enabled = false; this.chkEndIgnoreCase.Enabled = false; this.nudEndIteration.Enabled = false; this.lstMatchResultsREC.Enabled = false; this.txtMatchTextREC.Text = string.Empty; this.txtMatchTextREC.Enabled = false; this.lstMatchResultsREC.Items.Clear(); this.lstMatchResultsREC.Enabled = false; this.Refresh(); try { StringCollection results = s.RegExChunk( this.txtCaptureRURL.Text, this.txtRegExStart.Text, this.chkStartIgnoreCase.Checked, int.Parse(this.nudStartIteration.Value.ToString()), this.txtRegExEnd.Text, this.chkEndIgnoreCase.Checked, int.Parse(this.nudEndIteration.Value.ToString()), int.Parse(this.nudMaximumMatches.Value.ToString()), int.Parse(this.nudMaxChunkLength.Value.ToString())); this.lstMatchResultsREC.Items.Clear(); foreach (string str in results) { this.lstMatchResultsREC.Items.Add(str); } if (this.lstMatchResultsREC.Items.Count > 0) { this.lstMatchResultsREC.SelectedIndex = 0; } } catch (Exception ex) { this.txtMatchTextREC.Text = "FAIL"; this.lstMatchResultsREC.Items.Clear(); this.lstMatchResultsREC.Items.Add(DetailedException.WithEnterpriseContent(ref ex)); } this.btnRegExChunkTest.Enabled = true; this.txtRegExStart.Enabled = true; this.chkStartIgnoreCase.Enabled = true; this.nudStartIteration.Enabled = true; this.txtRegExEnd.Enabled = true; this.chkEndIgnoreCase.Enabled = true; this.nudEndIteration.Enabled = true; this.lstMatchResultsREC.Enabled = true; this.txtMatchTextREC.Enabled = true; this.lstMatchResultsREC.Enabled = true; }
static void Main(string[] args) { Scrape myScrape = new Scrape(); string value = myScrape.ScrapeWebepage("http://estakada.lt"); Console.WriteLine(value); Console.ReadLine(); }
static void Main(string[] args) { Scrape myScrape = new Scrape(); //string value = myScrape.ScrapeWebpage("http://msdn.microsoft.com"); string value = myScrape.ScrapeWebpage("http://msdn.microsoft.com", @"c:\users\public\testfolder\html2.txt"); Console.WriteLine(value); Console.ReadLine(); }
static void Main(string[] args) { // because we added a reference to this file, we can now use MyCodeLibrary // now we can use classes/methods from this namespace Scrape myScrape = new Scrape(); string value = myScrape.ScrapeWebpage("https://docs.microsoft.com/en-us/"); Console.WriteLine(value); }
public IList <Payment> GetPayments() { Scrape scrape = new Scrape(); scrape.user = user; scrape.pass = pass; scrape.Run(); return(scrape.Payments); }
static void Main(string[] args) { // be sure to set ScrapClient as StartUp Project to run Scrape myScrape = new Scrape(); string value = myScrape.ScrapeWebpage("http://msdn.microsoft.com"); Console.WriteLine(value); Console.ReadLine(); }