static void Main(string[] args)
{
string filePath = "c:\\Temp\\HTML";
List <string> webPages = new List <string>();
webPages.Add("https://www.c-sharpcorner.com/");
webPages.Add("https://www.reddit.com/");
webPages.Add("https://www.packtpub.com/");
Console.WriteLine("Scraping the following webpages...");
foreach (var webPage in webPages)
{
Console.WriteLine(webPage);
}
Scrape scrape = new Scrape();
List <SiteData> webContents = scrape.GetSites(webPages);
for (var i = 0; i < webPages.Count; i++)
{
var fileName = Path.Combine(filePath, $"HTML_{i}.html");
if (File.Exists(fileName))
{
File.Delete(fileName);
}
using (FileStream fileStream = new FileStream(fileName, FileMode.CreateNew))
{
var line = Encoding.UTF8.GetBytes($"<!--{webContents[i].URL} -->");
fileStream.Write(line);
line = Encoding.UTF8.GetBytes(webContents[i].Body);
fileStream.Write(line);
}
}
}
} // ITunesScraper(ITunesSettings, HttpClient)
#endregion
#region Public Methods
public async Task <Scrape> GetNewScrape(string upc, Guid scanId)
{
string getUrl = _PartnerUri + upc;
Scrape scrape = new Scrape(scanId, (int)Partners.ITunes, false, getUrl, upc, "iTunes");
try
{
HttpResponseMessage response = await _Client.GetAsync(getUrl);
string json = response.Content.ReadAsStringAsync().Result;
dynamic upcResultCount = JsonConvert.DeserializeObject <dynamic>(json);
if (response.IsSuccessStatusCode)
{
if (upcResultCount["resultCount"] > 0)
{
scrape.IsLive = true;
} // if
else
{
scrape.IsLive = false;
} // else
} // if
} // try
catch (Exception e)
{
Console.WriteLine($"Request to client was not successful. Exception message: {e.InnerException}");
} // catch
return(scrape);
} // GetNewScrape(string, Guid)
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
string value = myScrape.ScrapeWebpage("http://msdn.microsoft.com");
Console.WriteLine(value);
}
private void btnTestHttpCaptureText_Click(object sender, EventArgs e)
{
Scrape s = new Scrape();
this.btnTestHttpCaptureText.Enabled = false;
this.toolStripStatusLabel1.Text = "Loading URL ...";
this.groupBox1.Text = string.Empty;
this.txtCaptureRURL.Text = string.Empty;
this.txtSourceTextREC.Text = string.Empty;
this.txtURL.Enabled = false;
this.txtCaptureRURL.Enabled = false;
this.Refresh();
string capture = string.Empty;
try
{
WebScrapeResponse r = s.HttpCapture(this.txtURL.Text, string.Empty, string.Empty, null);
this.groupBox1.Text = string.Format("Done: ({0}) {1}", r.StatusCode, r.StatusText);
this.toolStripStatusLabel1.Text = "Loading complete";
this.txtCaptureRURL.Text = (r.ErrorMessage == string.Empty ? r.Content : r.ErrorMessage);
this.txtSourceTextREC.Text = this.txtCaptureRURL.Text;
this.tabControl1.TabPages["tabRegExChunk"].Focus();
}
catch (Exception ex)
{
this.groupBox1.Text = "Failed to retrieve URL";
this.toolStripStatusLabel1.Text = "Loading failed";
this.txtCaptureRURL.Text = DetailedException.WithUserContent(ref ex);
}
this.btnTestHttpCaptureText.Enabled = true;
this.txtURL.Enabled = true;
this.txtCaptureRURL.Enabled = true;
}
public void GetAch_Pick_All()
{
string pickAll = @"<tr class=''><td><a href='/w/Pick_All' title='Pick All'>Pick All</a></td><td>Yes</td><td>Use a lockpick to open ALL of New Varrock's treasure chests.</td><td>Exploration</td><td>New Varrock</td><td>15<p><br></p></td></tr>";
HtmlNode achNode = HtmlNode.CreateNode(pickAll);
AchievementObject achievementActual = Scrape.GetAchievementRow(achNode);
AchievementObject achievementExpected = new AchievementObject
{
AName = "Pick All",
AMembers = "Yes",
ADescription = "Use a lockpick to open ALL of New Varrock's treasure chests.",
ACategories = new List <string>
{
"Exploration"
},
ASubcategories = new List <string>
{
"New Varrock"
},
ARunescore = 15,
ALink = "/w/Pick_All",
AQuestReqs = new List <string>
{
"Dimension of Disaster: Shield of Arrav (partial)",
"Dimension of Disaster: Demon Slayer (partial)"
},
ASkillReqs = new List <string>
{
"70 Thieving"
}
};
AssertingAchObj(achievementExpected, achievementActual);
}
public void Check_3Skill_Requirements()
{
AchievementObject aBridgeNotFar = new AchievementObject
{
AName = "A Bridge Not Far",
ADescription = "Cross the River Lum using a crossbow.",
ARunescore = 10,
AMembers = "Yes",
ALink = "/w/A_Bridge_Not_Far",
ACategories = new List <string>
{
"Exploration"
},
ASubcategories = new List <string>
{
"Desert"
},
AQuestReqs = new List <string>(),
ASkillReqs = new List <string>()
};
aBridgeNotFar = Scrape.GetRequirements(aBridgeNotFar);
Assert.Contains("8 Agility", aBridgeNotFar.ASkillReqs);
Assert.Contains("19 Strength", aBridgeNotFar.ASkillReqs);
Assert.Contains("37 Ranged", aBridgeNotFar.ASkillReqs);
}
public void GetAch_Kill_Kril_Vol_2_VI()
{
string killKrilVol2VI = @"<tr class=''><td><a href='/w/Kill_K%27ril_Vol_2_VI' title='Kill K''ril Vol 2 VI'>Kill K'ril Vol 2 VI</a></td><td>Yes</td><td>Defeat K'ril Tsutsaroth in hard mode. (X/100)</td><td>Combat</td><td>Boss Kills</td><td>5<p><br></p></td></tr>";
HtmlNode achNode = HtmlNode.CreateNode(killKrilVol2VI);
AchievementObject achievementActual = Scrape.GetAchievementRow(achNode);
AchievementObject achievementExpected = new AchievementObject
{
AName = "Kill K'ril Vol 2 VI",
AMembers = "Yes",
ADescription = "Defeat K'ril Tsutsaroth in hard mode. (X/100)",
ACategories = new List <string>
{
"Combat"
},
ASubcategories = new List <string>
{
"Boss Kills"
},
ARunescore = 5,
AQuestReqs = new List <string>
{
"Troll Stronghold (partial)"
},
ASkillReqs = new List <string>
{
"60 Strength or 60 Agility",
"70 Constitution"
},
ALink = "/w/Kill_K%27ril_Vol_2_VI"
};
AssertingAchObj(achievementExpected, achievementActual);
}
public void GetAch_No_Smoke_Without_Pyre()
{
string noSmokeWithoutPyre = @"<tr class=''><td><a href='/w/No_Smoke_Without_Pyre' title='No Smoke Without Pyre'>No Smoke Without Pyre</a></td><td>Yes</td><td>Make a pyre ship from magic logs.</td><td>Exploration</td><td>Fremennik</td><td>25<p><br></p></td></tr>";
HtmlNode achNode = HtmlNode.CreateNode(noSmokeWithoutPyre);
AchievementObject achievementActual = Scrape.GetAchievementRow(achNode);
AchievementObject achievementExpected = new AchievementObject
{
AName = "No Smoke Without Pyre",
AMembers = "Yes",
ADescription = "Make a pyre ship from magic logs.",
ACategories = new List <string> {
"Exploration"
},
ASubcategories = new List <string>
{
"Fremennik"
},
ARunescore = 25,
ALink = "/w/No_Smoke_Without_Pyre",
AQuestReqs = new List <string>
{
"Completed Firemaking section of Barbarian Training"
},
ASkillReqs = new List <string>
{
"85 Crafting",
"85 Firemaking"
}
};
AssertingAchObj(achievementExpected, achievementActual);
}
public static void Demo()
{
Scrape myScrape = new Scrape();
string value = myScrape.ScrapeWebpage("https://en.wikipedia.org/wiki/John_Cleese");
Console.WriteLine(value);
}
public void Execute_ScrapeHasNotFinished_ReturnsScrapeList()
{
var startedAt = DateTime.UtcNow;
var source = new Source()
{
Id = 1, Name = "FreeBmdBirths", FriendlyName = "FreeBMD Births"
};
var scrape = new Scrape()
{
Id = 1, Source = source, StartedAt = startedAt
};
var scrapeList = new List <Scrape>()
{
scrape
};
var mockRepository = new Mock <IScrapeRepository>();
mockRepository.Setup(q => q.GetAll()).Returns(scrapeList.AsQueryable());
var listQuery = new GetScrapeListQuery(mockRepository.Object);
var result = listQuery.Execute();
Assert.NotNull(result);
Assert.Equal(1, result.Count);
Assert.Equal(1, result[0].Id);
Assert.Equal("FreeBMD Births", result[0].SourceName);
Assert.Equal(startedAt, result[0].StartedAt);
Assert.False(result[0].TimeTaken.HasValue);
Assert.Equal(0, result[0].NumberOfRecordsScraped);
}
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
String content = myScrape.ScrapeWepPage("http://wsb.pl", @"C:\Temp\web.txt");
try
{
//string content = File.ReadAllText(@"C:\Temp\web.txt");
Console.WriteLine(content);
}
catch (FileNotFoundException ex)
{
Console.WriteLine("Coś poszło nie tak ...");
Console.WriteLine($"Sprawdź czy istnieje plik {ex.FileName}.");
}
catch (DirectoryNotFoundException)
{
Console.WriteLine("Coś poszło nie tak ...");
Console.WriteLine("Sprawdź czy istnieje folder");
}
catch (Exception ex)
{
Console.WriteLine("Coś poszło nie tak ...");
Console.WriteLine($"Wystąpił błąd {ex.Message}");
}
finally
{
Console.WriteLine("Kończę działanie !");
}
Console.Read();
}
public void Check_1Skill_1Quest_Requirements()
{
AchievementObject aCleanSlate = new AchievementObject
{
AName = "A Clean Slate",
ADescription = "Cleanse the Corrupted Seren Stone with at least one crystal.",
ARunescore = 15,
AMembers = "Yes",
ALink = "/w/A_Clean_Slate",
ACategories = new List <string>
{
"Exploration"
},
ASubcategories = new List <string>
{
"Tirannwn"
},
AQuestReqs = new List <string>(),
ASkillReqs = new List <string>()
};
aCleanSlate = Scrape.GetRequirements(aCleanSlate);
Assert.Contains("Plague's End", aCleanSlate.AQuestReqs);
Assert.Contains("75 Prayer", aCleanSlate.ASkillReqs);
}
public void GetAll_MultipleRecordsToReturn_ReturnsAllRecords()
{
var source = new Source()
{
Id = 1, Name = "name", FriendlyName = "FriendlyName"
};
var scrape1 = new Scrape()
{
Id = 1, SourceId = 1, Source = source, Surname = "surname", StartedAt = DateTime.UtcNow
};
var scrape2 = new Scrape()
{
Id = 2, SourceId = 1, Source = source, Surname = "surname2", StartedAt = DateTime.UtcNow
};
var data = new List <Scrape> {
scrape1, scrape2
}.AsQueryable();
var mockDbSet = GetMockDbSet(data);
var mockContext = new Mock <IDatabaseContext>();
mockContext.Setup(c => c.Scrapes).Returns(mockDbSet.Object);
var repository = new ScrapeRepository(mockContext.Object);
var result = repository.GetAll();
Assert.NotNull(result);
Assert.Equal(2, result.Count <Scrape>());
}
private void threadFunc(object arg)
{
try
{
List <string> data = arg as List <string>;
int ThreadIndex = 0;
int.TryParse(data[0], out ThreadIndex);
List <string> param = new List <string>();
for (int i = 1; i < data.Count; i++)
{
param.Add(data[i]);
}
task = new Scrape(ThreadIndex, onWriteStatus, onSetProgress, param, delay, useProxy, proxylist);
bool result = false;
if (rbtnname.Checked == true)
{
result = task.doWorkByName(savepath);
}
else
{
result = task.doWorkBySeller(savepath);
}
}
catch (Exception e)
{
string error = e.Message;
}
}
public void GetAch_Great_Responsibility()
{
string greatResponsibility = @"<tr class=''><td><a href='/w/Great_Responsibility' title='Great Responsibility'>Great Responsibility</a></td><td>Yes</td><td>Have over 490,000 charge<sup class='noprint fact'>[<span class='fact-text' title='The preceding quote has been reproduced verbatim and is not a transcription error.'>sic</span>]</sup> in a jumbo generator in the Invention Guild.</td><td>Skills</td><td>Invention</td><td>20<p><br></p></td></tr>";
HtmlNode achNode = HtmlNode.CreateNode(greatResponsibility);
AchievementObject achievementActual = Scrape.GetAchievementRow(achNode);
AchievementObject achievementExpected = new AchievementObject
{
AName = "Great Responsibility",
AMembers = "Yes",
ADescription = "Have over 490,000 charge[sic] in a jumbo generator in the Invention Guild.",
ACategories = new List <string>
{
"Skills"
},
ASubcategories = new List <string>
{
"Invention"
},
ARunescore = 20,
ALink = "/w/Great_Responsibility",
AQuestReqs = new List <string>
{
"None"
},
ASkillReqs = new List <string>
{
"102 Invention"
}
};
AssertingAchObj(achievementExpected, achievementActual);
}
public void GetAch_A_Mini_Shipment()
{
string aMiniShipmentRow = @"<tr class=''><td><a href='/w/A_Mini_Shipment' title='A Mini Shipment'>A Mini Shipment</a></td><td>Yes</td><td>Loot the trunk in the Temple of Aminishi.</td><td>Combat<br>Completionist</td><td>Elite Dungeons<br>Master Quest Cape</td><td>5<p><br></p></td></tr>";
HtmlNode achNode = HtmlNode.CreateNode(aMiniShipmentRow);
AchievementObject achievementActual = Scrape.GetAchievementRow(achNode);
AchievementObject achievementExpected = new AchievementObject
{
AName = "A Mini Shipment",
AMembers = "Yes",
ADescription = "Loot the trunk in the Temple of Aminishi.",
ACategories = new List <string>
{
"Combat",
"Completionist"
},
ASubcategories = new List <string>
{
"Elite Dungeons",
"Master Quest Cape"
},
ARunescore = 5,
AQuestReqs = new List <string>
{
"Curse of the Black Stone (partial)"
},
ASkillReqs = new List <string>
{
"None"
},
ALink = "/w/A_Mini_Shipment"
};
AssertingAchObj(achievementExpected, achievementActual);
}
protected static async Task ScanListings()
{
string appid = ConfigurationManager.AppSettings["appID"];
int count = db.PostedListings.Count();
int i = 0;
//foreach (PostedListing p in db.PostedListings.Where(r => r.ListedItemID == "223072961963"))
foreach (PostedListing p in db.PostedListings)
{
try
{
string sourcrUrl = p.SourceUrl;
var result = await Scrape.GetDetail(sourcrUrl);
Console.WriteLine("processing " + (++i) + " of " + count.ToString());
if (!string.IsNullOrEmpty(result.availability))
{
if (result.availability.Contains("Out of stock"))
{
var myListing = await scrapeAPI.ebayAPIs.GetSingleItem(p.ListedItemID, appid);
if (myListing.Qty > 0)
{
Console.WriteLine("OUT OF STOCK " + p.Title);
string reviseResult = scrapeAPI.ebayAPIs.ReviseQty(p.ListedItemID, qty: 0);
dsutil.DSUtil.WriteFile(Log_File, p.ListedItemID + " " + p.Title);
dsutil.DSUtil.WriteFile(Log_File, p.SourceUrl);
dsutil.DSUtil.WriteFile(Log_File, reviseResult);
string ret = await dsutil.DSUtil.SendMailProd("*****@*****.**", "OUT OF STO " + p.Title, "revise listing", "localhost");
if (!string.IsNullOrEmpty(ret))
{
dsutil.DSUtil.WriteFile(Log_File, "prod email failed: " + ret);
ret = dsutil.DSUtil.SendMailDev("*****@*****.**", "OUT OF STO " + p.Title, "revise listing");
if (!string.IsNullOrEmpty(ret))
{
dsutil.DSUtil.WriteFile(Log_File, "dev email failed: " + ret);
}
}
}
else
{
dsutil.DSUtil.WriteFile(Log_File, p.ListedItemID + " " + p.Title);
dsutil.DSUtil.WriteFile(Log_File, p.SourceUrl);
dsutil.DSUtil.WriteFile(Log_File, "Listing quantity already set to 0 - no action taken.");
}
dsutil.DSUtil.WriteFile(Log_File, string.Empty);
}
}
}
catch (Exception exc)
{
dsutil.DSUtil.WriteFile(Log_File, "ERROR: " + p.ListedItemID + " " + exc.Message);
}
}
dsutil.DSUtil.WriteFile(Log_File, string.Format("Processed {0} listings.", count));
}
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
string outputString = myScrape.ScrapeWebPage("https://www.atom.io");
Console.WriteLine(outputString);
Console.ReadLine();
}
public IList<Payment> GetPayments()
{
Scrape scrape = new Scrape();
scrape.user = user;
scrape.pass = pass;
scrape.Run();
return scrape.Payments;
}
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
string value = myScrape.ScrapeWebPage("https://duckduckgo.com/");
Console.WriteLine(value);
Console.ReadLine();
}
static void Main(string[] args)
{
Console.WriteLine("Hello World!");
Scrape scrape = new Scrape();
string value = scrape.ScrapeWebpage("http://msdn.microsoft.com");
Console.WriteLine(value);
}
static void Main(string[] args)
{
Scrape myScrape = new Scrape(); //references from MyCodeLibrary
string value = myScrape.ScrapeWebPage("https://ois2.ttu.ee/uusois/uus_ois2.tud_leht");
Console.WriteLine(value);
Console.ReadLine();
}
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
string value = myScrape.ScrapeWebPage("http://google.com");
Console.WriteLine(value);
Console.ReadLine();
}
//to access parts of the .net framework class library.
//1. right click references in the solution explorer and add reference.
//2. Nuget
//-----
//Make our own class library to reference, here we scraped a web page and wrote it to a file.
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
string result = myScrape.ScrapeWebpage("http://www.msdn.microsoft.com", @"C:\Users\asus\Desktop\Programming\CSharp\learncs\ReferenceToAssembly\WriteText.txt");
Console.WriteLine(result);
Console.ReadLine();
}
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
string reply = myScrape.ScrapeWebPage("https://docs.microsoft.com/en-us/dotnet/csharp/programming-guide/file-system/how-to-write-to-a-text-file");
Console.WriteLine(reply);
}
private void btnRegExChunkTest_Click(object sender, EventArgs e)
{
Scrape s = new Scrape();
this.btnRegExChunkTest.Enabled = false;
this.txtRegExStart.Enabled = false;
this.chkStartIgnoreCase.Enabled = false;
this.nudStartIteration.Enabled = false;
this.txtRegExEnd.Enabled = false;
this.chkEndIgnoreCase.Enabled = false;
this.nudEndIteration.Enabled = false;
this.lstMatchResultsREC.Enabled = false;
this.txtMatchTextREC.Text = string.Empty;
this.txtMatchTextREC.Enabled = false;
this.lstMatchResultsREC.Items.Clear();
this.lstMatchResultsREC.Enabled = false;
this.Refresh();
try
{
StringCollection results = s.RegExChunk(
this.txtCaptureRURL.Text,
this.txtRegExStart.Text,
this.chkStartIgnoreCase.Checked,
int.Parse(this.nudStartIteration.Value.ToString()),
this.txtRegExEnd.Text,
this.chkEndIgnoreCase.Checked,
int.Parse(this.nudEndIteration.Value.ToString()),
int.Parse(this.nudMaximumMatches.Value.ToString()),
int.Parse(this.nudMaxChunkLength.Value.ToString()));
this.lstMatchResultsREC.Items.Clear();
foreach (string str in results)
{
this.lstMatchResultsREC.Items.Add(str);
}
if (this.lstMatchResultsREC.Items.Count > 0)
{
this.lstMatchResultsREC.SelectedIndex = 0;
}
}
catch (Exception ex)
{
this.txtMatchTextREC.Text = "FAIL";
this.lstMatchResultsREC.Items.Clear();
this.lstMatchResultsREC.Items.Add(DetailedException.WithEnterpriseContent(ref ex));
}
this.btnRegExChunkTest.Enabled = true;
this.txtRegExStart.Enabled = true;
this.chkStartIgnoreCase.Enabled = true;
this.nudStartIteration.Enabled = true;
this.txtRegExEnd.Enabled = true;
this.chkEndIgnoreCase.Enabled = true;
this.nudEndIteration.Enabled = true;
this.lstMatchResultsREC.Enabled = true;
this.txtMatchTextREC.Enabled = true;
this.lstMatchResultsREC.Enabled = true;
}
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
string value = myScrape.ScrapeWebepage("http://estakada.lt");
Console.WriteLine(value);
Console.ReadLine();
}
static void Main(string[] args)
{
Scrape myScrape = new Scrape();
//string value = myScrape.ScrapeWebpage("http://msdn.microsoft.com");
string value = myScrape.ScrapeWebpage("http://msdn.microsoft.com", @"c:\users\public\testfolder\html2.txt");
Console.WriteLine(value);
Console.ReadLine();
}
static void Main(string[] args)
{
// because we added a reference to this file, we can now use MyCodeLibrary
// now we can use classes/methods from this namespace
Scrape myScrape = new Scrape();
string value = myScrape.ScrapeWebpage("https://docs.microsoft.com/en-us/");
Console.WriteLine(value);
}
public IList <Payment> GetPayments()
{
Scrape scrape = new Scrape();
scrape.user = user;
scrape.pass = pass;
scrape.Run();
return(scrape.Payments);
}
static void Main(string[] args)
{
// be sure to set ScrapClient as StartUp Project to run
Scrape myScrape = new Scrape();
string value = myScrape.ScrapeWebpage("http://msdn.microsoft.com");
Console.WriteLine(value);
Console.ReadLine();
}
