public List <WebScraping> GetAll()
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
List <WebScraping> List = new List <WebScraping>();
string url = "http://www.touropia.com/best-places-to-visit-in-usa/";
var htmlWeb = new HtmlWeb();
HtmlDocument document = null;
document = htmlWeb.Load(url);
//getting all the span with class toptitle first
var items = document.DocumentNode.Descendants("span")
.Where(node => node.GetAttributeValue("class", "")
.Equals("toptitle")).ToList();
foreach (var node in items)
{
WebScraping item = new WebScraping();
item.Title = node.InnerText;
//getting the anchor tag inside the span tag
item.Url = node.Descendants("a").FirstOrDefault().GetAttributeValue("href", "");
List.Add(item);
//var getUrl = document.DocumentNode.Descendants("a");
// item.Url = node.GetAttributeValue("href", "");
}
return(List);
}
private void btnSave_Click(object sender, EventArgs e)
{
//making sure the latest values go into the BindingSource (that happen only when the control loses focus)
ucMovieInfo1.tbEncodedWith.Focus();
if (!ucMovieInfo1.ValidateInput())
{
MsgBox.Show("Please specify all required details!", "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
DAL.NewMTD.ParentId = _collectionId;
var opRes = DAL.InsertMTD(DAL.NewMTD, null);
if (!opRes.Success)
{
MsgBox.Show(
string.Format("The following error occurred while inserting the new Movie into the database:{0}{0}{1}{0}{0}", Environment.NewLine, opRes.CustomErrorMessage),
"Error!", MessageBoxButtons.OK, MessageBoxIcon.Error);
ucMovieInfo1.tbTitle.Focus();
return;
}
DAL.NewMTD.Id = (int)opRes.AdditionalDataReturn;
DAL.CurrentMTD = DAL.NewMTD;
Common.Helpers.UnsavedChanges = false;
if (ucMovieInfo1.GetCSMData && !string.IsNullOrEmpty(DAL.CurrentMTD.RecommendedLink))
{
if (!DAL.CurrentMTD.RecommendedLink.ToLower().Contains("commonsensemedia"))
{
Utils.Helpers.ShowToastForm(StartPosition2.BottomRight, MessageType.Warning, "Recommended data",
"The 'recommended' data is from a site which doesn't have scraper and parser built!", 5000, this);
}
else
{
opRes = WebScraping.GetCommonSenseMediaData(DAL.CurrentMTD.RecommendedLink);
if (!opRes.Success)
{
Utils.Helpers.ShowToastForm(StartPosition2.BottomRight, MessageType.Warning, "Recommended data",
opRes.CustomErrorMessage, 5000, this);
}
else
{
opRes = Desene.DAL.SaveCommonSenseMediaData(DAL.CurrentMTD.Id, (CSMScrapeResult)opRes.AdditionalDataReturn);
if (!opRes.Success)
{
Utils.Helpers.ShowToastForm(StartPosition2.BottomRight, MessageType.Warning, "Recommended data",
opRes.CustomErrorMessage, 5000, this);
}
}
}
}
DialogResult = DialogResult.OK;
Close();
}
internal static async Task <List <StockPrice> > DownloadAndSync_Yahoo()
{
int iCount = 0;
var spCollection = new List <StockPrice>();
foreach (var s in await SISDBContext.GetStocksCodedbAsync("Active", LoadFrom.Yahoo))
{
JObject o = JObject.Parse(Newtonsoft.Json.JsonConvert.SerializeObject(s));
var theCode = (string)o["Value"]["Code"];
System.Diagnostics.Debug.WriteLine($"Download started for stock {(string)o["Value"]["SName"]} code : {theCode}");
var sp = WebScraping.ScrapingStart(LoadFrom.Yahoo, theCode);
sp.StockID = (int)o["Value"]["SID"];
sp.StockName = (string)o["Value"]["SName"];
spCollection.Add(sp);
iCount++;
if (iCount > 9)
{
break; //[DK: remove] just for test
}
}
if (await SISDBContext.StockPriceBulkUpdatedbAsync(spCollection))
{
return(spCollection);
}
else
{
throw new Exception("DownloadAndSync_Yahoo: Unable to update the latest Stock Price into database");
}
}
public HomeController()
{
validacaoRepository = new ValidacaoRepository();
webScraping = new WebScraping();
relatorioSimplificadoRepository = new RelatorioSimplificadoRepository();
consultaAnteriorRepository = new ConsultaAnteriorRepository();
arpenspRepository = new ArpenspRepository();
}
/// <summary>
/// コンストラクター
/// </summary>
public MainViewModel()
{
_bookList = WebScraping.GetWebData();
ViewChange = new ViewChangeCommand(this);
Delete = new DeleteCommand(this);
FavoriteView = false;
_dataBaseManager = new DataBaseManager();
_favoriteData = DBManager.DataBaseRead();
SetData();
}
public HttpResponseMessage Post(WebScraping model)
{
try
{
WebScrapingService svc = new WebScrapingService();
int id = svc.Post(model);
ItemResponse <int> response = new ItemResponse <int>();
response.Item = id;
return(Request.CreateResponse(HttpStatusCode.OK, response));
}
catch (Exception ex)
{
return(Request.CreateErrorResponse(HttpStatusCode.BadRequest, ex.Message));
}
}
private void bRefreshCSMData_Click(object sender, EventArgs e)
{
try
{
Cursor = Cursors.WaitCursor;
if (!string.IsNullOrEmpty(DAL.CurrentMTD.RecommendedLink))
{
if (!DAL.CurrentMTD.RecommendedLink.ToLower().Contains("commonsensemedia"))
{
Utils.Helpers.ShowToastForm(StartPosition2.BottomRight, MessageType.Warning, "Recommended data",
"The 'recommended' data is from a site which doesn't have scraper and parser built!", 5000, ParentForm);
}
else
{
var opRes = WebScraping.GetCommonSenseMediaData(DAL.CurrentMTD.RecommendedLink);
if (!opRes.Success)
{
Utils.Helpers.ShowToastForm(StartPosition2.BottomRight, MessageType.Warning, "Recommended data",
opRes.CustomErrorMessage, 5000, ParentForm);
}
else
{
opRes = DAL.SaveCommonSenseMediaData(DAL.CurrentMTD.Id, (CSMScrapeResult)opRes.AdditionalDataReturn);
if (!opRes.Success)
{
Utils.Helpers.ShowToastForm(StartPosition2.BottomRight, MessageType.Warning, "Recommended data",
opRes.CustomErrorMessage, 5000, ParentForm);
}
else
{
_hasRecommendedDataSaved = true;
Utils.Helpers.ShowToastForm(StartPosition2.BottomRight, MessageType.Information, "Recommended data",
"CommonSenseMedia data was scraped, parsed and saved succesfully!", 5000, ParentForm);
}
}
}
}
}
finally
{
Cursor = Cursors.Default;
}
}
private void TryGetSynopsis()
{
if (!string.IsNullOrEmpty(tbSynopsis.Text))
{
return;
}
tbDescriptionLink.DataBindings[0].WriteValue();
if (string.IsNullOrEmpty(tbDescriptionLink.Text))
{
tbSynopsis.Text = string.Empty;
tbSynopsis.DataBindings[0].WriteValue();
}
else
{
try
{
pbLoader.Visible = true;
var opRes = WebScraping.GetSynopsis(tbDescriptionLink.Text);
if (!opRes.Success)
{
Utils.Helpers.ShowToastForm(StartPosition2.BottomRight, MessageType.Warning, "Synopsis retrieval",
opRes.CustomErrorMessage, 5000, (this.Parent as Form));
}
else
{
tbSynopsis.Text = (string)opRes.AdditionalDataReturn;
tbSynopsis.DataBindings[0].WriteValue();
if (Helpers.GenericSetButtonsState2 != null)
{
Helpers.GenericSetButtonsState2(true);
}
}
}
finally
{
pbLoader.Visible = false;
}
}
}
public int Post(WebScraping model)
{
int id = 0;
this.DataProvider.ExecuteNonQuery(
"WebScrapUrl_Insert",
inputParamMapper : delegate(SqlParameterCollection paramCol)
{
paramCol.AddWithValue("@Id", model.Id);
paramCol.AddWithValue("@Title", model.Title);
paramCol.AddWithValue("@Url", model.Url);
},
returnParameters : delegate(SqlParameterCollection paramCol)
{
id = (int)paramCol["@Id"].Value;
}
);
return(id);
}
public void TestWebScraping()
{
var ws = new WebScraping();
var data = ws.GetGitHub("gilbelei", "testegithub");
Assert.AreEqual(9, data.Count);
AssertData(data, "docx", 114688, 0);
AssertData(data, "exe", 879616, 0);
AssertData(data, "html", 6488680, 11276);
AssertData(data, "jpg", 807936, 0);
AssertData(data, "mp3", 1315962880, 0);
AssertData(data, "mpeg", 1028653056, 0);
AssertData(data, "pdf", 111149056, 0);
AssertData(data, "wmv", 204472320, 0);
AssertData(data, "xlsx", 210944, 0);
}
public EvangelhoRepository(WebScraping webScraping)
{
_webScraping = webScraping;
}
public PesquisaCPFCNPJRepository()
{
webScrapingArpensp = new WebScraping();
}
public LeituraRepository(WebScraping webScraping)
{
_webScraping = webScraping;
}
public RelatorioSimplificadoRepository()
{
webScrapingArpensp = new WebScraping();
WebScraping webScraping = new WebScraping();
}
