private static void Run(Options opts) { if (opts.InputWebsites != null) { foreach (string website in opts.InputWebsites) { // Verificar se estamos perante um website Uri uriResult; bool result = Uri.TryCreate(website, UriKind.Absolute, out uriResult) && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps); if (result == true && website.Contains("kuantokusta")) { WebsiteDetails websiteDetails = new WebsiteDetails(); websiteDetails.WebsiteUrl = website; websiteDetails.Category = website.Substring(website.LastIndexOf('/') + 1); AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails); } } } if (File.Exists(opts.filename)) { ReadWebsiteDetailsFromFile(opts.filename); } }
public void should_throw_exception_when_passing_null_website() { string errorMessage = ApiResponses.WebsiteDetailsCannotBeNull; WebsiteDetails website = null; _sut.Invoking(n => n.Add(website)).Should().Throw <InvalidWebsiteDetailsException>().WithMessage(errorMessage); }
public static bool InsertWebsiteDetailsIntoDataBase(WebsiteDetails websiteDetails) { using (SqlConnection conn = new SqlConnection(ConnectionToDatabase.ConnectionString)) { // 1. create a command object identifying the stored procedure SqlCommand cmd = new SqlCommand("InsertWebsiteToTrack", conn); // 2. set the command object so it knows to execute a stored procedure cmd.CommandType = CommandType.StoredProcedure; // 3. add parameter to command, which will be passed to the stored procedure cmd.Parameters.Add(new SqlParameter("@URL", websiteDetails.WebsiteUrl)); cmd.Parameters.Add(new SqlParameter("@Category", websiteDetails.Category)); try { conn.Open(); cmd.ExecuteNonQuery(); } catch (SqlException ex) { LogDatabaseAccessError(ex); return(false); } } return(true); }
private void validateWebsiteDetails(WebsiteDetails websiteDetails) { if (websiteDetails is null) { throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsCannotBeNull); } if (string.IsNullOrEmpty(websiteDetails.Url)) { throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsUrlCannotBeNullOrEmpty); } if (string.IsNullOrEmpty(websiteDetails.MainNodeXPathToNewsContainer)) { throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsXpathCannotBeNullOrEmpty); } if (string.IsNullOrEmpty(websiteDetails.NewsNodeTag)) { throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsNewsNodeCannotBeNullOrEmpty); } if (websiteDetails.Categories != null) { var categoriesIds = websiteDetails.Categories.Select(n => n.Id); if (categoriesIds.Count() != categoriesIds.Distinct().Count()) { throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsCategoriesCannotBeDuplicated); } } }
private void InsertWebsiteToTrack() { // Variáveis locais string ch; int x; string link = string.Empty; //do //{ //try //{ x = Program.V_View.AskUserChar(); ch = Char.ToString(Convert.ToChar(x)); if (ch.ToLower() == "Y".ToLower()) { // Inserir na base de dados do { // Criar detalhes do website para guardar na base de dados WebsiteDetails websiteDetails = new WebsiteDetails(); Program.V_View.DisplayMessage("INFO", "Insira aqui o link(Pressiona ESC para cancelar): "); websiteDetails.WebsiteUrl = Program.V_View.ReadLineWithCancel(); Colorful.Console.WriteLine(); Program.V_View.DisplayMessage("INFO", "Insira o nome da Categoria associado ao link colocado acima: "); websiteDetails.Category = Program.V_View.ReadLineWithCancel(); if (link != null) { AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails); Program.V_View.DisplayMessage("INFO", "Link Foi adicionado com sucesso à base de dados"); } } while (link != null); if (link == null && ((Scrapper.dt = AcessDataBase.GetAllUrl()) == null) && Scrapper.dt.Rows.Count == 0) { Program.V_View.DisplayMessage("CRITICAL", "Nenhum link foi adicionado para fazer a verificação"); Program.V_View.DisplayMessage("INFO", "O programa vai encerrar"); Thread.Sleep(1000); return; } else { PrintLinksToBeParsed(); } } else if (ch.ToLower() == "N".ToLower()) { Program.V_View.DisplayMessage("INFO", "O program vai encerrar"); Thread.Sleep(1000); return; } }
public void should_throw_exception_when_passing_null_website() { string errorMessage = ApiResponses.WebsiteDetailsCannotBeNull; WebsiteDetails website = null; WebsiteDetailsDb websiteDb = null; _mapper.Setup(n => n.Map <WebsiteDetailsDb>(It.IsAny <WebsiteDetailsDb>())).Returns(websiteDb); _mapper.Setup(n => n.Map <WebsiteDetails>(It.IsAny <WebsiteDetailsDb>())).Returns(website); _sut.Invoking(n => n.Save(website)).Should().Throw <InvalidWebsiteDetailsException>().WithMessage(errorMessage); }
public void should_return_null_when_not_found() { int id = 1321; WebsiteDetails website = null; WebsiteDetailsDb gotWebsite = website.Map(); _mapper.Setup(n => n.Map <WebsiteDetails>(It.IsAny <WebsiteDetailsDb>())).Returns(website); _websitesRepository.Setup(n => n.Get(id)).Returns(gotWebsite); var result = _sut.Get(id); result.Should().BeNull(); }
public ActionResult <WebsiteDetailsResponseViewModel> Save(WebsiteDetails website) { WebsiteDetailsResponseViewModel result = new(); try { var savedWebsite = _websiteDetailsService.Save(website); result.WebsiteDetails.Add(savedWebsite); } catch (InvalidWebsiteDetailsException ex) { result.ErrorMessages.Add(ex.Message); } return(Ok(result)); }
private ArticlesResponseViewModel scrapSingleWebsite(WebsiteDetails website, string rawHtml) { ArticlesResponseViewModel result = new(); if (isStringHtml(rawHtml)) { (List <Article> articles, List <string> errors) = _htmlScrapper.Scrap(website, rawHtml); result.Articles.AddRange(articles); result.ErrorMessages.AddRange(errors); return(result); } result.ErrorMessages.Add(rawHtml); return(result); }
public WebsiteDetails Add(WebsiteDetails websiteDetails) { validateWebsiteDetails(websiteDetails); var websiteDetailsDb = _mapper.Map <WebsiteDetailsDb>(websiteDetails); if (websiteDetails.Categories != null) { var chosenCategories = websiteDetails?.Categories.Select(n => n.Id); var chosenCategoriesDb = _categoriesRepository.Get(n => chosenCategories.Contains(n.Id)); websiteDetailsDb.Categories = chosenCategoriesDb.ToList(); } var result = _mapper.Map <WebsiteDetails>(_websitesRepository.Add(websiteDetailsDb)); _websitesRepository.Commit(); return(result); }
public static WebsiteDetailsDb Map(this WebsiteDetails website) { if (website is null) { return(null); } return(new() { id = website.Id, Url = website.Url, MainNodeXPathToNewsContainer = website.MainNodeXPathToNewsContainer, NewsNodeTag = website.NewsNodeTag, NewsNodeClass = website.NewsNodeClass, TitleNodeTag = website.TitleNodeTag, TitleNodeClass = website.TitleNodeClass, DescriptionNodeTag = website.DescriptionNodeTag, DescriptionNodeClass = website.DescriptionNodeClass, ImgNodeClass = website.ImgNodeClass, Category = website.Category, Categories = mapCategories(website.Categories) }); }
private static void ReadWebsiteDetailsFromFile(string filePath) { string[] lines = System.IO.File.ReadAllLines(filePath); int contador = 0; foreach (string line in lines) { // Use a tab to indent each line of the file. String[] parts = line.Split(';'); //Console.WriteLine(line); WebsiteDetails websiteDetails = new WebsiteDetails(); websiteDetails.WebsiteUrl = parts[0]; websiteDetails.Category = parts[1]; if ((AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails) == true)) { contador++; } } Program.V_View.DisplayMessage("INFO", $"Foram adicionados {contador} links a pesquisar à base de dados"); }
public WebsiteDetails Save(WebsiteDetails websiteDetails) { validateWebsiteDetails(websiteDetails); var websiteDetailsAlreadyExisting = _websitesRepository.GetWithCategories(websiteDetails.Id); websiteDetailsAlreadyExisting.Categories.Clear(); _mapper.Map(websiteDetails, websiteDetailsAlreadyExisting); if (websiteDetails.Categories != null) { var chosenCategories = websiteDetails.Categories.Select(n => n.Id); var chosenCategoriesDb = _categoriesRepository.Get(n => chosenCategories.Contains(n.Id)); websiteDetailsAlreadyExisting.Categories = chosenCategoriesDb.ToList(); } _mapper.Map(_websitesRepository.Save(websiteDetailsAlreadyExisting), websiteDetails); _websitesRepository.Commit(); return(websiteDetails); }
public (List <Article>, List <string>) Scrap(WebsiteDetails website, string rawHtml) { List <Article> articles = new(); List <string> errors = new(); try { var mainNode = getMainNode(rawHtml, website.MainNodeXPathToNewsContainer); var newsNodes = getNewsNodes(mainNode, website.NewsNodeTag, website.NewsNodeClass); for (int i = 0; i < newsNodes.Count; i++) { HtmlNode titleNode = getTitleNode(website.TitleNodeTag, website.TitleNodeClass, newsNodes[i]); string title = getTitle(titleNode); string directUrlToNews = getNewsUrl(titleNode); string image = getImageUrl(website.ImgNodeClass, newsNodes[i]); string description = getDescription(website.DescriptionNodeTag, website.DescriptionNodeClass, newsNodes[i]); articles.Add(new() { Title = title, Url = getDirectUrl(directUrlToNews, website.Url), ImageUrl = getDirectUrl(image, website.Url), Description = description, DateScrapped = getShortDateTime(_dateTimeProvider.Now), WebsiteDetailsId = website.Id }); } } catch (Exception ex) { errors.Add(ex.Message); } return(articles, errors); }
private static void ReadWebsiteDetailsFromFile(string filePath) { string[] lines = System.IO.File.ReadAllLines(filePath); int contador = 0; foreach (string line in lines) { // Use a tab to indent each line of the file. String[] parts = line.Split(';'); //Console.WriteLine(line); WebsiteDetails websiteDetails = new WebsiteDetails(); websiteDetails.WebsiteUrl = parts[0]; websiteDetails.Category = parts[1]; if ((AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails) == true)) { contador++; } } //Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] Foram adicionados {contador} links a pesquisar à base de dados", Program.M_Options.StyleConsole); }
public void TestGetDetailsName() { WebsiteDetails details = _application.GetDetails(_range); Assert.AreEqual(details.Name, "example.com"); }
public void TestGetDetailsPaths() { WebsiteDetails details = _application.GetDetails(_range); Assert.AreEqual(details.HourlyPaths.Count, 1); }
private static async Task RunAsync() { Colorful.Console.WriteAscii("KuantoKusta Scrapper", Color.FromArgb(244, 212, 255)); Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] Bem vindo ao Programa", M_Options.StyleConsole); //} WebScrappingPage scrappingPage = new WebScrappingPage(); Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] Vamos proceder ao ínicio da operação de obter os produtos e os respetivos preços", M_Options.StyleConsole); DataTable dt = AcessDataBase.GetAllUrl(); if (dt != null) { if (dt.Rows.Count == 0) { Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [WARNING] Não foi possível obter nenhum link", M_Options.StyleConsole); Colorful.Console.WriteStyled($"[{DateTime.Now:HH:mm:ss}] Pretende adicionar algum link através da consola? (Y/N): ", M_Options.StyleConsole); // Variáveis locais string ch; int x; string link = string.Empty; //do //{ x = Colorful.Console.Read(); //try //{ ch = Char.ToString(Convert.ToChar(x)); if (ch.ToLower() == "Y".ToLower()) { // Inserir na base de dados do { // Criar detalhes do website para guardar na base de dados WebsiteDetails websiteDetails = new WebsiteDetails(); Colorful.Console.WriteStyled($"[{DateTime.Now:HH:mm:ss}] Insira aqui o link (Pressiona ESC para cancelar): ", M_Options.StyleConsole); websiteDetails.WebsiteUrl = ReadLineWithCancel(); Colorful.Console.WriteLine(); Colorful.Console.WriteStyled($"[{DateTime.Now:HH:mm:ss}] Insira o nome da Categoria associado ao link colocado acima: ", M_Options.StyleConsole); websiteDetails.Category = Colorful.Console.ReadLine(); if (link != null) { AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails); Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] Link Foi adicionado com sucesso à base de dados", M_Options.StyleConsole); } } while (link != null); if (link == null && ((dt = AcessDataBase.GetAllUrl()) == null) && dt.Rows.Count == 0) { Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [CRITICAL] Nenhum link foi adicionado para fazer a verificação", M_Options.StyleConsole); Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] O programa vai encerrar", M_Options.StyleConsole); Thread.Sleep(1000); return; } else { PrintLinksToBeParsed(M_Options.StyleConsole, dt); } } else if (ch.ToLower() == "N".ToLower()) { Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] O programa vai encerrar", M_Options.StyleConsole); Thread.Sleep(1000); return; } //} //catch (OverflowException e) //{ // Colorful.Console.WriteLine("{0} Value read = {1}.", e.Message, x); // ch = Char.MinValue; // Colorful.Console.WriteLine(m1); //} //} while (ch != '+'); } else { PrintLinksToBeParsed(M_Options.StyleConsole, dt); } }
public void TestGetDetailsAddress() { WebsiteDetails details = _application.GetDetails(_range); Assert.AreEqual(details.Address, "https://example.com"); }
public static bool InsertWebsiteDetailsIntoDataBase(WebsiteDetails websiteDetails) { return(DataBaseAccessProcessing.InsertWebsiteDetailsIntoDataBase(websiteDetails)); }