private static void Run(Options opts)
{
if (opts.InputWebsites != null)
{
foreach (string website in opts.InputWebsites)
{
// Verificar se estamos perante um website
Uri uriResult;
bool result = Uri.TryCreate(website, UriKind.Absolute, out uriResult) &&
(uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps);
if (result == true && website.Contains("kuantokusta"))
{
WebsiteDetails websiteDetails = new WebsiteDetails();
websiteDetails.WebsiteUrl = website;
websiteDetails.Category = website.Substring(website.LastIndexOf('/') + 1);
AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails);
}
}
}
if (File.Exists(opts.filename))
{
ReadWebsiteDetailsFromFile(opts.filename);
}
}
public void should_throw_exception_when_passing_null_website()
{
string errorMessage = ApiResponses.WebsiteDetailsCannotBeNull;
WebsiteDetails website = null;
_sut.Invoking(n => n.Add(website)).Should().Throw <InvalidWebsiteDetailsException>().WithMessage(errorMessage);
}
public static bool InsertWebsiteDetailsIntoDataBase(WebsiteDetails websiteDetails)
{
using (SqlConnection conn = new SqlConnection(ConnectionToDatabase.ConnectionString))
{
// 1. create a command object identifying the stored procedure
SqlCommand cmd = new SqlCommand("InsertWebsiteToTrack", conn);
// 2. set the command object so it knows to execute a stored procedure
cmd.CommandType = CommandType.StoredProcedure;
// 3. add parameter to command, which will be passed to the stored procedure
cmd.Parameters.Add(new SqlParameter("@URL", websiteDetails.WebsiteUrl));
cmd.Parameters.Add(new SqlParameter("@Category", websiteDetails.Category));
try
{
conn.Open();
cmd.ExecuteNonQuery();
}
catch (SqlException ex)
{
LogDatabaseAccessError(ex);
return(false);
}
}
return(true);
}
private void validateWebsiteDetails(WebsiteDetails websiteDetails)
{
if (websiteDetails is null)
{
throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsCannotBeNull);
}
if (string.IsNullOrEmpty(websiteDetails.Url))
{
throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsUrlCannotBeNullOrEmpty);
}
if (string.IsNullOrEmpty(websiteDetails.MainNodeXPathToNewsContainer))
{
throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsXpathCannotBeNullOrEmpty);
}
if (string.IsNullOrEmpty(websiteDetails.NewsNodeTag))
{
throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsNewsNodeCannotBeNullOrEmpty);
}
if (websiteDetails.Categories != null)
{
var categoriesIds = websiteDetails.Categories.Select(n => n.Id);
if (categoriesIds.Count() != categoriesIds.Distinct().Count())
{
throw new InvalidWebsiteDetailsException(ApiResponses.WebsiteDetailsCategoriesCannotBeDuplicated);
}
}
}
private void InsertWebsiteToTrack()
{
// Variáveis locais
string ch;
int x;
string link = string.Empty;
//do
//{
//try
//{
x = Program.V_View.AskUserChar();
ch = Char.ToString(Convert.ToChar(x));
if (ch.ToLower() == "Y".ToLower())
{
// Inserir na base de dados
do
{
// Criar detalhes do website para guardar na base de dados
WebsiteDetails websiteDetails = new WebsiteDetails();
Program.V_View.DisplayMessage("INFO", "Insira aqui o link(Pressiona ESC para cancelar): ");
websiteDetails.WebsiteUrl = Program.V_View.ReadLineWithCancel();
Colorful.Console.WriteLine();
Program.V_View.DisplayMessage("INFO", "Insira o nome da Categoria associado ao link colocado acima: ");
websiteDetails.Category = Program.V_View.ReadLineWithCancel();
if (link != null)
{
AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails);
Program.V_View.DisplayMessage("INFO", "Link Foi adicionado com sucesso à base de dados");
}
} while (link != null);
if (link == null && ((Scrapper.dt = AcessDataBase.GetAllUrl()) == null) && Scrapper.dt.Rows.Count == 0)
{
Program.V_View.DisplayMessage("CRITICAL", "Nenhum link foi adicionado para fazer a verificação");
Program.V_View.DisplayMessage("INFO", "O programa vai encerrar");
Thread.Sleep(1000);
return;
}
else
{
PrintLinksToBeParsed();
}
}
else if (ch.ToLower() == "N".ToLower())
{
Program.V_View.DisplayMessage("INFO", "O program vai encerrar");
Thread.Sleep(1000);
return;
}
}
public void should_throw_exception_when_passing_null_website()
{
string errorMessage = ApiResponses.WebsiteDetailsCannotBeNull;
WebsiteDetails website = null;
WebsiteDetailsDb websiteDb = null;
_mapper.Setup(n => n.Map <WebsiteDetailsDb>(It.IsAny <WebsiteDetailsDb>())).Returns(websiteDb);
_mapper.Setup(n => n.Map <WebsiteDetails>(It.IsAny <WebsiteDetailsDb>())).Returns(website);
_sut.Invoking(n => n.Save(website)).Should().Throw <InvalidWebsiteDetailsException>().WithMessage(errorMessage);
}
public void should_return_null_when_not_found()
{
int id = 1321;
WebsiteDetails website = null;
WebsiteDetailsDb gotWebsite = website.Map();
_mapper.Setup(n => n.Map <WebsiteDetails>(It.IsAny <WebsiteDetailsDb>())).Returns(website);
_websitesRepository.Setup(n => n.Get(id)).Returns(gotWebsite);
var result = _sut.Get(id);
result.Should().BeNull();
}
public ActionResult <WebsiteDetailsResponseViewModel> Save(WebsiteDetails website)
{
WebsiteDetailsResponseViewModel result = new();
try
{
var savedWebsite = _websiteDetailsService.Save(website);
result.WebsiteDetails.Add(savedWebsite);
}
catch (InvalidWebsiteDetailsException ex)
{
result.ErrorMessages.Add(ex.Message);
}
return(Ok(result));
}
private ArticlesResponseViewModel scrapSingleWebsite(WebsiteDetails website, string rawHtml)
{
ArticlesResponseViewModel result = new();
if (isStringHtml(rawHtml))
{
(List <Article> articles, List <string> errors) = _htmlScrapper.Scrap(website, rawHtml);
result.Articles.AddRange(articles);
result.ErrorMessages.AddRange(errors);
return(result);
}
result.ErrorMessages.Add(rawHtml);
return(result);
}
public WebsiteDetails Add(WebsiteDetails websiteDetails)
{
validateWebsiteDetails(websiteDetails);
var websiteDetailsDb = _mapper.Map <WebsiteDetailsDb>(websiteDetails);
if (websiteDetails.Categories != null)
{
var chosenCategories = websiteDetails?.Categories.Select(n => n.Id);
var chosenCategoriesDb = _categoriesRepository.Get(n => chosenCategories.Contains(n.Id));
websiteDetailsDb.Categories = chosenCategoriesDb.ToList();
}
var result = _mapper.Map <WebsiteDetails>(_websitesRepository.Add(websiteDetailsDb));
_websitesRepository.Commit();
return(result);
}
public static WebsiteDetailsDb Map(this WebsiteDetails website)
{
if (website is null)
{
return(null);
}
return(new()
{
id = website.Id,
Url = website.Url,
MainNodeXPathToNewsContainer = website.MainNodeXPathToNewsContainer,
NewsNodeTag = website.NewsNodeTag,
NewsNodeClass = website.NewsNodeClass,
TitleNodeTag = website.TitleNodeTag,
TitleNodeClass = website.TitleNodeClass,
DescriptionNodeTag = website.DescriptionNodeTag,
DescriptionNodeClass = website.DescriptionNodeClass,
ImgNodeClass = website.ImgNodeClass,
Category = website.Category,
Categories = mapCategories(website.Categories)
});
}
private static void ReadWebsiteDetailsFromFile(string filePath)
{
string[] lines = System.IO.File.ReadAllLines(filePath);
int contador = 0;
foreach (string line in lines)
{
// Use a tab to indent each line of the file.
String[] parts = line.Split(';');
//Console.WriteLine(line);
WebsiteDetails websiteDetails = new WebsiteDetails();
websiteDetails.WebsiteUrl = parts[0];
websiteDetails.Category = parts[1];
if ((AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails) == true))
{
contador++;
}
}
Program.V_View.DisplayMessage("INFO", $"Foram adicionados {contador} links a pesquisar à base de dados");
}
public WebsiteDetails Save(WebsiteDetails websiteDetails)
{
validateWebsiteDetails(websiteDetails);
var websiteDetailsAlreadyExisting = _websitesRepository.GetWithCategories(websiteDetails.Id);
websiteDetailsAlreadyExisting.Categories.Clear();
_mapper.Map(websiteDetails, websiteDetailsAlreadyExisting);
if (websiteDetails.Categories != null)
{
var chosenCategories = websiteDetails.Categories.Select(n => n.Id);
var chosenCategoriesDb = _categoriesRepository.Get(n => chosenCategories.Contains(n.Id));
websiteDetailsAlreadyExisting.Categories = chosenCategoriesDb.ToList();
}
_mapper.Map(_websitesRepository.Save(websiteDetailsAlreadyExisting), websiteDetails);
_websitesRepository.Commit();
return(websiteDetails);
}
public (List <Article>, List <string>) Scrap(WebsiteDetails website, string rawHtml)
{
List <Article> articles = new();
List <string> errors = new();
try
{
var mainNode = getMainNode(rawHtml, website.MainNodeXPathToNewsContainer);
var newsNodes = getNewsNodes(mainNode, website.NewsNodeTag, website.NewsNodeClass);
for (int i = 0; i < newsNodes.Count; i++)
{
HtmlNode titleNode = getTitleNode(website.TitleNodeTag, website.TitleNodeClass, newsNodes[i]);
string title = getTitle(titleNode);
string directUrlToNews = getNewsUrl(titleNode);
string image = getImageUrl(website.ImgNodeClass, newsNodes[i]);
string description = getDescription(website.DescriptionNodeTag, website.DescriptionNodeClass, newsNodes[i]);
articles.Add(new()
{
Title = title,
Url = getDirectUrl(directUrlToNews, website.Url),
ImageUrl = getDirectUrl(image, website.Url),
Description = description,
DateScrapped = getShortDateTime(_dateTimeProvider.Now),
WebsiteDetailsId = website.Id
});
}
}
catch (Exception ex)
{
errors.Add(ex.Message);
}
return(articles, errors);
}
private static void ReadWebsiteDetailsFromFile(string filePath)
{
string[] lines = System.IO.File.ReadAllLines(filePath);
int contador = 0;
foreach (string line in lines)
{
// Use a tab to indent each line of the file.
String[] parts = line.Split(';');
//Console.WriteLine(line);
WebsiteDetails websiteDetails = new WebsiteDetails();
websiteDetails.WebsiteUrl = parts[0];
websiteDetails.Category = parts[1];
if ((AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails) == true))
{
contador++;
}
}
//Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] Foram adicionados {contador} links a pesquisar à base de dados", Program.M_Options.StyleConsole);
}
public void TestGetDetailsName()
{
WebsiteDetails details = _application.GetDetails(_range);
Assert.AreEqual(details.Name, "example.com");
}
public void TestGetDetailsPaths()
{
WebsiteDetails details = _application.GetDetails(_range);
Assert.AreEqual(details.HourlyPaths.Count, 1);
}
private static async Task RunAsync()
{
Colorful.Console.WriteAscii("KuantoKusta Scrapper", Color.FromArgb(244, 212, 255));
Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] Bem vindo ao Programa", M_Options.StyleConsole);
//}
WebScrappingPage scrappingPage = new WebScrappingPage();
Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] Vamos proceder ao ínicio da operação de obter os produtos e os respetivos preços", M_Options.StyleConsole);
DataTable dt = AcessDataBase.GetAllUrl();
if (dt != null)
{
if (dt.Rows.Count == 0)
{
Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [WARNING] Não foi possível obter nenhum link", M_Options.StyleConsole);
Colorful.Console.WriteStyled($"[{DateTime.Now:HH:mm:ss}] Pretende adicionar algum link através da consola? (Y/N): ", M_Options.StyleConsole);
// Variáveis locais
string ch;
int x;
string link = string.Empty;
//do
//{
x = Colorful.Console.Read();
//try
//{
ch = Char.ToString(Convert.ToChar(x));
if (ch.ToLower() == "Y".ToLower())
{
// Inserir na base de dados
do
{
// Criar detalhes do website para guardar na base de dados
WebsiteDetails websiteDetails = new WebsiteDetails();
Colorful.Console.WriteStyled($"[{DateTime.Now:HH:mm:ss}] Insira aqui o link (Pressiona ESC para cancelar): ", M_Options.StyleConsole);
websiteDetails.WebsiteUrl = ReadLineWithCancel();
Colorful.Console.WriteLine();
Colorful.Console.WriteStyled($"[{DateTime.Now:HH:mm:ss}] Insira o nome da Categoria associado ao link colocado acima: ", M_Options.StyleConsole);
websiteDetails.Category = Colorful.Console.ReadLine();
if (link != null)
{
AcessDataBase.InsertWebsiteDetailsIntoDataBase(websiteDetails);
Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] Link Foi adicionado com sucesso à base de dados", M_Options.StyleConsole);
}
} while (link != null);
if (link == null && ((dt = AcessDataBase.GetAllUrl()) == null) && dt.Rows.Count == 0)
{
Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [CRITICAL] Nenhum link foi adicionado para fazer a verificação", M_Options.StyleConsole);
Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] O programa vai encerrar", M_Options.StyleConsole);
Thread.Sleep(1000);
return;
}
else
{
PrintLinksToBeParsed(M_Options.StyleConsole, dt);
}
}
else if (ch.ToLower() == "N".ToLower())
{
Colorful.Console.WriteLineStyled($"[{DateTime.Now:HH:mm:ss}] [INFO] O programa vai encerrar", M_Options.StyleConsole);
Thread.Sleep(1000);
return;
}
//}
//catch (OverflowException e)
//{
// Colorful.Console.WriteLine("{0} Value read = {1}.", e.Message, x);
// ch = Char.MinValue;
// Colorful.Console.WriteLine(m1);
//}
//} while (ch != '+');
}
else
{
PrintLinksToBeParsed(M_Options.StyleConsole, dt);
}
}
public void TestGetDetailsAddress()
{
WebsiteDetails details = _application.GetDetails(_range);
Assert.AreEqual(details.Address, "https://example.com");
}
public static bool InsertWebsiteDetailsIntoDataBase(WebsiteDetails websiteDetails)
{
return(DataBaseAccessProcessing.InsertWebsiteDetailsIntoDataBase(websiteDetails));
}
