public static void Main()
{
var db = new ApplicationDbContext();
var repo = new DbRepository <JokeCategory>(db);
var categoriesService = new CategoriesService(repo);
var configuration = Configuration.Default.WithDefaultLoader();
var browsingContext = BrowsingContext.New(configuration);
var url = "https://visualstudio.uservoice.com/forums/121579-visual-studio-2015";
var document = browsingContext.OpenAsync(url).Result;
var ideasTitles = document.QuerySelector("uvListItem.uvIdea.uvIdea-list .uvIdeaHeader .uvIdeaTitle a").InnerHtml;
for (int i = 0; i < 15; i++)
{
//var ideaTitle = document.QuerySelector(".uvIdeaTitle a").TextContent.Trim();
//var ideaContent = document.QuerySelector(".uvIdeaDescription.uvIdeaDescription-truncated").InnerHtml;
var jokeContent = document.QuerySelector("#content_box .post-content").TextContent.Trim();
if (!string.IsNullOrWhiteSpace(jokeContent))
{
var categoryName = document.QuerySelector("#content_box .thecategory a").TextContent.Trim();
var category = categoriesService.EnsureCategory(categoryName);
var joke = new Joke {
Category = category, Content = jokeContent
};
db.Jokes.Add(joke);
db.SaveChanges();
Console.WriteLine(i);
}
}
}
public static void Main()
{
var db = new ApplicationDbContext();
var repo = new DbRepository <JokeCategory>(db);
var categoriesService = new CategoriesService(repo);
var configuration = Configuration.Default.WithDefaultLoader();
var browsingContext = BrowsingContext.New(configuration);
for (int i = 1; i <= 10000; i++)
{
var url = $"http://vicove.com/vic-{i}";
var document = browsingContext.OpenAsync(url).Result;
var jokeContent = document.QuerySelector("#content_box .post-content").TextContent.Trim();
if (!string.IsNullOrWhiteSpace(jokeContent))
{
var categoryName = document.QuerySelector("#content_box .thecategory a").TextContent.Trim();
var category = categoriesService.EnsureCategory(categoryName);
var joke = new Joke {
Category = category, Content = jokeContent
};
db.Jokes.Add(joke);
db.SaveChanges();
Console.WriteLine(i);
}
}
}
public static void Craw(ApplicationDbContext db)
{
var repo = new DbRepository <JokeCategory>(db);
var categoriesService = new CategoriesService(repo);
var configuration = AngleSharp.Configuration.Default.WithDefaultLoader();
var browsingContext = BrowsingContext.New(configuration);
var jokes = new List <Joke>();
for (int i = 1; i <= 200; i++)
{
var url = string.Format("http://vicove.com/vic-{0}", i);
var document = browsingContext.OpenAsync(url).Result;
var jokeContent = document.QuerySelector("#content_box .post-content").TextContent.Trim();
if (!string.IsNullOrWhiteSpace(jokeContent))
{
var categoryName = document.QuerySelector("#content_box .thecategory a").TextContent.Trim();
var category = categoriesService.EnsureCategory(categoryName);
var joke = new Joke {
Category = category, Title = "Joke " + i, Content = jokeContent
};
jokes.Add(joke);
if (jokes.Count == 100)
{
break;
}
}
}
db.Jokes.AddOrUpdate(jokes.ToArray());
db.SaveChanges();
}
private static void TryAddJoke(string jokeContent, string categoryName)
{
try
{
var category = categoriesService.EnsureCategory(categoryName);
var joke = new Joke {
Category = category, Content = jokeContent
};
db.Jokes.Add(joke);
}
catch (Exception)
{
Console.WriteLine("Invalid joke model skipping this one");
}
}
static void Main()
{
// TODO: Add DI container an AutoMapper
var db = new ApplicationDbContext();
var dbRopository = new DbRepository <JokeCategory>(db);
var categoryesService = new CategoriesService(dbRopository);
// AngleSharp configurations
IConfiguration configuration = Configuration.Default.WithDefaultLoader();
IBrowsingContext browsingContex = BrowsingContext.New(configuration);
long countOfAddedJokes = 0;
for (int i = 1; i < 65000; i++)
{
string url = $"http://vicove.com/vic-{i}";
IDocument document = browsingContex.OpenAsync(url).Result;
string jokeContent = document.QuerySelector("#content_box .post-content p").TextContent.Trim();
string categoryName = document.QuerySelector("#content_box .thecategory a").TextContent.Trim();
bool isValidJoke = !string.IsNullOrWhiteSpace(jokeContent);
bool isValidCategory = !string.IsNullOrWhiteSpace(categoryName);
if (isValidJoke && isValidCategory)
{
countOfAddedJokes++;
JokeCategory category = categoryesService.EnsureCategory(categoryName);
Joke joke = new Joke
{
Content = jokeContent,
Category = category
};
db.Jokes.Add(joke);
db.SaveChanges();
if (countOfAddedJokes % 100 == 0)
{
Console.WriteLine($"Added {countOfAddedJokes} jokes");
}
}
}
Console.WriteLine();
Console.WriteLine($"--- DONE => {countOfAddedJokes} joke added");
}