public HomeController(ILoggerFactory loggerFactory, RedditImageDownloaderContext context)
{
_logger = loggerFactory?.CreateLogger <HomeController>() ?? throw new ArgumentNullException(nameof(loggerFactory));
_context = context ?? throw new ArgumentNullException(nameof(context));
_logger.LogDebug("Home Controller created");
}
private void GetFromSource(Source source)
{
_logger.LogDebug($"GetFromSource() | source[{source}]");
var result = Client.GetStreamAsync(source.FeedUrl).Result;
_logger.LogDebug("GetFromSource() | Got data from feed url");
using (var xmlReader = XmlReader.Create(result))
{
_logger.LogDebug("GetFromSource() | Created XML Reader, loading into syndication feed");
var feed = SyndicationFeed.Load(xmlReader);
if (feed != null)
{
_logger.LogDebug($"GetFromSource() | Created syndication feed, processing [{feed.Items.Count()}] items");
var options =
RedditImageDownloaderContext.BuildContextOptions(_loggerFactory, _connectionStrings.Default);
using (var context = new RedditImageDownloaderContext(_loggerFactory, options))
{
foreach (var item in feed.Items)
{
var entry = context.Entries.FirstOrDefault(x => x.PostId == item.Id && x.SourceId == source.Id);
if (entry != null)
{
_logger.LogWarning($"GetFromSource() | Item with Id[{item.Id}] exists, skipping");
continue;
}
if (item.Content is TextSyndicationContent text)
{
var imgSrc = ParseContent(text.Text);
if (string.IsNullOrWhiteSpace(imgSrc))
{
_logger.LogWarning("GetFromSource() | no img src, skipping");
continue;
}
_logger.LogDebug("GetFromSource() | creating entry and adding");
entry = new Entry {
SourceId = source.Id, PostId = item.Id, Url = imgSrc, Link = item.Links.FirstOrDefault()?.Uri?.AbsoluteUri ?? string.Empty
};
context.Entries.Add(entry);
context.SaveChanges();
}
else
{
_logger.LogWarning(
$"GetFromSource() | item.Content was not a TextSyndicationContent, it is [{item.Content.GetType()}]");
}
}
}
}
else
{
_logger.LogWarning("GetFromSource() | Error loading content into syndication feed");
}
}
}
public void AddSource(string name, string niceName, string url, string feed)
{
var options = RedditImageDownloaderContext.BuildContextOptions(_loggerFactory, _connectionStrings.Default);
using (var context = new RedditImageDownloaderContext(_loggerFactory, options))
{
var source = new Source
{
Name = name,
NiceName = niceName,
Url = url,
FeedUrl = feed
};
context.Sources.Add(source);
context.SaveChanges();
}
}
public void DownloadImages()
{
_logger.LogDebug("DownloadImages()");
CheckDirectories();
var options = RedditImageDownloaderContext.BuildContextOptions(_loggerFactory, _connectionStrings.Default);
using (var context = new RedditImageDownloaderContext(_loggerFactory, options))
{
var entries = context.Entries
.Include(e => e.Source)
.Where(e => e.Downloaded == null && e.Processed == EntityLiterals.No)
.ToList();
Parallel.ForEach(entries, DownloadImage);
context.SaveChanges();
}
}
public void ParseSources()
{
_logger.LogDebug("ParseSources()");
var options = RedditImageDownloaderContext.BuildContextOptions(_loggerFactory, _connectionStrings.Default);
#if DEBUG
using (var context = new RedditImageDownloaderContext(_loggerFactory, options))
{
var source01 = new Source {
Name = "ImaginaryTamriel", NiceName = "Imaginary Tamriel", Url = "https://www.reddit.com/r/ImaginaryTamriel/", FeedUrl = "https://www.reddit.com/r/ImaginaryTamriel/.rss"
};
var source02 = new Source {
Name = "ImaginaryKnights", NiceName = "Imaginary Knights", Url = "https://www.reddit.com/r/ImaginaryKnights/", FeedUrl = "https://www.reddit.com/r/ImaginaryKnights/.rss"
};
var source03 = new Source {
Name = "ImaginaryWarriors", NiceName = "Imaginary Warriors", Url = "https://www.reddit.com/r/ImaginaryWarriors/", FeedUrl = "https://www.reddit.com/r/ImaginaryWarriors/.rss"
};
var source04 = new Source {
Name = "ImaginaryBattlefields", NiceName = "Imaginary Battlefields", Url = "https://www.reddit.com/r/ImaginaryBattlefields/", FeedUrl = "https://www.reddit.com/r/ImaginaryBattlefields/.rss"
};
var source05 = new Source {
Name = "ImaginaryWizards", NiceName = "Imaginary Wizards", Url = "https://www.reddit.com/r/ImaginaryWizards/", FeedUrl = "https://www.reddit.com/r/ImaginaryWizards/.rss"
};
var source06 = new Source {
Name = "ImaginaryNobles", NiceName = "Imaginary Nobles", Url = "https://www.reddit.com/r/ImaginaryNobles/", FeedUrl = "https://www.reddit.com/r/ImaginaryNobles/.rss"
};
var source07 = new Source {
Name = "ImaginaryRuins", NiceName = "Imaginary Ruins", Url = "https://www.reddit.com/r/ImaginaryRuins/", FeedUrl = "https://www.reddit.com/r/ImaginaryRuins/.rss"
};
var source08 = new Source {
Name = "ImaginaryHorrors", NiceName = "Imaginary Horrors", Url = "https://www.reddit.com/r/ImaginaryHorrors/", FeedUrl = "https://www.reddit.com/r/ImaginaryHorrors/.rss"
};
var source09 = new Source {
Name = "ImaginaryVikings", NiceName = "Imaginary Vikings", Url = "https://www.reddit.com/r/ImaginaryVikings/", FeedUrl = "https://www.reddit.com/r/ImaginaryVikings/.rss"
};
var source10 = new Source {
Name = "ImaginaryCosmere", NiceName = "Imaginary Cosmere", Url = "https://www.reddit.com/r/imaginarycosmere/", FeedUrl = "https://www.reddit.com/r/imaginarycosmere/.rss"
};
var source11 = new Source {
Name = "ImaginaryBeasts", NiceName = "Imaginary Beasts", Url = "https://www.reddit.com/r/ImaginaryBeasts/", FeedUrl = "https://www.reddit.com/r/ImaginaryBeasts/.rss"
};
var source12 = new Source {
Name = "ImaginaryTemples", NiceName = "Imaginary Temples", Url = "https://www.reddit.com/r/ImaginaryTemples/", FeedUrl = "https://www.reddit.com/r/ImaginaryTemples/.rss"
};
var source13 = new Source {
Name = "ImaginaryTechnology", NiceName = "Imaginary Technology", Url = "https://www.reddit.com/r/ImaginaryTechnology/", FeedUrl = "https://www.reddit.com/r/ImaginaryTechnology/.rss"
};
var source14 = new Source {
Name = "ImaginaryAdventurers", NiceName = "Imaginary Adventurers", Url = "https://www.reddit.com/r/ImaginaryAdventurers/", FeedUrl = "https://www.reddit.com/r/ImaginaryAdventurers/.rss"
};
var source15 = new Source {
Name = "ImaginaryCastles", NiceName = "Imaginary Castles", Url = "https://www.reddit.com/r/ImaginaryCastles/", FeedUrl = "https://www.reddit.com/r/ImaginaryCastles/.rss"
};
var source16 = new Source {
Name = "ImaginaryWorlds", NiceName = "Imaginary Worlds", Url = "https://www.reddit.com/r/ImaginaryWorlds/", FeedUrl = "https://www.reddit.com/r/ImaginaryWorlds/.rss"
};
var source17 = new Source {
Name = "ImaginaryMonsterGirls", NiceName = "Imaginary Monster Girls", Url = "https://www.reddit.com/r/ImaginaryMonsterGirls/", FeedUrl = "https://www.reddit.com/r/ImaginaryMonsterGirls/.rss"
};
var source18 = new Source {
Name = "Dragons", NiceName = "Dragons", Url = "https://www.reddit.com/r/Dragons/", FeedUrl = "https://www.reddit.com/r/Dragons/.rss"
};
var source19 = new Source {
Name = "ImaginaryTaverns", NiceName = "Imaginary Taverns", Url = "https://www.reddit.com/r/ImaginaryTaverns/", FeedUrl = "https://www.reddit.com/r/ImaginaryTaverns/.rss"
};
var source20 = new Source {
Name = "ImaginaryLeviathans", NiceName = "Imaginary Leviathans", Url = "https://www.reddit.com/r/ImaginaryLeviathans/", FeedUrl = "https://www.reddit.com/r/ImaginaryLeviathans/.rss"
};
var source21 = new Source {
Name = "ImaginaryDerelicts", NiceName = "Imaginary Derelicts", Url = "https://www.reddit.com/r/ImaginaryDerelicts/", FeedUrl = "https://www.reddit.com/r/ImaginaryDerelicts/.rss"
};
var source22 = new Source {
Name = "ImaginaryStarships", NiceName = "Imaginary Starships", Url = "https://www.reddit.com/r/ImaginaryStarships/", FeedUrl = "https://www.reddit.com/r/ImaginaryStarships/.rss"
};
var source23 = new Source {
Name = "ReasonableFantasy", NiceName = "Reasonable Fantasy", Url = "https://www.reddit.com/r/ReasonableFantasy/", FeedUrl = "https://www.reddit.com/r/ReasonableFantasy/.rss"
};
var source24 = new Source {
Name = "ImaginaryJedi", NiceName = "Imaginary Jedi", Url = "https://www.reddit.com/r/ImaginaryJedi/", FeedUrl = "https://www.reddit.com/r/ImaginaryJedi/.rss"
};
var source25 = new Source {
Name = "ImaginaryFutureWar", NiceName = "Imaginary Future War", Url = "https://www.reddit.com/r/ImaginaryFutureWar/", FeedUrl = "https://www.reddit.com/r/ImaginaryFutureWar/.rss"
};
var source26 = new Source {
Name = "ImaginaryWastelands", NiceName = "Imaginary Wastelands", Url = "https://www.reddit.com/r/ImaginaryWastelands/", FeedUrl = "https://www.reddit.com/r/ImaginaryWastelands/.rss"
};
var source27 = new Source {
Name = "SpecArt", NiceName = "Spec Art", Url = "https://www.reddit.com/r/SpecArt/", FeedUrl = "https://www.reddit.com/r/SpecArt/.rss"
};
var source28 = new Source {
Name = "ImaginaryHybrids", NiceName = "Imaginary Hybrids", Url = "https://www.reddit.com/r/ImaginaryHybrids/", FeedUrl = "https://www.reddit.com/r/ImaginaryHybrids/.rss"
};
var source29 = new Source {
Name = "ImaginaryPolice", NiceName = "Imaginary Police", Url = "https://www.reddit.com/r/ImaginaryPolice/", FeedUrl = "https://www.reddit.com/r/ImaginaryPolice/.rss"
};
var source30 = new Source {
Name = "ImaginaryMythology", NiceName = "Imaginary Mythology", Url = "https://www.reddit.com/r/ImaginaryMythology/", FeedUrl = "https://www.reddit.com/r/ImaginaryMythology/.rss"
};
var existingSource = context.Sources.FirstOrDefault(s => s.Name == source01.Name);
if (existingSource == null)
{
context.Sources.Add(source01);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source02.Name);
if (existingSource == null)
{
context.Sources.Add(source02);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source03.Name);
if (existingSource == null)
{
context.Sources.Add(source03);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source04.Name);
if (existingSource == null)
{
context.Sources.Add(source04);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source05.Name);
if (existingSource == null)
{
context.Sources.Add(source05);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source06.Name);
if (existingSource == null)
{
context.Sources.Add(source06);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source07.Name);
if (existingSource == null)
{
context.Sources.Add(source07);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source08.Name);
if (existingSource == null)
{
context.Sources.Add(source08);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source09.Name);
if (existingSource == null)
{
context.Sources.Add(source09);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source10.Name);
if (existingSource == null)
{
context.Sources.Add(source10);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source11.Name);
if (existingSource == null)
{
context.Sources.Add(source11);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source12.Name);
if (existingSource == null)
{
context.Sources.Add(source12);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source13.Name);
if (existingSource == null)
{
context.Sources.Add(source13);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source14.Name);
if (existingSource == null)
{
context.Sources.Add(source14);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source15.Name);
if (existingSource == null)
{
context.Sources.Add(source15);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source16.Name);
if (existingSource == null)
{
context.Sources.Add(source16);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source17.Name);
if (existingSource == null)
{
context.Sources.Add(source17);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source18.Name);
if (existingSource == null)
{
context.Sources.Add(source18);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source19.Name);
if (existingSource == null)
{
context.Sources.Add(source19);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source20.Name);
if (existingSource == null)
{
context.Sources.Add(source20);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source21.Name);
if (existingSource == null)
{
context.Sources.Add(source21);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source22.Name);
if (existingSource == null)
{
context.Sources.Add(source22);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source23.Name);
if (existingSource == null)
{
context.Sources.Add(source23);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source24.Name);
if (existingSource == null)
{
context.Sources.Add(source24);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source25.Name);
if (existingSource == null)
{
context.Sources.Add(source25);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source26.Name);
if (existingSource == null)
{
context.Sources.Add(source26);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source27.Name);
if (existingSource == null)
{
context.Sources.Add(source27);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source28.Name);
if (existingSource == null)
{
context.Sources.Add(source28);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source29.Name);
if (existingSource == null)
{
context.Sources.Add(source29);
}
existingSource = context.Sources.FirstOrDefault(s => s.Name == source30.Name);
if (existingSource == null)
{
context.Sources.Add(source30);
}
context.SaveChanges();
}
#endif
using (var context = new RedditImageDownloaderContext(_loggerFactory, options))
{
var sources = context.Sources
.Where(s => s.Active == EntityLiterals.Yes && s.Deleted == EntityLiterals.No)
.ToList();
_logger.LogDebug($"ParseSources() | Processing [{sources.Count}] sources");
Parallel.ForEach(sources, GetFromSource);
}
}
