protected AbstractItem(AbstractSource source, string title, string link, int?id = null, bool isNew = true, DateTime?addedDate = null)
{
this.source = source;
/*source.PropertyChanged += (e, args) =>
* {
* switch (args.PropertyName)
* {
* case "UpdatesColorBrush":
* RaiseEvent("UpdatesBrush");
* break;
* }
* };*/
this.SourceId = source.ID.Value;
this.SourceName = source.SourceName;
this.Provider = source.ProviderID;
this.Name = title;
this.ActionContent = link;
this.AddedDate = !addedDate.HasValue? DateTime.Now : addedDate.Value;
this.ID = id;
this.New = isNew;
}
private static List <JobItem> ParseItems(HtmlNodeCollection items, AbstractSource source)
{
List <JobItem> list = new List <JobItem>();
foreach (var i in items)
{
string location, title, link;
var linkNode = i.SelectSingleNode(".//div[@class='jobTitleContainer']/a");
title = linkNode.InnerText;
link = linkNode.Attributes["href"].Value;
if (link.Contains('?'))
{
link = link.Substring(0, link.IndexOf('?'));
}
string id = GetId(link);
location = i.SelectSingleNode(".//div[@class='jobLocationSingleLine']/a").InnerText;
string name = String.Format("{0} ({1}, {2})", title, location, id);
var item = new JobItem(source, name, link);
list.Add(item);
}
return(list);
}
protected override void DoUpdateSource(AbstractSource source)
{
var oldSource = Sources.First(x => x.ID == source.ID);
source.CopyTo(oldSource);
SaveSources();
}
public static List <IDataItem> GetNewItems(AbstractSource source)
{
if (source.GetMetaDataValue(JobsProvider.META_SOURCE).ToString() == Jobs.JobsProvider.SourceNames.Monster.ToString())
{
int Pages;
try
{
Pages = int.Parse(source.GetMetaDataValue(JobsProvider.META_PAGES).ToString());
}
catch
{
Pages = int.Parse(DefaultPages.ToString());
}
ConcurrentBag <IDataItem> items = new ConcurrentBag <IDataItem>();
int pageCounter = 0;
object counterLock = new object();
try
{
for (int i = 1; i <= Pages; i++)
{
string url = source.GetMetaDataValue(JobsProvider.META_URL) + "&pg=" + i;
var pageItems = GetItemsOnPage(url, source);
FilterUtil.FilterAndAdd(pageItems, items, pageCounter, counterLock, ((ISource2)source).Services);
}
}
catch (Exception e)
{
//Log
}
while (true)
{
lock (counterLock)
{
if (pageCounter == 0)
{
break;
}
}
Thread.Sleep(2000);
}
return(items.ToList());
}
else
{
return(null);
}
}
private string CreateBaseUrl(AbstractSource source)
{
string query = source.GetMetaDataValue(META_QUERY);
string location = source.GetMetaDataValue(META_LOCATION);
string range = source.GetMetaDataValue(META_RANGE);
string baseUrl = "https://www.dice.com/jobs/q-{0}-sort-date-l-{1}-radius-{2}";
string url = String.Format(baseUrl, query, location, range);
return(url);
}
protected override List <AbstractItem> GetNewItems(AbstractSource source)
{
List <AbstractItem> items = new List <AbstractItem>();
WebClient wc = new WebClient();
for (int p = 1; p <= MaxPages; p++)
{
string url;
string page;
switch (source.GetMetaDataValue(Site))
{
case JapanType:
url = JapanSite;
break;
case ChinaType:
default:
url = ChinaSite;
break;
}
url += "/page/" + p;
page = wc.DownloadString(url);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(page);
var itemNodes = doc.DocumentNode.SelectNodes("//div[@id='archive']");
itemNodes = itemNodes[0].SelectNodes(".//li");
foreach (var node in itemNodes)
{
try
{
var a = node.SelectSingleNode(".//div[@class='excerpt-post-title']/a");
string name = WebUtility.HtmlDecode(a.InnerText);
string link = a.Attributes["href"].Value;
var post = new Post(source, name, link);
items.Add(post);
}
catch (Exception e)
{
}
}
}
return(items);
}
protected override List <AbstractItem> GetNewItems(AbstractSource source)
{
if (source.ProviderID == PROVIDER)
{
int Pages;
try
{
Pages = int.Parse(source.GetMetaDataValue(META_PAGES));
}
catch
{
Pages = int.Parse(DefaultPages);
}
int Size;
try
{
Size = int.Parse(source.GetMetaDataValue(META_PAGE_SIZE));
}
catch
{
Size = int.Parse(DefaultPageSize);
}
string QUERY_BASE = CreateBaseUrl(source);
List <AbstractItem> items = new List <AbstractItem>();
try
{
for (int i = 1; i <= Pages; i++)
{
string url = QUERY_BASE + "-startPage-" + i + "-limit-" + Size + "-jobs.html";
items.AddRange(GetItemsOnPage(url, source));
}
}
catch (Exception e)
{
//Log
}
return(items);
}
else
{
return(null);
}
}
protected override List <AbstractItem> GetNewItems(AbstractSource source)
{
if (source.ProviderID == PROVIDER)
{
List <string> exclusions = GetExclusions(source);
for (int i = 0; i < exclusions.Count; i++)
{
exclusions[i] = exclusions[i].ToLower();
}
List <AbstractItem> shows = new List <AbstractItem>();
WebClient wc = new WebClient();
string page = wc.DownloadString(BASE_URL);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(page);
foreach (var table in doc.DocumentNode.SelectNodes("//table"))
{
if (table.InnerText.Contains("(Sub)")) //Find Table
{
var items = table.SelectNodes(".//li");
foreach (var i in items)
{
if (i.InnerText.Contains("(Sub)")) //Get Subbed only
{
var link = i.SelectSingleNode(".//a");
string url = link.Attributes["href"].Value;
string name = link.InnerText;
name = WebUtility.HtmlDecode(name);
if (!IsExcluded(name, exclusions))
{
Show show = new Show(source, name, url);
shows.Add(show);
}
}
}
}
}
return(shows);
}
else
{
return(null);
}
}
internal static SmackSource CreateFrom(AbstractSource source)
{
if (source is SmackSource)
{
return((SmackSource)source);
}
SmackSource js = new SmackSource(source.SourceName);
js.SetMetaData(source.GetMetaData());
js.SetID(source.ID.Value);
return(js);
}
private List <string> GetExclusions(AbstractSource source)
{
List <string> exclude = new List <string>();
string exclusionString = source.GetMetaDataValue(META_EXCLUSIONS);
if (!String.IsNullOrEmpty(exclusionString))
{
exclude.AddRange(from s in exclusionString.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
select s.Trim());
}
return(exclude);
}
private string Serialize(AbstractSource src)
{
string meta = String.Join(MetaDelim + "",
from kv in src.GetMetaData() select kv.Key + "=" + kv.Value);
string data = String.Join(FieldDelim + "",
src.ID.Value,
src.ProviderID,
src.SourceName,
src.Url,
meta);
return(data);
}
protected override List <AbstractItem> GetNewItems(AbstractSource source)
{
V1.AbstractSource oldSource = ConvertToOldFormat(source);
var items = provider.SupportGetNewItems(oldSource);
List <AbstractItem> newItems = new List <AbstractItem>();
foreach (var i in items)
{
newItems.Add(new ItemAdapter(i, source));
}
return(newItems);
}
private V1.AbstractSource ConvertToOldFormat(AbstractSource source)
{
if (source is SourceAdapter)
{
return(((SourceAdapter)source).GetOldSource());
}
else
{
V1.AbstractSource src = new V1.GenericSource(source.SourceName, source.ProviderID);
src.SetID(source.ID.Value);
var md = ConvertToOldFormat(source.GetMetaData().Values);
src.SetMetaData(md);
return(src);
}
}
private List <BasicItem> GetItemsOnPage(string url, AbstractSource source)
{
List <BasicItem> items = new List <BasicItem>();
WebClient wc = new WebClient();
wc.UseDefaultCredentials = true;
string page = wc.DownloadString(url);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(page);
var resultsNode = doc.DocumentNode.SelectSingleNode("//div[@id='serp']");
var itemNodes = resultsNode.SelectNodes("div");
foreach (var i in itemNodes)
{
string location, posted, title, link;
var linkNode = i.SelectSingleNode(".//a[@class='dice-btn-link']");
title = linkNode.InnerText.Trim();
link = linkNode.Attributes["href"].Value;
if (link.Contains('?'))
{
link = link.Substring(0, link.IndexOf('?'));
}
string id = GetId(link);
location = i.SelectSingleNode(".//li[@class='location']").InnerText.Trim();
posted = i.SelectSingleNode(".//li[@class='posted']").InnerText.Trim();
string name = String.Format("{0} ({1}, {2})", title, location, id);
var item = new BasicItem(source, name, link);
items.Add(item);
}
return(items);
}
protected override List <AbstractItem> GetNewItems(AbstractSource source)
{
List <AbstractItem> items = new List <AbstractItem>();
WebClient wc = new WebClient();
wc.UseDefaultCredentials = true;
int maxPages = int.Parse(source.GetMetaDataValue(META_MAX_PAGES));
for (int p = 1; p <= maxPages; p++)
{
string page;
string url = BASE_URL + "?page=" + p;
page = wc.DownloadString(url);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(page);
var resultsContainer = doc.DocumentNode.SelectSingleNode("//div[@id='browse-results']");
var talks = resultsContainer.SelectNodes(".//div[@class='talk-link']");
foreach (var talk in talks)
{
var innerContainer = talk.SelectSingleNode(".//div[@class='media__message']");
var link = innerContainer.SelectSingleNode(".//a");
if (link != null)
{
string name = link.InnerText.Trim();
string link_url = link.Attributes["href"].Value;
link_url = "https://www.ted.com/" + link_url;
BasicItem item = new BasicItem(source, name, link_url);
items.Add(item);
}
}
}
return(items);
}
private static List <JobItem> GetItemsOnPage(string url, AbstractSource source)
{
List <JobItem> items = new List <JobItem>();
WebClient wc = new WebClient();
string page = wc.DownloadString(url);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(page);
var resultsNode = doc.DocumentNode.SelectSingleNode("//table[@class='listingsTable']");
var odds = resultsNode.SelectNodes(".//tr[@class='odd']");
var evens = resultsNode.SelectNodes(".//tr[@class='even']");
items.AddRange(ParseItems(odds, source));
items.AddRange(ParseItems(evens, source));
return(items);
}
protected override List <AbstractItem> GetNewItems(AbstractSource source)
{
if (source.ProviderID == PROVIDER)
{
int Pages;
try
{
Pages = int.Parse(source.GetMetaDataValue(META_PAGES));
}
catch
{
Pages = int.Parse(DefaultPages);
}
List <AbstractItem> items = new List <AbstractItem>();
try
{
for (int i = 1; i <= Pages; i++)
{
string url = source.Url + "&pg=" + i;
items.AddRange(GetItemsOnPage(url, source));
}
}
catch (Exception e)
{
//Log
}
return(items);
}
else
{
return(null);
}
}
public static AbstractItem CreateGenericItem(int id, AbstractSource source, string title, string meta, bool isNew,
DateTime addedDate)
{
return(new GenericItem(id, source, title, meta, isNew, addedDate));
}
protected override void DoInsertSource(AbstractSource source)
{
source.SetID(nextSourceId++);
SaveSources();
}
public GenericItem(int id, AbstractSource source, string title, string meta, bool isNew,
DateTime addedDate)
: base(source, title, meta, id, isNew, addedDate)
{
}
public override void RemoveFromDataStore(AbstractSource source)
{
SaveSources();
}
public ItemAdapter(Extensions.V1.AbstractItem i, AbstractSource src) : base(src, i.Name, i.ActionContent, i.ID, i.New, i.AddedDate)
{
this.i = i;
}
protected override List <AbstractItem> GetNewItems(AbstractSource source)
{
return(new List <AbstractItem>());
}
public Show(AbstractSource source, string title, string link, int?id = null, bool isNew = true, DateTime?addedDate = null)
: base(source, title, link, id, isNew, addedDate)
{
}
protected override List <AbstractItem> GetNewItems(AbstractSource source)
{
SmackSource js = SmackSource.CreateFrom(source);
return(GetNewItems(js));
}
