//Updated July 15th
public static void WriteShows()
{
try
{
HtmlNodeCollection collection = DownloadMgmtUtils.GetHtmlNodeCollection
(ConfigurationManager.AppSettings["show_url"] +
ConfigurationManager.AppSettings["show_url_list_postfix"], "//a");
MongoCollection mongoCollection = MongoUtils.GetMongoCollection
(
@"mongodb://" + ConfigurationManager.AppSettings["mongoHost"] + @"/",
ConfigurationManager.AppSettings["port"],
ConfigurationManager.AppSettings["db"],
ConfigurationManager.AppSettings["show_collection"]
);
foreach (
HtmlNode link in
collection.Where(link => !ConfigurationManager.AppSettings["show_innerhtml_excl"].Split(',')
.Any(s => link.InnerHtml.Contains(s))).Where(link => link.Attributes[0].Value != null))
{
mongoCollection.Insert(new TvShow()
{
Name = link.InnerHtml, Path = link.Attributes[0].Value
});
}
}
catch (Exception e)
{
Log.AppendToLog("Error : FATAL Write Show Issue : " + e, ConfigurationManager.AppSettings["log_file"]);
}
}
//Updated July 30th - Combined with WriteEpisodes
public static void RefreshEpisodes()
{
try
{
MongoCollection mongoCollection = MongoUtils.GetMongoCollection
(
@"mongodb://" + ConfigurationManager.AppSettings["mongoHost"] + @"/",
ConfigurationManager.AppSettings["port"],
ConfigurationManager.AppSettings["db"],
ConfigurationManager.AppSettings["show_collection"]
);
foreach (var show in mongoCollection.FindAllAs <TvShow>().ToList <TvShow>())
{
if (!ConfigurationManager.AppSettings["show_excl"].Split(',').Any(s => show.Name.Contains(s)))
{
HtmlNodeCollection collection =
DownloadMgmtUtils.GetHtmlNodeCollection(
ConfigurationManager.AppSettings["show_url"] + show.Path, "//a");
if (collection != null)
{
foreach (var link in collection.Where(link =>
!ConfigurationManager.AppSettings["episode_innerhtml_excl"].Split(',')
.Any(s => link.InnerHtml.Contains(s)) &&
link.Attributes.Count > 0 && !link.Attributes[0].Value.Contains("tvnews") &&
link.Attributes[HrefIndex(link)].Value.Contains("/ep")))
{
UpdateShowFromHtml(show, link.InnerHtml, collection, collection.IndexOf(link));
}
}
mongoCollection.Save(show);
}
}
}
catch (Exception e)
{
Log.AppendToLog("Error : FATAL Refresh Episodes Issue : " + e,
ConfigurationManager.AppSettings["log_file"]);
}
}
//Updated July 30th
public static void RefreshShows()
{
try
{
HtmlNodeCollection collection = DownloadMgmtUtils.GetHtmlNodeCollection
(ConfigurationManager.AppSettings["show_url"] +
ConfigurationManager.AppSettings["show_url_list_postfix"], "//a");
MongoCollection mongoCollection = MongoUtils.GetMongoCollection
(
@"mongodb://" + ConfigurationManager.AppSettings["mongoHost"] + @"/",
ConfigurationManager.AppSettings["port"],
ConfigurationManager.AppSettings["db"],
ConfigurationManager.AppSettings["show_collection"]
);
foreach (HtmlNode link in from link in collection
where
!ConfigurationManager.AppSettings["show_innerhtml_excl"].Split(',')
.Any(s => link.InnerHtml.Contains(s))
let tvShowList =
mongoCollection.FindAs <TvShow>(Query.EQ("Name", link.InnerHtml)).ToList <TvShow>()
where tvShowList.Count == 0
select link)
{
mongoCollection.Insert(new TvShow()
{
Name = link.InnerHtml, Path = link.Attributes[0].Value
});
}
}
catch (Exception e)
{
Log.AppendToLog("Error : FATAL Refresh Show Issue : " + e, ConfigurationManager.AppSettings["log_file"]);
}
}
//Updated August 19th
public static List <GetReturnObject> GetSearchImdbMovies(string Query)
{
List <GetReturnObject> resultList = new List <GetReturnObject>();
try
{
ImdbObject obj = JsonSerializer.DeserializeFromString <ImdbObject>
(DownloadMgmtUtils.GetImdbJsonQuery("http://www.imdb.com/xml/find?json=1&nr=0&tt=on&q=" + Query));
List <ImdbElement> initialResultsList = new List <ImdbElement>();
if (obj.title_exact == null)
{
if (obj.title_popular == null)
{
if (obj.title_substring == null)
{
initialResultsList.Add(new ImdbElement()
{
Title = "Error", Description = " not found"
});
}
else
{
initialResultsList = obj.title_substring;
}
}
else
if (obj.title_substring == null)
{
initialResultsList = obj.title_popular;
}
else
{
initialResultsList = obj.title_substring.Union(obj.title_popular).ToList();
}
}
else
if (obj.title_popular == null)
{
if (obj.title_substring == null)
{
initialResultsList = obj.title_exact;
}
else
{
initialResultsList = obj.title_substring.Union(obj.title_exact).ToList();
}
}
else
if (obj.title_substring == null)
{
initialResultsList = obj.title_popular.Union(obj.title_exact).ToList();
}
else
{
initialResultsList = obj.title_substring.Union(obj.title_exact.Union(obj.title_popular)).ToList();
}
foreach (var e in initialResultsList)
{
e.Description = e.Description.Split(' ')[0].Split(',')[0];
e.Title = e.Title.Split(',')[0];
}
List <ImdbElement> filteredResultList = initialResultsList.Where(e => !ConfigurationManager.AppSettings["imdb_search_excl"].Split(',').Any(s => e.Description.Contains(s))).ToList();
filteredResultList.Sort(delegate(ImdbElement c1, ImdbElement c2) { return(c2.Description.CompareTo(c1.Description)); });
foreach (var r in filteredResultList)
{
resultList.Add(new GetReturnObject()
{
Movie = true,
Show = false,
Name = r.Title,
ImdbCode = r.Id,
Year = Convert.ToInt32(r.Description)
});
}
}
catch (Exception e)
{ Log.AppendToLog(" FATAL search IMDB movies error. " + e, ConfigurationManager.AppSettings["log_file"]); }
return(resultList);
}
//Updated July 26th
public static void WriteImdbMovies()
{
bool end = false, firstMovie = true;
int results = 252, incrementer = 1;
try
{
while (!end)
{
if (incrementer < results)
{
HtmlNodeCollection collection =
DownloadMgmtUtils.GetHtmlNodeCollection(firstMovie ? ConfigurationManager.AppSettings["imdb_root"] : ImdbUrlIncrementor(incrementer), "//a");
if (firstMovie)
{
HtmlNodeCollection divCollection = DownloadMgmtUtils.GetHtmlNodeCollection(ConfigurationManager.AppSettings["imdb_root"], "//div");
var ppp = divCollection.Where(d => d.Attributes.Count > 0 && d.Attributes[0].Value == "left").ToList();
results = Convert.ToInt32(ppp[0].InnerText.Replace("1-250 of", "")
.Replace("\n", "")
.Replace(",", "")
.Replace("titles.", ""));
firstMovie = false;
}
foreach (HtmlNode n in collection)
{
if (n.Attributes.Count != 1 || !n.Attributes[0].Value.Contains("/title/"))
{
continue;
}
MongoCollection mongoCollection = MongoUtils.GetMongoCollection
(
@"mongodb://" + ConfigurationManager.AppSettings["mongoHost"] + @"/",
ConfigurationManager.AppSettings["port"],
ConfigurationManager.AppSettings["db"],
ConfigurationManager.AppSettings["movie_collection"]
);
var mongoMovie = mongoCollection.FindAs <Movie>(Query.EQ("ImdbCode", n.Attributes[0].Value.Replace("/title/", "").Replace("/", ""))).ToList <Movie>();
//Check to see if the movie already exists
if (mongoMovie.Count == 0)
{
mongoCollection.Insert(new Movie()
{
ImdbCode = n.Attributes[0].Value.Replace("/title/", "").Replace("/", ""),
ImdbTitle = n.InnerText,
Urls = new List <string>()
{
"http://www.imdb.com" + n.Attributes[0].Value
},
Downloaded = false
});
}
else
{
mongoMovie[0].ImdbTitle = n.Attributes[0].Value.Replace("/title/", "").Replace("/", "");
mongoMovie[0].Urls.Add("http://www.imdb.com" + n.Attributes[0].Value);
mongoCollection.Save(mongoMovie[0]);
}
}
incrementer = incrementer + 250;
}
else
{
end = true;
}
}
}
catch (Exception e)
{ Log.AppendToLog("Error : FATAL write from IMDB. " + e, ConfigurationManager.AppSettings["log_file"]); }
}
//Updated February 15th - updated YTS logic
public static void WriteYtsMovieData()
{
int setNumber = 1;
int movies = 1000000;
try
{
MongoCollection mongoCollection = MongoUtils.GetMongoCollection
(
@"mongodb://" + ConfigurationManager.AppSettings["mongoHost"] + @"/",
ConfigurationManager.AppSettings["port"],
ConfigurationManager.AppSettings["db"],
ConfigurationManager.AppSettings["movie_collection"]
);
while (movies > setNumber * 50)
{
var results = DownloadMgmtUtils.GetJsonObject(YtsUrlIncrementor(setNumber));
if (setNumber == 1)
{
movies = results.movie_count;
}
foreach (var m in results.movies)
{
//Check to see if the movie exists in the collection
var mongoMovie = mongoCollection.FindAs <Movie>(Query.EQ("ImdbCode", m.imdb_code)).ToList <Movie>();
if (mongoMovie.Count == 0 && m.torrents != null)
{
var downloadDetails = new List <DownloadDetails>();
for (int i = 0; i < m.torrents.Count; i++)
{
downloadDetails.Add(
new DownloadDetails()
{
Size = m.torrents[i].size,
SizeBytes = m.torrents[i].size_bytes,
TorrentUrl = m.torrents[i].url,
Seeds = Convert.ToInt16(m.torrents[i].seeds),
Quality = m.torrents[i].quality
}
);
}
mongoCollection.Insert(new Movie()
{
YtsMovieTitle = m.title.Replace(":", ""),
ImdbCode = m.imdb_code,
CoverImg = m.background_image,
Genre = m.genres[0],
Year = Convert.ToInt16(m.year),
Urls = new List <string>()
{
m.url
},
DownloadLogistics = downloadDetails
});
}
//if movie already exists, add/overwrite detail
else if (m.torrents != null)
{ //ILASM!
var downloadDetails = new List <DownloadDetails>();
for (int i = 0; i < m.torrents.Count; i++)
{
downloadDetails.Add(
new DownloadDetails()
{
Size = m.torrents[i].size,
SizeBytes = m.torrents[i].size_bytes,
TorrentUrl = m.torrents[i].url,
Seeds = Convert.ToInt16(m.torrents[i].seeds),
Quality = m.torrents[i].quality
}
);
}
mongoMovie[0].YtsMovieTitle = m.title.Replace(":", "");
mongoMovie[0].ImdbTitle = mongoMovie[0].ImdbTitle;
mongoMovie[0].CoverImg = m.background_image;
mongoMovie[0].Genre = m.genres[0];
mongoMovie[0].Year = Convert.ToInt16(m.year);
mongoMovie[0].DownloadLogistics = downloadDetails;
if (!mongoMovie[0].Urls.Contains(m.url))
{
mongoMovie[0].Urls.Add(m.url);
}
mongoCollection.Save(mongoMovie[0]);
}
}
setNumber++;
}
}
catch (Exception e)
{ Log.AppendToLog("Error : FATAL write from YTS. " + e, ConfigurationManager.AppSettings["log_file"]); }
}
