public void Net_UrlInfo_ToString()
{
var MyRoot = "http://testURL";
var MyController = "MyController";
var MyAction = "MyAction";
var TestItem = new UrlInfo(MyRoot, MyController, MyAction);
// Check formatting
Assert.IsTrue(TestItem.ToString().ToLowerInvariant() == String.Format("{0}:80/{1}/{2}", MyRoot, MyController, MyAction).ToLowerInvariant());
}
public void test()
{
Uri uri = new Uri("http://*****:*****@www.wojilu.net/myapp/Photo/1984?title=eee#top");
UrlInfo u = new UrlInfo(uri, "/myapp/", "myPathInfo");
Console.WriteLine("Scheme=>" + u.Scheme);
Console.WriteLine("UserName=>" + u.UserName);
Console.WriteLine("Password=>" + u.Password);
Console.WriteLine("Host=>" + u.Host);
Console.WriteLine("Port=>" + u.Port);
Console.WriteLine("Path=>" + u.Path);
Console.WriteLine("PathAndQuery=>" + u.PathAndQuery);
Console.WriteLine("PathInfo=>" + u.PathInfo);
Console.WriteLine("AppPath=>" + u.AppPath);
Console.WriteLine("PathAndQueryWithouApp=>" + u.PathAndQueryWithouApp);
Console.WriteLine("Query=>" + u.Query);
Console.WriteLine("Fragment=>" + u.Fragment);
Console.WriteLine("SiteUrl=>" + u.SiteUrl);
Console.WriteLine("SiteAndAppPath=>" + u.SiteAndAppPath);
Console.WriteLine("ToString=>" + u.ToString());
/*
* Scheme=>http
* UserName=>zhangsan
* Password=>123
* Host=>www.wojilu.net
* Port=>80
* Path=>/myapp/Photo/1984
* PathAndQuery=>/myapp/Photo/1984?title=eee
* PathInfo=>myPathInfo
* AppPath=>/myapp/
* PathAndQueryWithouApp=>/Photo/1984?title=eee
* Query=>?title=eee
* Fragment=>#top
* SiteUrl=>http://zhangsan:[email protected]
* SiteAndAppPath=>http://zhangsan:[email protected]/myapp/
* ToString=>http://zhangsan:[email protected]/myapp/Photo/1984?title=eee#top
*
*/
}
//TODO: Create a new object Report and insert it into DB
public void ScrapeNewReleasedMovie()
{
_logger?.LogInformation("************** Start Scraping New Released Movies Job - {Date} **************", DateTime.Now.ToString("u", DateTimeFormatInfo.InvariantInfo));
UrlInfo urlInfo = new UrlInfo()
{
EntryType = JavlibEntryType.NewRelease
};
int pageCount = GetPageCount(urlInfo);
if (pageCount > 0)
{
_logger?.LogInformation($"Found {pageCount} pages. Now scanning movies on each page");
for (int currentPage = 1; currentPage <= pageCount; currentPage++)
{
List <Movie> lstMovieCurrentPage = ScanPageList(new UrlInfo {
EntryType = JavlibEntryType.NewRelease, Page = currentPage
}).GroupBy(x => x.Url).Select(x => x.First()).ToList();
if (lstMovieCurrentPage.Count > 0)
{
_logger?.LogInformation("Treating {pageCount} movies in page {currentPage}", lstMovieCurrentPage.Count, currentPage);
foreach (Movie movie in lstMovieCurrentPage.GroupBy(x => x.Number.ToUpper()).Select(x => x.First()))
{
ScanMovieDetails(new UrlInfo()
{
EntryType = JavlibEntryType.Movie, ExactUrl = movie.Url
}, movie);
_movieService.UpdateStatus(movie, MovieStatus.Scanned);
if (lstMovieCurrentPage.FindAll(c => c.Number == movie.Number).Count > 1)
{
movie.FavLevel = JavlibFavLevel.DlChineseSub;
}
_movieService.SaveMovie(movie, true);
}
}
}
}
else
{
_logger?.LogWarning("Nothing found when scraping new released movie. UrlInfo: {0}", urlInfo.ToString());
}
_logger?.LogInformation("************** Scraping New Released Movies Job End - {Date} **************", DateTime.Now.ToString("u", DateTimeFormatInfo.InvariantInfo));
}
public void Net_UrlInfo()
{
var TestItem = new UrlInfo("http://test");
Assert.IsTrue(TestItem.ToString() == "http://test:80");
}
//TDOO: Update to exclude multiple actor, vr, 4h and over
public void ScanMovieDetails(UrlInfo urlInfo, Movie movie)
{
try
{
string requestUrl = GetJavLibraryUrl(urlInfo);
HtmlDocument htmlDocument = TryGetHtmlDocument(requestUrl);
if (htmlDocument != null)
{
if (movie.IdMovie == 0)
{
var titlePath = "//h3[@class='post-title text']";
var titleNode = htmlDocument.DocumentNode.SelectSingleNode(titlePath).InnerText.Trim();
var number = titleNode.Substring(0, titleNode.IndexOf(" ")).ToUpper();
var title = titleNode.Substring(titleNode.IndexOf(" ") + 1).ReplaceInvalidChar();
var picPath = "//img[@id='video_jacket_img']";
var picUrl = htmlDocument.DocumentNode.SelectSingleNode(picPath).Attributes["src"].Value;
movie.CoverUrl = picUrl.StartsWith("http") ? picUrl : "http:" + picUrl;
if (movie.Title == null)
{
movie.Title = title;
}
if (movie.Number == null)
{
movie.Number = number;
}
var dtReleasePath = "//div[@id='video_date']//td[@class='text']";
var release = htmlDocument.DocumentNode.SelectSingleNode(dtReleasePath);
DateTime rDate = DateTime.MinValue;
if (release != null && !string.IsNullOrEmpty(release.InnerText))
{
if (DateTime.TryParse(release.InnerText.Trim(), out rDate))
{
movie.DtRelease = rDate;
}
}
var lengthPath = "//div[@id='video_length']//span[@class='text']";
var duration = htmlDocument.DocumentNode.SelectSingleNode(lengthPath);
if (duration != null && !string.IsNullOrEmpty(duration.InnerText))
{
movie.Duration = int.Parse(duration.InnerText.Trim());
}
var actorPath = "//span[@class='star']//a";
movie.Actor = GenerateMovieRoles(htmlDocument, actorPath, movie, JavlibRoleType.Actor);
var dirPath = "//span[@class='director']//a";
movie.Director = GenerateMovieRoles(htmlDocument, dirPath, movie, JavlibRoleType.Director);
var comPath = "//span[@class='maker']//a";
movie.Company = GenerateMovieRoles(htmlDocument, comPath, movie, JavlibRoleType.Company);
var pubPath = "//span[@class='label']//a";
movie.Publisher = GenerateMovieRoles(htmlDocument, pubPath, movie, JavlibRoleType.Publisher);
var catPath = "//span[@class='genre']//a";
movie.Category = GenerateMovieRoles(htmlDocument, catPath, movie, JavlibRoleType.Category);
}
var nbWantPath = "//span[@id='subscribed']//a";
movie.NbWant = int.Parse(htmlDocument.DocumentNode.SelectSingleNode(nbWantPath).InnerText.Trim());
var nbWatchedPath = "//span[@id='watched']//a";
movie.NbWatched = int.Parse(htmlDocument.DocumentNode.SelectSingleNode(nbWatchedPath).InnerText.Trim());
var nbOwnedPath = "//span[@id='owned']//a";
movie.NbOwned = int.Parse(htmlDocument.DocumentNode.SelectSingleNode(nbOwnedPath).InnerText.Trim());
int?wantLevel = movie.NbWant + movie.NbWatched + movie.NbOwned;
if (wantLevel.HasValue)
{
if (wantLevel.Value >= _javlibSettings.DownloadSubPoint)
{
movie.FavLevel = JavlibFavLevel.DlChineseSub;
}
else if (wantLevel.Value >= _javlibSettings.DownloadMoviePoint && movie.FavLevel != JavlibFavLevel.DlChineseSub)
{
movie.FavLevel = JavlibFavLevel.DlMovie;
}
else if (wantLevel.Value >= _javlibSettings.DownloadTorrentPoint && movie.FavLevel != JavlibFavLevel.DlChineseSub && movie.FavLevel != JavlibFavLevel.DlMovie)
{
movie.FavLevel = JavlibFavLevel.DlTorrent;
}
}
}
else
{
_logger?.LogWarning("Nothing found when scanning movie details. UrlInfo: {urlInfo}", urlInfo.ToString());
}
}
catch (Exception ex)
{
_logger?.LogError(ex, "Error occurred when scanning movie {movieNumber} details: {urlInfo}", movie.Number, urlInfo.ToString());
}
}
