//This function is leaved as async as it can be a copy function in future => This will take some time
public async Task GenerateMovieFile(MovieMagnet magnet)
{
_logger?.LogInformation("GenerateMovieFile Start - {movieNumber}", magnet.MovieNumber);
FileInfo movieFile = FindMovieFile(magnet.SavePath, magnet.MovieNumber);
if (movieFile != null)
{
string movieNameWithoutExt = magnet.MovieNumber.ToUpper() + (magnet.HasSub ? "-C" : "");
string destMovieName = movieNameWithoutExt + Path.GetExtension(movieFile.FullName);
string destMovieFolder = Path.Combine(_localFileSetting.DestSaveRootPath, movieNameWithoutExt);
string destMoviePath = Path.Combine(destMovieFolder, destMovieName);
if (!Directory.Exists(destMovieFolder))
{
_logger.LogInformation("Create destination movie folder: {destMovieFolder}", destMovieFolder);
Directory.CreateDirectory(destMovieFolder);
}
_logger.LogInformation("Moving movie {movieNumber} to {destMoviePath}", magnet.MovieNumber, destMoviePath);
await Task.Run(() => movieFile.MoveTo(destMoviePath, true));
_logger.LogInformation("Movie {movieNumber} has been moved to dest location. Now Archiving the temp download folder", magnet.MovieNumber);
MoveDirectory(magnet.SavePath, _localFileSetting.ArchivedDownloadPath);
magnet.SavePath = destMovieFolder;
magnet.IdStatus = MagnetStatus.Finished;
magnet.DtFinish = DateTime.Now;
}
else
{
_logger.LogWarning("Cannot find movie {movieNumber} in path: {savePath}", magnet.MovieNumber, magnet.SavePath);
}
}
public void TestSaveListMagnetFailed()
{
List <MovieMagnet> movieMagnets = new List <MovieMagnet>();
MovieMagnet magnet = new MovieMagnet()
{
IdMovieMag = 9,
IdMovie = 73,
MovieNumber = "FSDSS-298",
MagName = "magnadsdsdesf",
MagnetUrl = "dsfdsfdsf",
Hash = "dvsffvdsv",
Size = 1234,
IdMagSource = MagnetSource.Sukebei,
DtMagnet = new DateTime(2021, 1, 1)
};
MovieMagnet magnet2 = new MovieMagnet()
{
IdMovieMag = 25,
IdMovie = 82,
MovieNumber = "GVH-301",
MagName = "sdfezfdsf",
MagnetUrl = "dsfdsfdsfezfzef",
Hash = "dvsffvdsv",
Size = 3154,
IdMagSource = MagnetSource.Javbus,
DtMagnet = new DateTime(2021, 1, 2)
};
movieMagnets.Add(magnet);
movieMagnets.Add(magnet2);
_movieMagService.SaveMovieMagnetList(movieMagnets);
}
public async Task TreatDownloadedMovie(MovieMagnet magnet)
{
try
{
await _localFileService.GenerateMovieFile(magnet);
List <MovieMagnet> movieMagnets = new List <MovieMagnet>();
if (magnet.HasSub)
{
_logger.LogInformation("Movie {movieNumber} has downloaded sub version. Checking non-sub folder and delete it.", magnet.MovieNumber);
movieMagnets = _movieMagnetService.FindMovieMagnetByStatus(MagnetStatus.Finished, magnet.IdMovie);
foreach (MovieMagnet movieMagnet in movieMagnets)
{
await _localFileService.DeleteFolder(movieMagnet.SavePath);
movieMagnet.IdStatus = MagnetStatus.Archived;
}
}
movieMagnets.Add(magnet);
_movieMagnetService.SaveMovieMagnetList(movieMagnets);
await _qbittorrentService.DeleteTorrentAsync(magnet.Hash, false);
Movie movie = _movieService.FindMovieById(magnet.IdMovie);
_movieService.UpdateStatus(movie, magnet.HasSub ? MovieStatus.Finished : MovieStatus.Downloaded);
_movieService.SaveMovie(movie);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error occurred when generating the downloaded movie file - {movieNumber}", magnet.MovieNumber);
magnet.IdStatus = MagnetStatus.InError;
_movieMagnetService.SaveMovieMagnet(magnet);
}
}
public bool TorrentIsNotActive(MovieMagnet magnet, TorrentInfo torrent)
{
if ((DateTime.Now - magnet.DtStart.Value).Days >= 5)
{
return(true);
}
return((DateTime.Now - magnet.DtStart.Value).Days > 2 && torrent.Progress < 0.5 && torrent.EstimatedTime.Value.Days >= 100);
}
public void TestInsertMagnet()
{
var magnet = new MovieMagnet()
{
MovieNumber = "CAWD-321", IdMovie = 1492, MagName = "magname", MagnetUrl = "123456&dl", Hash = "123456", IdMagSource = MagnetSource.Sukebei
};
var magnetInserted = _movieMagService.SaveMovieMagnet(magnet);
Assert.True(magnetInserted.IdMagSource > 0);
}
public async Task TestGenerateMovieFile()
{
MovieMagnet magnet = new MovieMagnet()
{
SavePath = @"E:\Media\Sister\Data\Finished\snis-842-C", HasSub = true, MovieNumber = "snis-842"
};
await _localFileService.GenerateMovieFile(magnet);
Assert.Equal(MagnetStatus.Finished, magnet.IdStatus);
_output.WriteLine(magnet.ToString());
}
public List <MovieMagnet> SearchMagnetFromSukebei(Movie movie)
{
string javbusUrl = string.Format(_movieMagnetService.LoadMagSourceUrl(MagnetSource.Sukebei), movie.Number);
HtmlDocument htmlDocument = _htmlService.GetHtmlDocumentAsync(javbusUrl, maxRetry: 1).Result;
List <MovieMagnet> lstMovieMagnets = new List <MovieMagnet>();
if (htmlDocument != null)
{
string xpath = "//tr";
HtmlNodeCollection nodes = htmlDocument.DocumentNode.SelectNodes(xpath);
if (nodes != null && nodes.Count > 1)
{
foreach (var node in nodes.Skip(1))
{
MovieMagnet movieMagnet = new MovieMagnet()
{
IdMovie = movie.IdMovie, MovieNumber = movie.Number, IdMagSource = MagnetSource.Sukebei
};
movieMagnet.MagName = node.ChildNodes[3].InnerText.Trim();
var magHref = node.ChildNodes[5].OuterHtml;
var size = node.ChildNodes[7].InnerText.Trim();
movieMagnet.Size = size.GetByteSize();
DateTime dtMag = DateTime.MinValue;
if (!string.IsNullOrEmpty(node.ChildNodes[9].InnerText) && DateTime.TryParse(node.ChildNodes[9].InnerText.Trim(), out dtMag))
{
movieMagnet.DtMagnet = dtMag;
}
var url = magHref.Substring(magHref.IndexOf("<a href=\"magnet:?xt") + 9);
movieMagnet.MagnetUrl = url.Substring(0, url.IndexOf("\""));
movieMagnet.GenerateHash();
if (movieMagnet.MagName.Contains("高清") || movieMagnet.MagName.ToLower().Contains("hd") || movieMagnet.Size > 2048)
{
movieMagnet.IsHD = true;
}
if ((movieMagnet.MagName.Contains("字幕") || movieMagnet.MagName.Contains("中文")) && movieMagnet.IsHD)
{
movieMagnet.HasSub = true;
}
lstMovieMagnets.Add(movieMagnet);
}
}
}
return(lstMovieMagnets);
}
public void TestAddMovieMagnetHistory()
{
var dbContext = new DapperContext(new ConfigBuilder().Build());
var movieHistoryRepo = new MovieHistoryRepo(dbContext);
var movieMagRepo = new MovieMagnetRepo(dbContext, movieHistoryRepo);
MovieMagnet magnet = new MovieMagnet()
{
IdMovieMag = 9, IdMovie = 73, MovieNumber = "FSDSS-298", MagName = "magnamesf", Size = 5400, DtMagnet = new DateTime(2020, 1, 1)
};
List <MovieHistory> movieHistories = movieMagRepo.AddHistory(magnet);
movieHistories.ForEach(h => _output.WriteLine(h.DescHistory));
}
public async Task AddTorrentAsync(MovieMagnet magnet)
{
_logger?.LogInformation("Add Torrent for movie {movieNumber}: {hash}", magnet.MovieNumber, magnet.Hash);
await EnsureLoggedInAsync();
AddTorrentUrlsRequest request = new AddTorrentUrlsRequest(new Uri(magnet.MagnetUrl));
await _client.AddTorrentsAsync(request);
string savePath = Path.Combine(_qbittorrentSetting.DownloadRootPath, magnet.MovieNumber.ToUpper() + "_" + magnet.IdMovieMag);
await _client.SetLocationAsync(magnet.Hash, savePath);
await _client.SetTorrentCategoryAsync(magnet.Hash, _qbittorrentSetting.Category);
magnet.IdStatus = MagnetStatus.Downloading;
magnet.SavePath = savePath;
magnet.DtStart = DateTime.Now;
_logger?.LogInformation("Add Torrent for movie {movieNumber} ended", magnet.MovieNumber);
}
public void TestAddTorrent()
{
List <Task> taskList = new List <Task>();
MovieMagnet magnet = new MovieMagnet()
{
IdMovieMag = 247,
IdMovie = 157, MagName = "mide-978-C", MovieNumber = "MIDE-978", Size = 5376, IsHD = true, HasSub = true, IdMagSource = MagnetSource.Javbus, IdStatus = MagnetStatus.IsReady,
MagnetUrl = "magnet:?xt=urn:btih:D731203D3B903F675F96CD1D4FDC547A6428EB80&dn=mide-978-C", Hash = "D731203D3B903F675F96CD1D4FDC547A6428EB80"
};
var _movieMagnetService = new MovieMagServiceBuilder().Build();
_output.WriteLine("task created");
taskList.Add(_qbittorrentService.AddTorrentAsync(magnet).ContinueWith(t => _movieMagnetService.SaveMovieMagnet(magnet)));
_output.WriteLine("waiting task to over: {0}", taskList.Count());
Task.WaitAll(taskList.ToArray());
}
public MovieMagnet Save(MovieMagnet movieMagnet)
{
movieMagnet.DtUpdate = DateTime.Now;
using (var trans = new TransactionScope())
{
List <MovieHistory> movieHistories = AddHistory(movieMagnet);
if (movieMagnet.IdMovieMag == 0)
{
movieMagnet.IdMovieMag = (int)db.InsertEntity(movieMagnet);
}
else if (!db.UpdateEntity(movieMagnet))
{
throw new Exception($"MovieMagnet not found in DB: {movieMagnet.ToString()}");
}
_movieHistoryRepo.SaveList(movieMagnet.IdMovie, movieHistories);
trans.Complete();
}
return(movieMagnet);
}
public MovieMagnet SaveMovieMagnet(MovieMagnet movieMagnet)
{
return(_movieMagnetRepo.Save(movieMagnet));
}
//TODO: Consider to group the history into one record
public List <MovieHistory> AddHistory(MovieMagnet magnet)
{
List <MovieHistory> movieHistories = new List <MovieHistory>();
if (magnet.IdMovieMag == 0)
{
movieHistories.Add(new MovieHistory()
{
DescHistory = string.Format("Create new movie magnet {0}", magnet.Hash)
});
}
else
{
MovieMagnet origin = FindById(magnet.IdMovieMag);
string result = string.Empty;
if (HistoryDiffHelpers.GetDifferencesFieldString(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "MagName"), origin.MagName, magnet.MagName, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldString(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "MagnetUrl"), origin.MagnetUrl, magnet.MagnetUrl, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldString(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "Hash"), origin.Hash, magnet.Hash, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldDemical(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "Size"), origin.Size, magnet.Size, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldDate(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "Magnet Date"), origin.DtMagnet, magnet.DtMagnet, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldBoolean(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "IsHD"), origin.IsHD, magnet.IsHD, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldBoolean(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "HasSub"), origin.HasSub, magnet.HasSub, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldString(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "Magnet Source"), origin.IdMagSource.ToString(), magnet.IdMagSource.ToString(), ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldDate(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "DtStart"), origin.DtStart, magnet.DtStart, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldDate(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "DtFinish"), origin.DtFinish, magnet.DtFinish, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldString(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "SavePath"), origin.SavePath, magnet.SavePath, ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
if (HistoryDiffHelpers.GetDifferencesFieldString(string.Format("MagnetId {0} - {1}", magnet.IdMovieMag, "Magnet Status"), origin.IdStatus.ToString(), magnet.IdStatus.ToString(), ref result))
{
movieHistories.Add(new MovieHistory()
{
DescHistory = result
});
}
}
return(movieHistories);
}
public void MonitorMovieDownload()
{
try
{
_logger.LogInformation("************** Start Monitoring Movie Download Job - {Date} **************", DateTime.Now.ToString("u", DateTimeFormatInfo.InvariantInfo));
//Check every magnet in status downloading. If not found in qbittorrent or status = downloaded, do fileService
int nbMovieAdded = 0;
List <Task> lstGenerateMovieFileTask = new List <Task>();
List <Task> lstDeleteTorrentTask = new List <Task>();
List <Task> lstAddTorrentTask = new List <Task>();
foreach (MovieMagnet magnet in _movieMagnetService.FindMovieMagnetByStatus(MagnetStatus.Downloading))
{
TorrentInfo torrentInfo = _qbittorrentService.GetTorrentInfo(magnet.Hash);
if (torrentInfo == null || torrentInfo.State == TorrentState.Uploading || torrentInfo.Progress >= 1)
{
if (_localFileService.CheckMovieDownloaded(magnet.SavePath, magnet.MovieNumber.ToUpper()))
{
_logger?.LogInformation("MovieMagnet {magnetId} for movie {movieNumber} has finished downloading", magnet.IdMovieMag, magnet.MovieNumber);
lstGenerateMovieFileTask.Add(TreatDownloadedMovie(magnet));
}
else
{
_logger?.LogError("Cannot find movie {movieNumber} in path {moviePath}. Please check manually", magnet.MovieNumber, magnet.SavePath);
magnet.IdStatus = MagnetStatus.InError;
_movieMagnetService.SaveMovieMagnet(magnet);
}
}
else if (TorrentIsNotActive(magnet, torrentInfo))
{
_logger?.LogWarning("MovieMagnet {magnetId} for movie {movieNumber} is no longer active, Mark it as dead", magnet.IdMovieMag, magnet.MovieNumber);
lstDeleteTorrentTask.Add(_qbittorrentService.DeleteTorrentAsync(magnet.Hash, true));
magnet.IdStatus = MagnetStatus.Dead;
_movieMagnetService.SaveMovieMagnet(magnet);
}
}
Task.WaitAll(lstDeleteTorrentTask.ToArray());
Task.WaitAll(lstGenerateMovieFileTask.ToArray());
//Send torrent to qbittorrent for the top nbFinished
List <Movie> lstMoviesToDownload = _movieService.LoadMoviesToDownload();
if (lstMoviesToDownload.Count > 0)
{
int nbMovieAddMax = _qbittorrentService.GetTorrentCanAddCount();
foreach (Movie movie in lstMoviesToDownload)
{
if (nbMovieAdded >= nbMovieAddMax)
{
break;
}
MovieMagnet magnet = _movieMagnetService.FindBestMatchMagnetByMovie(movie);
if (magnet != null)
{
//TODO: Here we can add antoher async function to get the torrentContent and set other files priority to notDownload
lstAddTorrentTask.Add(_qbittorrentService.AddTorrentAsync(magnet)
.ContinueWith(t => _movieMagnetService.SaveMovieMagnet(magnet)));
nbMovieAdded++;
}
}
Task.WaitAll(lstAddTorrentTask.ToArray());
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Some errors occurred in MonitorMovieDownload");
}
finally
{
_logger.LogInformation("************** Monitoring Movie Download Job End - {Date} **************", DateTime.Now.ToString("u", DateTimeFormatInfo.InvariantInfo));
}
}
public List <MovieMagnet> SearchMagnetFromJavbus(Movie movie)
{
string javbusUrl = string.Format(_movieMagnetService.LoadMagSourceUrl(MagnetSource.Javbus), movie.Number);
HtmlDocument htmlDocument = _htmlService.GetHtmlDocumentAsync(javbusUrl, maxRetry: 1).Result;
List <MovieMagnet> lstMovieMagnets = new List <MovieMagnet>();
if (htmlDocument != null)
{
string outerHtml = htmlDocument.DocumentNode.InnerHtml;
var gidPattern = "var gid = (.*?);";
var ucPattern = "var uc = (.*?);";
var picPattern = "var img = '(.*?)';";
var gidMatch = Regex.Match(outerHtml, gidPattern);
var ucMatch = Regex.Match(outerHtml, ucPattern);
var picMatch = Regex.Match(outerHtml, picPattern);
var gid = gidMatch.Groups[1].Value;
var uc = ucMatch.Groups[1].Value;
var pic = picMatch.Groups[1].Value;
HtmlDocument magDocument = null;
var magUrl = $"https://www.javbus.com/ajax/uncledatoolsbyajax.php?gid={gid}&lang=zh&img={pic}&uc={uc}&floor=552";
magDocument = _htmlService.GetHtmlDocumentAsync(magUrl, new Dictionary <string, string>()
{
{ "referer", javbusUrl }
}).Result;
if (magDocument != null)
{
var magPattern = "//tr[@style=' border-top:#DDDDDD solid 1px']";
HtmlNodeCollection nodes = magDocument.DocumentNode.SelectNodes(magPattern);
if (nodes != null)
{
DateTime?lastDtReleaseInNode = null;
foreach (var node in nodes)
{
MovieMagnet movieMagnet = new MovieMagnet()
{
IdMovie = movie.IdMovie, MovieNumber = movie.Number, IdMagSource = MagnetSource.Javbus
};
if (node.ChildNodes.Count >= 2)
{
if (node.ChildNodes[1].InnerText.Contains("高清"))
{
movieMagnet.IsHD = true;
}
if (node.ChildNodes[1].InnerText.Contains("字幕") && movieMagnet.IsHD)
{
movieMagnet.HasSub = true;
}
movieMagnet.MagName = node.ChildNodes[1].ChildNodes[1].InnerText.Trim();
movieMagnet.MagnetUrl = node.ChildNodes[1].ChildNodes[1].Attributes["href"].Value;
movieMagnet.GenerateHash();
}
if (node.ChildNodes.Count >= 4)
{
string sizePart = node.ChildNodes[3].InnerText.Trim();
movieMagnet.Size = sizePart.GetByteSize();
}
if (node.ChildNodes.Count >= 5 && !string.Equals(node.ChildNodes[5].InnerText.Trim(), "0000-00-00"))
{
movieMagnet.DtMagnet = DateTime.Parse(node.ChildNodes[5].InnerText.Trim());
if (!lastDtReleaseInNode.HasValue || movieMagnet.DtMagnet < lastDtReleaseInNode)
{
lastDtReleaseInNode = movieMagnet.DtMagnet;
}
}
else
{
movieMagnet.HasSub = false;
}
lstMovieMagnets.Add(movieMagnet);
}
//JavBus Bug: The movie is not a SubVersion but has a Sub tag
foreach (MovieMagnet mag in lstMovieMagnets)
{
if (mag.DtMagnet.HasValue && (mag.DtMagnet.Value - lastDtReleaseInNode.Value).Days < 3)
{
mag.HasSub = false;
}
}
}
}
}
return(lstMovieMagnets);
}
