public int Execute()
{
using (var db = new SongRecommendContext()) {
// 대상 조회
var targetSong = from baseSong in db.BaseWordCollectingSong
join proposeSong in db.ProposeSong
on baseSong.SongId equals proposeSong.SongId into proposeSongs
from defaultPropose in proposeSongs.DefaultIfEmpty()
where baseSong.Status == "Tokenized" && defaultPropose == null
select baseSong;
// Rate 계산
foreach (var song in targetSong)
{
try {
var rateResult = AnalyzeRateSvc.Execute(song.Lyric);
song.Rate = rateResult.Rate;
song.Status = "Analyzed";
if (song.Rate > 70)
{
//---------------------------
// 좋아요 가져오기
//---------------------------
HttpClient client = new HttpClient();
var jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={song.SongId}").Result;
var like = 0;
try {
like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT");
}
catch { }
//---------------------------
// 크롤링 설정
//---------------------------
var pageRequester = new PageRequester(_config);
var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null);
crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync;
//---------------------------
// 크롤링 시작
//---------------------------
crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={song.SongId}"));
db.ProposeSong.Add(new ProposeSong {
SongId = song.SongId,
PlayListSeq = song.PlayListSeq,
Title = song.Title,
Singer = song.Singer,
Lyric = song.Lyric,
Rate = song.Rate ?? 0,
Like = like,
Genre = _genre,
ReleaseDate = _releaseDate,
AddDate = DateTime.Now
});
_successCount++;
}
}
catch { }
}
db.SaveChanges();
return(_successCount);
}
}
private void ProcessPageCrawlCompletedAsync(object sender, PageCrawlCompletedArgs e)
{
var crawledPage = e.CrawledPage;
var doc = crawledPage.HtmlDocument.DocumentNode;
var songNodes = doc.SelectNodes("//table/tbody/tr");
//---------------------------
// 크롤링 유효성 검사
//---------------------------
if (songNodes == null || songNodes.Count == 0)
{
_isCrawlingSuccess = false;
return;
}
_isCrawlingSuccess = true;
foreach (var node in songNodes)
{
try {
using (var db = new SongRecommendContext()) {
//---------------------------
// 노래정보 파싱
//---------------------------
var songId = node.SelectSingleNode(".//input[@class='input_check'] | .//input[@class='input_check ']").GetAttributeValue("value", 0);
var title = node.SelectSingleNode(".//div[@class='ellipsis rank01']//a | .//div[@class='ellipsis rank01']//span[@class='fc_lgray']").InnerText;
var singer = node.SelectSingleNode(".//div[@class='ellipsis rank02']//span").InnerText;
if (songId == 0 || db.ProposeSong.Find(songId) != null)
{
continue;
}
//---------------------------
// 가사 가져오기
//---------------------------
HttpClient client = new HttpClient();
string jsonString = client.GetStringAsync($"https://www.melon.com/song/lyricInfo.json?songId={songId}").Result;
var lyric = JObject.Parse(jsonString).Value <string>("lyric");
if (lyric == null || lyric.Length == 0)
{
continue;
}
//---------------------------
// 적합도 분석
//---------------------------
var rate = AnalyzeRateSvc.Execute(lyric).Rate;
//---------------------------
// DB 저장
//---------------------------
if (rate > 70)
{
//---------------------------
// 좋아요 가져오기
//---------------------------
jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={songId}").Result;
var like = 0;
try {
like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT");
}
catch { }
//---------------------------
// 크롤링 설정
//---------------------------
var pageRequester = new PageRequester(_config);
var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null);
crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync;
//---------------------------
// 크롤링 시작
//---------------------------
crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={songId}"));
db.ProposeSong.Add(new ProposeSong {
SongId = songId,
PlayListSeq = PlayListSeq,
Title = title,
Singer = singer,
Lyric = lyric,
Rate = rate,
Like = like,
Genre = _genre,
ReleaseDate = _releaseDate,
AddDate = DateTime.Now
});
db.SaveChanges();
_successCount++;
}
}
}
catch {
}
}
}
public AnalyzeSongResult Execute()
{
//---------------------------
// 가사 가져오기
//---------------------------
HttpClient client = new HttpClient();
string jsonString = client.GetStringAsync($"https://www.melon.com/song/lyricInfo.json?songId={SongId}").Result;
var lyric = JObject.Parse(jsonString).Value <string>("lyric");
if (lyric == null || lyric.Length == 0)
{
return(null);
}
var analyzeResult = AnalyzeRateSvc.Execute(lyric);
//---------------------------
// 좋아요 가져오기
//---------------------------
jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={SongId}").Result;
var like = 0;
try {
like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT");
}
catch { }
//---------------------------
// 크롤링 설정
//---------------------------
var pageRequester = new PageRequester(_config);
var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null);
crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync;
//---------------------------
// 크롤링 시작
//---------------------------
crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={SongId}"));
var song = new ProposeSong {
SongId = SongId,
Title = _title,
Singer = _singer,
Lyric = lyric,
Rate = analyzeResult.Rate,
Like = like,
Genre = _genre,
ReleaseDate = _releaseDate,
AddDate = DateTime.Now
};
if (analyzeResult.Rate > 70)
{
using (var db = new SongRecommendContext()) {
if (db.ProposeSong.Find(SongId) == null)
{
db.ProposeSong.Add(song);
db.SaveChanges();
}
}
}
var resultLyric = lyric;
foreach (var word in analyzeResult.Words)
{
resultLyric = resultLyric.Replace(word.Word, $@"<span class='v-chip theme--dark light-green darken-2'><span class='v-chip__content tooltip'>{word.Word}<span class='tooltiptext'>{(int)word.Rate}%</span></span></span>");
}
var result = new AnalyzeSongResult {
SongId = SongId,
Title = _title,
Singer = _singer,
Lyric = resultLyric,
Rate = analyzeResult.Rate,
AlbumCover = _albumCover,
AlbumName = _albumName
};
return(result);
}
