public int Execute() { using (var db = new SongRecommendContext()) { // 대상 조회 var targetSong = from baseSong in db.BaseWordCollectingSong join proposeSong in db.ProposeSong on baseSong.SongId equals proposeSong.SongId into proposeSongs from defaultPropose in proposeSongs.DefaultIfEmpty() where baseSong.Status == "Tokenized" && defaultPropose == null select baseSong; // Rate 계산 foreach (var song in targetSong) { try { var rateResult = AnalyzeRateSvc.Execute(song.Lyric); song.Rate = rateResult.Rate; song.Status = "Analyzed"; if (song.Rate > 70) { //--------------------------- // 좋아요 가져오기 //--------------------------- HttpClient client = new HttpClient(); var jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={song.SongId}").Result; var like = 0; try { like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT"); } catch { } //--------------------------- // 크롤링 설정 //--------------------------- var pageRequester = new PageRequester(_config); var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null); crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync; //--------------------------- // 크롤링 시작 //--------------------------- crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={song.SongId}")); db.ProposeSong.Add(new ProposeSong { SongId = song.SongId, PlayListSeq = song.PlayListSeq, Title = song.Title, Singer = song.Singer, Lyric = song.Lyric, Rate = song.Rate ?? 0, Like = like, Genre = _genre, ReleaseDate = _releaseDate, AddDate = DateTime.Now }); _successCount++; } } catch { } } db.SaveChanges(); return(_successCount); } }
private void ProcessPageCrawlCompletedAsync(object sender, PageCrawlCompletedArgs e) { var crawledPage = e.CrawledPage; var doc = crawledPage.HtmlDocument.DocumentNode; var songNodes = doc.SelectNodes("//table/tbody/tr"); //--------------------------- // 크롤링 유효성 검사 //--------------------------- if (songNodes == null || songNodes.Count == 0) { _isCrawlingSuccess = false; return; } _isCrawlingSuccess = true; foreach (var node in songNodes) { try { using (var db = new SongRecommendContext()) { //--------------------------- // 노래정보 파싱 //--------------------------- var songId = node.SelectSingleNode(".//input[@class='input_check'] | .//input[@class='input_check ']").GetAttributeValue("value", 0); var title = node.SelectSingleNode(".//div[@class='ellipsis rank01']//a | .//div[@class='ellipsis rank01']//span[@class='fc_lgray']").InnerText; var singer = node.SelectSingleNode(".//div[@class='ellipsis rank02']//span").InnerText; if (songId == 0 || db.ProposeSong.Find(songId) != null) { continue; } //--------------------------- // 가사 가져오기 //--------------------------- HttpClient client = new HttpClient(); string jsonString = client.GetStringAsync($"https://www.melon.com/song/lyricInfo.json?songId={songId}").Result; var lyric = JObject.Parse(jsonString).Value <string>("lyric"); if (lyric == null || lyric.Length == 0) { continue; } //--------------------------- // 적합도 분석 //--------------------------- var rate = AnalyzeRateSvc.Execute(lyric).Rate; //--------------------------- // DB 저장 //--------------------------- if (rate > 70) { //--------------------------- // 좋아요 가져오기 //--------------------------- jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={songId}").Result; var like = 0; try { like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT"); } catch { } //--------------------------- // 크롤링 설정 //--------------------------- var pageRequester = new PageRequester(_config); var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null); crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync; //--------------------------- // 크롤링 시작 //--------------------------- crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={songId}")); db.ProposeSong.Add(new ProposeSong { SongId = songId, PlayListSeq = PlayListSeq, Title = title, Singer = singer, Lyric = lyric, Rate = rate, Like = like, Genre = _genre, ReleaseDate = _releaseDate, AddDate = DateTime.Now }); db.SaveChanges(); _successCount++; } } } catch { } } }
public AnalyzeSongResult Execute() { //--------------------------- // 가사 가져오기 //--------------------------- HttpClient client = new HttpClient(); string jsonString = client.GetStringAsync($"https://www.melon.com/song/lyricInfo.json?songId={SongId}").Result; var lyric = JObject.Parse(jsonString).Value <string>("lyric"); if (lyric == null || lyric.Length == 0) { return(null); } var analyzeResult = AnalyzeRateSvc.Execute(lyric); //--------------------------- // 좋아요 가져오기 //--------------------------- jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={SongId}").Result; var like = 0; try { like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT"); } catch { } //--------------------------- // 크롤링 설정 //--------------------------- var pageRequester = new PageRequester(_config); var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null); crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync; //--------------------------- // 크롤링 시작 //--------------------------- crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={SongId}")); var song = new ProposeSong { SongId = SongId, Title = _title, Singer = _singer, Lyric = lyric, Rate = analyzeResult.Rate, Like = like, Genre = _genre, ReleaseDate = _releaseDate, AddDate = DateTime.Now }; if (analyzeResult.Rate > 70) { using (var db = new SongRecommendContext()) { if (db.ProposeSong.Find(SongId) == null) { db.ProposeSong.Add(song); db.SaveChanges(); } } } var resultLyric = lyric; foreach (var word in analyzeResult.Words) { resultLyric = resultLyric.Replace(word.Word, $@"<span class='v-chip theme--dark light-green darken-2'><span class='v-chip__content tooltip'>{word.Word}<span class='tooltiptext'>{(int)word.Rate}%</span></span></span>"); } var result = new AnalyzeSongResult { SongId = SongId, Title = _title, Singer = _singer, Lyric = resultLyric, Rate = analyzeResult.Rate, AlbumCover = _albumCover, AlbumName = _albumName }; return(result); }