public IActionResult UpdateProposeSongs()
{
using (var db = new SongRecommendContext()) {
var count = 0;
foreach (var song in db.ProposeSong)
{
//---------------------------
// 좋아요 가져오기
//---------------------------
HttpClient client = new HttpClient();
var jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={song.SongId}").Result;
var like = 0;
try {
like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT");
}
catch { }
song.Like = like;
song.Rate = AnalyzeRateSvc.Execute(song.Lyric).Rate;
count++;
}
db.SaveChanges();
return(Ok(count));
}
}
private void ProcessPageCrawlCompletedAsync(object sender, PageCrawlCompletedArgs e)
{
try {
using (var db = new SongRecommendContext()) {
//---------------------------
// 가사 가져오기
//---------------------------
HttpClient client = new HttpClient();
string jsonString = client.GetStringAsync($"https://www.melon.com/song/lyricInfo.json?songId={SongId}").Result;
var lyric = JObject.Parse(jsonString).Value <string>("lyric");
if (string.IsNullOrEmpty(lyric))
{
_isCrawlingSuccess = false;
_message = "가사가 없습니다";
return;
}
//---------------------------
// DB 저장
//---------------------------
var crawledPage = e.CrawledPage;
var doc = crawledPage.HtmlDocument.DocumentNode;
//---------------------------
// 노래정보 파싱
//---------------------------
var title = doc.SelectSingleNode(".//div[@class='song_name']").ChildNodes[2].InnerText.Trim();
if (string.IsNullOrEmpty(title))
{
_isCrawlingSuccess = false;
_message = "곡명이 없습니다";
return;
}
var singer = doc.SelectSingleNode(".//div[@class='artist']").InnerText.Trim();
if (string.IsNullOrEmpty(singer))
{
_isCrawlingSuccess = false;
_message = "가수명이 없습니다";
return;
}
db.BaseWordCollectingSong.Add(new BaseWordCollectingSong {
SongId = SongId,
Title = title,
Singer = singer,
Lyric = Regex.Replace(lyric, @"<br>|<br/>|</br>", " ", RegexOptions.IgnoreCase).Trim(),
Status = CrawlingStatus.Ready
});
db.SaveChanges();
_isCrawlingSuccess = true;
_message = $"{title} 을 추가했습니다";
}
}
catch (Exception ex) {
_isCrawlingSuccess = false;
_message = ex.Message;
}
}
public int Execute()
{
IEnumerable <BaseWordCollectingSong> songList = null;
//---------------------------
// 목록 가져오기
//---------------------------
using (var db = new SongRecommendContext()) {
songList = db.BaseWordCollectingSong.Where(x => x.Status == CrawlingStatus.PositiveSet).ToList();
}
foreach (var song in songList)
{
using (var db = new SongRecommendContext()) {
var exceptTokenType = new List <KoreanPos> {
KoreanPos.Space, KoreanPos.Josa, KoreanPos.Determiner, KoreanPos.Alpha, KoreanPos.Punctuation, KoreanPos.Foreign, KoreanPos.Hashtag, KoreanPos.Number, KoreanPos.CashTag, KoreanPos.URL, KoreanPos.KoreanParticle
};
var words = song.Lyric.Normalize() // 형태소 분석 토큰
.Tokenize()
.Where(x => exceptTokenType.Contains(x.Pos) == false)
.Select(x => x.Text)
.Distinct();
//------------------------------------
// 3. 단어 DB에 Update 또는 저장
//------------------------------------
foreach (var word in words)
{
var baseWord = db.BaseWord.Find(word);
if (baseWord == null)
{
db.BaseWord.Add(new BaseWord {
Word = word,
PositivePoint = song.IsPositive == true ? 1 : 0,
NegativePoint = song.IsPositive == false ? 1 : 0,
});
}
else
{
if (song.IsPositive == true)
{
baseWord.PositivePoint++;
}
if (song.IsPositive == false)
{
baseWord.NegativePoint++;
}
}
}
var findSong = db.BaseWordCollectingSong.Find(song.SongId);
findSong.Status = CrawlingStatus.Tokenized;
db.SaveChanges();
_successCount++;
}
}
return(_successCount);
}
public IActionResult CalcSongsRate()
{
using (var db = new SongRecommendContext()) {
foreach (var song in db.ProposeSong)
{
song.Rate = AnalyzeRateSvc.Execute(song.Lyric).Rate;
}
db.SaveChanges();
return(Ok());
}
}
public IActionResult CalcWordsRate()
{
using (var db = new SongRecommendContext()) {
var totalPositive = db.BaseWord.Sum(x => x.PositivePoint * 100);
var totalNegative = db.BaseWord.Sum(x => x.NegativePoint * 100);
foreach (var word in db.BaseWord)
{
word.Rate = AnalyzeRateSvc.GetAdjustedRate(word.PositivePoint * 100, word.NegativePoint * 100, totalPositive, totalNegative);
}
db.SaveChanges();
return(Ok());
}
}
public void Execute()
{
using (var localDb = new SongRecommendLocalContext())
using (var azureDb = new SongRecommendContext()) {
foreach (var song in localDb.BaseWordCollectingSongs)
{
var findSong = azureDb.BaseWordCollectingSong.Find(song.SongId);
if (findSong == null)
{
var newSong = new Model.BaseWordCollectingSong {
SongId = song.SongId,
PlayListSeq = song.PlayListSeq,
Title = song.Title,
Singer = song.Singer,
Lyric = song.Lyric,
Status = song.Status,
Rate = song.Rate,
Message = song.Message,
IsPositive = song.PositiveGrade == 100 ? true : false
};
azureDb.BaseWordCollectingSong.Add(newSong);
}
}
foreach (var word in localDb.BaseWords)
{
var findWord = azureDb.BaseWord.Find(word.Word);
if (findWord == null)
{
var newWord = new Model.BaseWord {
Word = word.Word,
PositivePoint = word.PositivePoint / 100,
NegativePoint = word.NegativePoint / 100,
};
azureDb.BaseWord.Add(newWord);
}
}
azureDb.SaveChanges();
}
}
public IActionResult ChangePositive([FromBody] ChangePositiveDto request)
{
using (var db = new SongRecommendContext()) {
var findSong = db.BaseWordCollectingSong.Find(request.SongId);
if (findSong == null)
{
return(BadRequest());
}
findSong.IsPositive = request.IsPositive;
if (findSong.Status == CrawlingStatus.Ready)
{
findSong.Status = CrawlingStatus.PositiveSet;
}
db.SaveChanges();
return(Ok());
}
}
private void ProcessPageCrawlCompletedAsync(object sender, PageCrawlCompletedArgs e)
{
var crawledPage = e.CrawledPage;
var doc = crawledPage.HtmlDocument.DocumentNode;
var songNodes = doc.SelectNodes("//table/tbody/tr");
//---------------------------
// 크롤링 유효성 검사
//---------------------------
if (songNodes == null || songNodes.Count == 0)
{
_isCrawlingSuccess = false;
return;
}
_isCrawlingSuccess = true;
foreach (var node in songNodes)
{
try {
using (var db = new SongRecommendContext()) {
//---------------------------
// 노래정보 파싱
//---------------------------
var songId = node.SelectSingleNode(".//input[@class='input_check'] | .//input[@class='input_check ']").GetAttributeValue("value", 0);
var title = node.SelectSingleNode(".//div[@class='ellipsis rank01']//a | .//div[@class='ellipsis rank01']//span[@class='fc_lgray']").InnerText;
var singer = node.SelectSingleNode(".//div[@class='ellipsis rank02']//span").InnerText;
if (songId == 0 || db.ProposeSong.Find(songId) != null)
{
continue;
}
//---------------------------
// 가사 가져오기
//---------------------------
HttpClient client = new HttpClient();
string jsonString = client.GetStringAsync($"https://www.melon.com/song/lyricInfo.json?songId={songId}").Result;
var lyric = JObject.Parse(jsonString).Value <string>("lyric");
if (lyric == null || lyric.Length == 0)
{
continue;
}
//---------------------------
// 적합도 분석
//---------------------------
var rate = AnalyzeRateSvc.Execute(lyric).Rate;
//---------------------------
// DB 저장
//---------------------------
if (rate > 70)
{
//---------------------------
// 좋아요 가져오기
//---------------------------
jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={songId}").Result;
var like = 0;
try {
like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT");
}
catch { }
//---------------------------
// 크롤링 설정
//---------------------------
var pageRequester = new PageRequester(_config);
var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null);
crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync;
//---------------------------
// 크롤링 시작
//---------------------------
crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={songId}"));
db.ProposeSong.Add(new ProposeSong {
SongId = songId,
PlayListSeq = PlayListSeq,
Title = title,
Singer = singer,
Lyric = lyric,
Rate = rate,
Like = like,
Genre = _genre,
ReleaseDate = _releaseDate,
AddDate = DateTime.Now
});
db.SaveChanges();
_successCount++;
}
}
}
catch {
}
}
}
private void ProcessPageCrawlCompletedAsync(object sender, PageCrawlCompletedArgs e)
{
var crawledPage = e.CrawledPage;
var doc = crawledPage.HtmlDocument.DocumentNode;
var songNodes = doc.SelectNodes("//table/tbody/tr");
//---------------------------
// 크롤링 유효성 검사
//---------------------------
if (songNodes == null || songNodes.Count == 0)
{
_isCrawlingSuccess = false;
return;
}
_isCrawlingSuccess = true;
foreach (var node in songNodes)
{
try {
using (var db = new SongRecommendContext()) {
//---------------------------
// 노래정보 파싱
//---------------------------
var songId = node.SelectSingleNode(".//input[@class='input_check'] | .//input[@class='input_check ']").GetAttributeValue("value", 0);
var title = node.SelectSingleNode(".//div[@class='ellipsis rank01']//a | .//div[@class='ellipsis rank01']//span[@class='fc_lgray']").InnerText;
var singer = node.SelectSingleNode(".//div[@class='ellipsis rank02']//span").InnerText;
if (songId == 0 || db.BaseWordCollectingSong.Find(songId) != null)
{
continue;
}
//---------------------------
// 가사 가져오기
//---------------------------
HttpClient client = new HttpClient();
string jsonString = client.GetStringAsync($"https://www.melon.com/song/lyricInfo.json?songId={songId}").Result;
var lyric = JObject.Parse(jsonString).Value <string>("lyric");
if (lyric == null || lyric.Length == 0)
{
continue;
}
//---------------------------
// DB 저장
//---------------------------
db.BaseWordCollectingSong.Add(new BaseWordCollectingSong {
SongId = songId,
PlayListSeq = PlayListSeq,
Title = title,
Singer = singer,
Lyric = Regex.Replace(lyric, @"<br>|<br/>|</br>", " ", RegexOptions.IgnoreCase).Trim(),
Status = CrawlingStatus.Ready
});
db.SaveChanges();
_successCount++;
}
}
catch {
}
}
}
public int Execute()
{
using (var db = new SongRecommendContext()) {
// 대상 조회
var targetSong = from baseSong in db.BaseWordCollectingSong
join proposeSong in db.ProposeSong
on baseSong.SongId equals proposeSong.SongId into proposeSongs
from defaultPropose in proposeSongs.DefaultIfEmpty()
where baseSong.Status == "Tokenized" && defaultPropose == null
select baseSong;
// Rate 계산
foreach (var song in targetSong)
{
try {
var rateResult = AnalyzeRateSvc.Execute(song.Lyric);
song.Rate = rateResult.Rate;
song.Status = "Analyzed";
if (song.Rate > 70)
{
//---------------------------
// 좋아요 가져오기
//---------------------------
HttpClient client = new HttpClient();
var jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={song.SongId}").Result;
var like = 0;
try {
like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT");
}
catch { }
//---------------------------
// 크롤링 설정
//---------------------------
var pageRequester = new PageRequester(_config);
var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null);
crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync;
//---------------------------
// 크롤링 시작
//---------------------------
crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={song.SongId}"));
db.ProposeSong.Add(new ProposeSong {
SongId = song.SongId,
PlayListSeq = song.PlayListSeq,
Title = song.Title,
Singer = song.Singer,
Lyric = song.Lyric,
Rate = song.Rate ?? 0,
Like = like,
Genre = _genre,
ReleaseDate = _releaseDate,
AddDate = DateTime.Now
});
_successCount++;
}
}
catch { }
}
db.SaveChanges();
return(_successCount);
}
}
public AnalyzeSongResult Execute()
{
//---------------------------
// 가사 가져오기
//---------------------------
HttpClient client = new HttpClient();
string jsonString = client.GetStringAsync($"https://www.melon.com/song/lyricInfo.json?songId={SongId}").Result;
var lyric = JObject.Parse(jsonString).Value <string>("lyric");
if (lyric == null || lyric.Length == 0)
{
return(null);
}
var analyzeResult = AnalyzeRateSvc.Execute(lyric);
//---------------------------
// 좋아요 가져오기
//---------------------------
jsonString = client.GetStringAsync($"https://www.melon.com/commonlike/getSongLike.json?contsIds={SongId}").Result;
var like = 0;
try {
like = JObject.Parse(jsonString).Value <IEnumerable <JToken> >("contsLike").First().Value <int>("SUMMCNT");
}
catch { }
//---------------------------
// 크롤링 설정
//---------------------------
var pageRequester = new PageRequester(_config);
var crawler = new PoliteWebCrawler(_config, null, null, null, pageRequester, null, null, null, null);
crawler.PageCrawlCompletedAsync += ProcessDetailPageCrawlCompletedAsync;
//---------------------------
// 크롤링 시작
//---------------------------
crawler.Crawl(new Uri($"https://www.melon.com/song/detail.htm?songId={SongId}"));
var song = new ProposeSong {
SongId = SongId,
Title = _title,
Singer = _singer,
Lyric = lyric,
Rate = analyzeResult.Rate,
Like = like,
Genre = _genre,
ReleaseDate = _releaseDate,
AddDate = DateTime.Now
};
if (analyzeResult.Rate > 70)
{
using (var db = new SongRecommendContext()) {
if (db.ProposeSong.Find(SongId) == null)
{
db.ProposeSong.Add(song);
db.SaveChanges();
}
}
}
var resultLyric = lyric;
foreach (var word in analyzeResult.Words)
{
resultLyric = resultLyric.Replace(word.Word, $@"<span class='v-chip theme--dark light-green darken-2'><span class='v-chip__content tooltip'>{word.Word}<span class='tooltiptext'>{(int)word.Rate}%</span></span></span>");
}
var result = new AnalyzeSongResult {
SongId = SongId,
Title = _title,
Singer = _singer,
Lyric = resultLyric,
Rate = analyzeResult.Rate,
AlbumCover = _albumCover,
AlbumName = _albumName
};
return(result);
}