/// <summary>
/// 全ページのアイテム情報を取得する
/// </summary>
/// <param name="saleInformation"></param>
/// https://images-na.ssl-images-amazon.com/images/G/09/associates/paapi/dg/index.html?ItemSearch.html
bool ItemSearchAll(SaleInformation saleInformation)
{
int page = 0;
bool result = false;
do
{
Task.Delay(requestWaitTimerMSec).Wait();
page++;
result = ItemSearch(saleInformation, page);
if (result == false)
{
break;
}
// 残りページなし
if ((saleInformation.TotalResults - page * 10) <= 0)
{
break;
}
// AWS の仕様上、10ページまでしか取得できない
} while (page < 10);
return(result);
}
/// <summary>
/// セールページをスクレイピングしてセール期間を判別
/// </summary>
/// <param name="saleInformation"></param>
/// <returns></returns>
public (string startDate, string endDate) ExtractSalePeriod(SaleInformation saleInformation, Stream stream)
{
// 指定したサイトのHTMLをストリームで取得する
// AngleSharp.Parser.Html.HtmlParserオブジェクトにHTMLをパースさせる
var parser = new HtmlParser();
var doc = parser.Parse(stream);
// ページによって h3 の場合も h4 の場合もある
foreach (var tag in new[] { "h3", "h4" })
{
var elements = doc.QuerySelectorAll(tag);
foreach (var element in elements)
{
// 例：期間限定：8/18（金）～8/31（木）
var result = Regex.Match(element.InnerHtml, @"期間限定：(?<StartDate>.*?)（.*）～(?<EndDate>.*?)（.*）");
if (result.Success == true)
{
return(result.Groups["StartDate"].Value, result.Groups["EndDate"].Value);
}
}
}
return(null, null);
}
public void InsertTest()
{
using (var context = new KindleDbContext(fixture.DataSource))
{
var saleInformation = new SaleInformation
{
Url = "http://blogs.msdn.com/dotnet",
Items = new List <ItemDetail>
{
new ItemDetail {
Title = "Intro to C#"
},
new ItemDetail {
Title = "Intro to VB.NET"
},
new ItemDetail {
Title = "Intro to F#"
}
}
};
context.SaleInformations.Add(saleInformation);
context.SaveChanges();
}
}
public void SetSalePeriodTest(int nowYear, int nowMonth, int nowDay, string startDate, string endDate, int count, int total, string expectedStartDate, string expectedEndDate)
{
var saleInformation = new SaleInformation();
DateTime now = new DateTime(nowYear, nowMonth, nowDay);
saleInformation.SetSalePeriod(now, startDate, endDate, count, total);
Assert.Equal(expectedStartDate, saleInformation.StartDate.ToString("yyyy/MM/dd"));
Assert.Equal(expectedEndDate, saleInformation.EndDate.ToString("yyyy/MM/dd"));
}
/// <summary>
/// セール期間を判別してセットする
/// </summary>
/// <param name="saleInformation"></param>
/// <param name="count">現在何件目のセール情報</param>
/// <param name="total">セール情報のトータル</param>
void CheckSalePeriod(SaleInformation saleInformation, int count, int total)
{
if (saleInformation.Error == true)
{
return;
}
// タイトルから終了日を判別
saleInformation.SetSalePeriod(DateTime.Now, null, ExtractEndDate(saleInformation.Name), count, total);
if (saleInformation.EndDate != null)
{
if (saleInformation.EndDate < DateTime.Now)
{
saleInformation.SaleFinished = true;
}
else
{
saleInformation.SaleStarted = true;
}
return;
}
// セールページをスクレイピングしてセール期間を判別
using (var stream = client.GetStreamAsync(GetAssociateLinkByBrowseNode(saleInformation.NodeId)).Result)
{
(var startDate, var endDate) = ExtractSalePeriod(saleInformation, stream);
saleInformation.SetSalePeriod(DateTime.Now, startDate, endDate, count, total);
}
if (saleInformation.StartDate != null && saleInformation.EndDate != null)
{
if (saleInformation.EndDate < DateTime.Now)
{
saleInformation.SaleFinished = true;
}
else if (saleInformation.StartDate < DateTime.Now)
{
saleInformation.SaleStarted = true;
saleInformation.SaleFinished = false;
}
return;
}
// ここまできたら、セールページは存在するが、タイトルからもセールページからもセール期間が判別できないので、
// 何も処理しない
}
public void GetSalePeriodTest()
{
var saleInformation = new SaleInformation();
saleInformation.SaleStarted = false;
saleInformation.SaleFinished = false;
Assert.Equal("不明", saleInformation.GetSalePeriod());
saleInformation.SaleStarted = true;
saleInformation.EndDate = DateTime.Parse("2017/08/31");
Assert.Equal("8/31まで", saleInformation.GetSalePeriod());
saleInformation.StartDate = DateTime.Parse("2017/08/01");
Assert.Equal("8/1～8/31", saleInformation.GetSalePeriod());
saleInformation.SaleFinished = true;
Assert.Equal("終了", saleInformation.GetSalePeriod());
}
/// <summary>
/// 投稿済みの記事を更新する
/// </summary>
void UpdateArticle(SaleInformation saleInformation)
{
try
{
var postInformation = blogger.UpdatePost(CreateArticle(saleInformation), new PostInformation()
{
Url = saleInformation.Url, PostId = saleInformation.PostId
});
saleInformation.Url = postInformation.Url;
saleInformation.PostId = postInformation.PostId;
logger?.LogInformation($"{saleInformation.Url}\n{saleInformation.PostId}");
}
catch (Exception e)
{
logger?.LogError("記事の更新失敗\n" + e.Message);
throw;
}
}
/// <summary>
/// ブログに投稿する
/// </summary>
void PostToBlog(SaleInformation saleInformation)
{
if (saleInformation.Error == true)
{
return;
}
try
{
var postInformation = blogger.InsertPost(CreateArticle(saleInformation));
saleInformation.Url = postInformation.Url;
saleInformation.PostId = postInformation.PostId;
logger?.LogInformation($"投稿完了\n{saleInformation.Url}\n{saleInformation.PostId}");
}
catch (Exception e)
{
logger?.LogError("投稿失敗\n" + e.Message);
throw;
}
}
/// <summary>
/// 記事の内容を作成する
/// </summary>
/// <param name="saleInformation"></param>
/// <returns></returns>
Article CreateArticle(SaleInformation saleInformation)
{
var article = new Article()
{
title = saleInformation.Name
};
// 既に終了済みだったらタイトルに記載する
if (saleInformation.SaleFinished == true)
{
article.title = "【終了】" + article.title;
}
// タイトルからラベルを抽出
article.labels = ExtractLabels(article.title);
article.content += $@"<p>
対象は{saleInformation.TotalResults}冊。<br>
Amazon のセールページは<a class='amazon' href='{GetAssociateLinkByBrowseNode(saleInformation.NodeId)}' target='_blank'>こちら</a>。
</p>";
article.content += @"<!--more-->";
// Bootstrap のグリッドシステムを使って配置する
// スマホでは２列で、タブレット以上では４列で表示する
int colMax = 4;
article.content += "\n<div class=\"container-fluid\">\n";
int count = 0;
foreach (var item in saleInformation.Items)
{
// ４列ごとに row で括る
if (count % colMax == 0)
{
article.content += "<div class=\"row flex\">\n";
}
article.content += $@"
<div class=""col-xs-6 col-sm-3 col-md-3 col-lg-3"">
<a class='amazon' href='{item.DetailPageUrl}' target='_blank'><img src='{item.MediumImageUrl}' /></a><br>
<a class='amazon' href='{item.DetailPageUrl}' target='_blank'>{item.Title}</a>
</div>";
article.content += "\n";
if (count % colMax == colMax - 1)
{
article.content += "</div>\n";
}
count++;
}
// アイテム数が４の倍数でない場合、最後の row が閉じられていないのでここで閉じる
if (count % colMax != 0)
{
article.content += "</div>\n";
}
article.content += "</div>\n";
if (saleInformation.TotalResults > 100)
{
article.content += $@"<p><a class='amazon' href='{GetAssociateLinkByBrowseNode(saleInformation.NodeId)}' target='_blank'>もっと見る</a></p>";
}
return(article);
}
/// <summary>
/// 指定したページのアイテム情報を取得する
/// </summary>
/// <param name="saleInformation"></param>
/// https://images-na.ssl-images-amazon.com/images/G/09/associates/paapi/dg/index.html?ItemSearch.html
bool ItemSearch(SaleInformation saleInformation, int page)
{
IDictionary <string, string> request = new Dictionary <string, String>
{
["Service"] = service,
["Version"] = apiVersion,
["Operation"] = "ItemSearch",
["SearchIndex"] = "KindleStore",
["ResponseGroup"] = "Medium",
["BrowseNode"] = saleInformation.NodeId,
["ItemPage"] = page.ToString()
};
logger?.LogInformation($"商品情報取得開始({page}ページ目)");
// リクエストを送信して xml を取得
var result = GetXml(request);
XmlDocument doc = new XmlDocument();
doc.Load(result);
XmlNamespaceManager xmlNsManager = new XmlNamespaceManager(doc.NameTable);
xmlNsManager.AddNamespace("ns", ns);
// エラー情報の取得
try
{
var error = doc.SelectSingleNode("ns:ItemSearchResponse/ns:Items/ns:Request/ns:Errors/ns:Error/ns:Code", xmlNsManager).InnerText;
logger?.LogWarning("エラー情報あり：" + error);
saleInformation.Error = true;
}
catch (Exception)
{
//Console.WriteLine("エラー情報なし：" + e.Message);
saleInformation.Error = false;
}
try
{
var totalResults = doc.SelectSingleNode("ns:ItemSearchResponse/ns:Items/ns:TotalResults", xmlNsManager).InnerText;
saleInformation.TotalResults = int.Parse(totalResults);
}
catch (Exception e)
{
logger?.LogError("TotalResults取得不可：" + e.Message);
}
// 商品情報を取得
try
{
XmlNodeList nodeList = doc.SelectNodes("ns:ItemSearchResponse/ns:Items/ns:Item", xmlNsManager);
foreach (XmlNode node in nodeList)
{
try
{
saleInformation.Items.Add(new ItemDetail()
{
Title = node.SelectSingleNode("ns:ItemAttributes/ns:Title", xmlNsManager)?.InnerText,
PublicationDate = node.SelectSingleNode("ns:ItemAttributes/ns:PublicationDate", xmlNsManager)?.InnerText,
Content = node.SelectSingleNode("ns:EditorialReviews/ns:EditorialReview/ns:Content", xmlNsManager)?.InnerText,
Asin = node.SelectSingleNode("ns:ASIN", xmlNsManager)?.InnerText,
DetailPageUrl = node.SelectSingleNode("ns:DetailPageURL", xmlNsManager)?.InnerText,
MediumImageUrl = node.SelectSingleNode("ns:MediumImage/ns:URL", xmlNsManager)?.InnerText,
LargeImageUrl = node.SelectSingleNode("ns:LargeImage/ns:URL", xmlNsManager)?.InnerText
});
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
}
}
catch (Exception e)
{
logger?.LogError("nodeListなし：" + e.Message);
}
logger?.LogInformation($"商品情報取得完了({page}ページ目、{saleInformation.Items.Count()}/{saleInformation.TotalResults}件)");
return(true);
}
