public static List <UrlStoreModel> GetHaveContentUrls(string rootUrl)
{
if (string.IsNullOrEmpty(rootUrl))
{
return(null);
}
List <UrlStoreModel> urls = new List <UrlStoreModel>();
RedisUtil.Exec(c =>
{
HashEntry[] items = c.HashGetAll(GetBeforeUrlKey(rootUrl));
UrlStoreModel model = null;
Tuple <int, string> value = null;
foreach (HashEntry item in items)
{
value = ParseHashValue(item.Value.ToString());
model = new UrlStoreModel()
{
Url = item.Name.ToString(),
Level = value.Item1,
localFileName = value.Item2
};
if (model.IsLocalization())
{
urls.Add(model);
}
}
}, db);
return(urls);
}
public void Process()
{
UrlStoreModel firstUrl = UrlOperate.GetFirstUrl(this._ruleConfig.Url);
if (firstUrl == null)
{
return;
}
Console.WriteLine("Current url {0}.", firstUrl.Url);
ArticleModel model = null;
try
{
string htmlContent = this.GetLocalContent(firstUrl);
if (string.IsNullOrEmpty(htmlContent))
{
Console.WriteLine("Load html content from url.");
model = this._articleBuilder.BuildModel(firstUrl.Url, this._currentRule);
}
else
{
Console.WriteLine("Load html content from local.");
model = this._articleBuilder.BuildModel(firstUrl.Url, htmlContent, this._currentRule);
}
Console.WriteLine("Current article title : {0}.", model != null ? model.Title : "");
}
catch (Exception ex)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Article build failed : " + ex.Message);
Console.ResetColor();
LogHelper.WriteError("Url store info : " + JsonConvert.SerializeObject(firstUrl) + "/r/nArticle build failed : " + ex.Message);
}
//db
int articleId = this._sqlOperate.AddArticleModel(model);
Console.WriteLine("Insert article {0} to db.", articleId);
UrlOperate.MoveUrl(this._ruleConfig.Url, firstUrl);
Console.WriteLine("Move url {0}.", firstUrl.Url);
this.Process();
}
public static bool MoveUrl(string rootUrl, UrlStoreModel urlModel)
{
if (string.IsNullOrEmpty(rootUrl) || urlModel == null)
{
return(false);
}
bool succ = false;
RedisUtil.Exec(c =>
{
if (c.HashDelete(GetBeforeUrlKey(rootUrl), urlModel.Url))
{
succ = c.HashSet(GetAfterUrlKey(rootUrl), urlModel.Url, GetHashValue(urlModel.Level, urlModel.localFileName));
}
}, db);
return(succ);
}
private string GetLocalContent(UrlStoreModel firstUrl)
{
if (firstUrl == null)
{
return(null);
}
string htmlContent = null;
if (firstUrl.IsLocalization())
{
_fileOperate.Read(this._ruleConfig.Url, firstUrl.localFileName);
if (_fileOperate.IsExists())
{
htmlContent = _fileOperate.GetHtml();
}
}
return(htmlContent);
}
