protected override void Save(FeedRequest fr, Response response)
{
if (response == null || response.StatusCode != HttpStatusCode.OK)
{
return;
}
if (response.IsRaw)
{
return;
}
var content = base.ConvertEncoding(response.Data.ToString(), Encoding.GetEncoding(response.Charset), Encoding.UTF8);
var setting = JsonConvert.DeserializeObject <FeedSetting>(response.Request.Tag);
var snapshot = new Snapshot
{
FeedId = setting.Id,
RequestUrl = fr.Request.Uri.ToString(),
ResponseUrl = response.ResponseUri.ToString(),
Content = content,
Expression = fr.Expression
};
var json = JsonConvert.SerializeObject(snapshot, Formatting.Indented);
var fileName = Path.Combine(snapshotPath, setting.Id + "_" + DateTime.Now.Ticks + ".json");
if (setting.Delay > 0)
{
fileName = Path.Combine(delayPath, setting.Id + "_" + DateTime.Now.AddMinutes(setting.Delay).Ticks + ".json");
}
Logger.GetLogger("").Info(snapshot.RequestUrl + " response save to " + fileName);
File.WriteAllText(fileName, json, Encoding.UTF8);
}
protected override List <FeedRequest> GetRequests()
{
Logger.GetLogger("").Info("start get feed");
try
{
var requests = new List <FeedRequest>();
var compile = new UrlCompile();
var files = Directory.GetFiles(jobPath);
foreach (var file in files)
{
var extension = Path.GetExtension(file).ToLower();
if (extension != ".feed")
{
continue;
}
var parser = new RuiJiParser();
var result = parser.ParseFile(file);
if (result)
{
var request = parser.GetResult <Request>().Result;
var setting = parser.GetResult <FeedSetting>().Result;
if (request == null || setting == null)
{
continue;
}
var addrs = compile.GetResult(request.Uri.ToString());
for (int i = 0; i < addrs.Length; i++)
{
var addr = addrs[i].ToString();
var r = request.Clone() as Request;
r.Uri = new Uri(addr);
setting.Id += "_" + i;
r.Tag = JsonConvert.SerializeObject(setting);
var fr = new FeedRequest();
fr.Request = r;
fr.Setting = setting;
fr.Expression = parser.GetResult <ExtractBlock>().Expression;
requests.Add(fr);
}
}
}
return(requests);
}
catch (Exception ex)
{
Logger.GetLogger("").Info("get feed error " + ex.Message);
return(new List <FeedRequest>());
}
}
protected abstract void Save(FeedRequest request, Response response);