public object UAs(int offset, int limit, int groupId)
{
var paging = new Paging();
paging.CurrentPage = (offset / limit) + 1;
paging.PageSize = limit;
return(new
{
rows = UALiteDb.GetModels(paging, groupId),
total = paging.Count
});
}
public FeedSnapshot DoTask(FeedModel feed, bool persistence = false)
{
try
{
Logger.GetLogger(baseUrl).Info("do task -> request address " + feed.Address);
var request = new Request(feed.Address);
request.RunJS = (feed.RunJS == Status.ON);
if (feed.Headers != null)
{
request.Headers = feed.Headers;
if (request.Headers.Count(m => m.Name == "Referer") == 0)
{
request.Headers.Add(new WebHeader("Referer", request.Uri.AbsoluteUri));
}
}
request.Method = feed.Method;
if (feed.Method == "POST" && !string.IsNullOrEmpty(feed.Data))
{
request.Data = feed.Data;
}
var ua = UALiteDb.GetOne();
if (!string.IsNullOrEmpty(ua))
{
request.Headers.Add(new WebHeader("User-Agent", ua));
}
var response = NodeVisitor.Crawler.Request(request);
if (response != null)
{
Logger.GetLogger(baseUrl).Info("request " + feed.Address + " response code is " + response.StatusCode);
}
if (response == null)
{
Logger.GetLogger(baseUrl).Error("request " + feed.Address + " response is null");
}
if (response != null && response.StatusCode == HttpStatusCode.OK)
{
var content = Convert(response.Data.ToString(), Encoding.GetEncoding(response.Charset), Encoding.UTF8);
var snap = new FeedSnapshot
{
Url = feed.Address,
Content = content,
Type = feed.Type,
BlockExpression = feed.BlockExpression,
RuiJiExpression = feed.RuiJiExpression
};
if (persistence)
{
var json = JsonConvert.SerializeObject(snap, Formatting.Indented);
var fileName = baseDir + @"snapshot\" + feed.Id + "_" + DateTime.Now.Ticks + ".json";
if (feed.Delay > 0)
{
fileName = baseDir + @"delay\" + feed.Id + "_" + DateTime.Now.AddMinutes(feed.Delay).Ticks + ".json";
}
Logger.GetLogger(baseUrl).Info(feed.Address + " response save to " + fileName);
File.WriteAllText(fileName, json, Encoding.UTF8);
}
return(snap);
}
}
catch (Exception ex)
{
Logger.GetLogger(baseUrl).Info("do task -> request address failed " + ex.Message);
}
return(null);
}
public bool RemoveUAs(string ids)
{
var removes = ids.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries).Select(m => Convert.ToInt32(m)).ToArray();
return(UALiteDb.Remove(removes));
}
public bool UpdateUA(UAModel ua)
{
UALiteDb.AddOrUpdate(ua);
return(true);
}
public object RandomUA()
{
return(UALiteDb.GetOne());
}
public object UA(int id)
{
return(UALiteDb.Get(id));
}
public bool RemoveUAGroup(int id)
{
return(UALiteDb.RemoveByGorup(id) ? UAGroupLiteDb.Remove(id) : false);
}
