Exemple #1
0
        protected override void Save(FeedRequest fr, Response response)
        {
            if (response == null || response.StatusCode != HttpStatusCode.OK)
            {
                return;
            }

            if (response.IsRaw)
            {
                return;
            }

            var content = base.ConvertEncoding(response.Data.ToString(), Encoding.GetEncoding(response.Charset), Encoding.UTF8);
            var setting = JsonConvert.DeserializeObject <FeedSetting>(response.Request.Tag);

            var snapshot = new Snapshot
            {
                FeedId      = setting.Id,
                RequestUrl  = fr.Request.Uri.ToString(),
                ResponseUrl = response.ResponseUri.ToString(),
                Content     = content,
                Expression  = fr.Expression
            };

            var json = JsonConvert.SerializeObject(snapshot, Formatting.Indented);

            var fileName = Path.Combine(snapshotPath, setting.Id + "_" + DateTime.Now.Ticks + ".json");

            if (setting.Delay > 0)
            {
                fileName = Path.Combine(delayPath, setting.Id + "_" + DateTime.Now.AddMinutes(setting.Delay).Ticks + ".json");
            }

            Logger.GetLogger("").Info(snapshot.RequestUrl + " response save to " + fileName);

            File.WriteAllText(fileName, json, Encoding.UTF8);
        }
Exemple #2
0
        protected override List <FeedRequest> GetRequests()
        {
            Logger.GetLogger("").Info("start get feed");

            try
            {
                var requests = new List <FeedRequest>();
                var compile  = new UrlCompile();
                var files    = Directory.GetFiles(jobPath);

                foreach (var file in files)
                {
                    var extension = Path.GetExtension(file).ToLower();
                    if (extension != ".feed")
                    {
                        continue;
                    }

                    var parser = new RuiJiParser();
                    var result = parser.ParseFile(file);

                    if (result)
                    {
                        var request = parser.GetResult <Request>().Result;
                        var setting = parser.GetResult <FeedSetting>().Result;

                        if (request == null || setting == null)
                        {
                            continue;
                        }

                        var addrs = compile.GetResult(request.Uri.ToString());

                        for (int i = 0; i < addrs.Length; i++)
                        {
                            var addr = addrs[i].ToString();

                            var r = request.Clone() as Request;
                            r.Uri       = new Uri(addr);
                            setting.Id += "_" + i;
                            r.Tag       = JsonConvert.SerializeObject(setting);

                            var fr = new FeedRequest();
                            fr.Request    = r;
                            fr.Setting    = setting;
                            fr.Expression = parser.GetResult <ExtractBlock>().Expression;

                            requests.Add(fr);
                        }
                    }
                }

                return(requests);
            }
            catch (Exception ex)
            {
                Logger.GetLogger("").Info("get feed error " + ex.Message);

                return(new List <FeedRequest>());
            }
        }
Exemple #3
0
 protected abstract void Save(FeedRequest request, Response response);