Ejemplo n.º 1
0
        private static getTaskResult DoGetLog(LogCondition logCondition, DateTime head, string url)
        {
            //var infoTmpList = new List<LogInfo>();
            long b;
            var  errorInfoTmp = new ErrorInfo {
                IsError = false, HasMore = false
            };

            var rsObj = GetLog(url, () =>
            {
                errorInfoTmp.IsError        = true;
                errorInfoTmp.ErrorRetryInfo = new RetryInfo {
                    url = url, head = head, hashedUrl = url.Message2KeyWord()
                };
                //errorInfoTmp = error;
            }, out b);

            var infoTmpList = dataProcessor(logCondition, head, b, rsObj, () =>
            {
                var urlTmp = string.Format(url + "&lastTimestamp={0}&lastScanRowKey={1}",
                                           rsObj["lastTimestamp"], rsObj["lastScanRowKey"]);

                errorInfoTmp.HasMore       = true;
                errorInfoTmp.ErrorMoreInfo = new MoreInfo {
                    url = urlTmp, head = head, hashedUrl = url.Message2KeyWord()
                };
            });

            return(new getTaskResult {
                LogInfoList = infoTmpList, LogErrorInfo = errorInfoTmp
            });                                                                                 // infoTmpList;
        }
Ejemplo n.º 2
0
        private static bool takeValueWhenMatch(LogCondition logCondition, List <LogInfo> infoTmpList, LogInfo infoTmp, string keyTmp, string valTmp)
        {
            var flag = true;

            switch (keyTmp.ToLower())
            {
            case "logtype":
                infoTmp.Logtype = valTmp;
                if (logCondition.TagValueArr.Contains(valTmp))
                {
                    //okrsObj.PrintDump();
                    //db.Logs.Add(infoTmp);
                    infoTmpList.Add(infoTmp);
                }
                else
                {
                    flag = false;
                }
                break;

            case "uid":
                infoTmp.Uid = valTmp;
                break;

            case "platform":
                infoTmp.Platform = valTmp;
                break;

            case "servicecode":
                infoTmp.Servicecode = valTmp;
                //Console.WriteLine("servicecode:" + valTmp);
                break;

            case "servicetype":
                infoTmp.Servicetype = valTmp;
                //Console.WriteLine("servicetype:" + valTmp);
                break;

            case "serviceversion":
                infoTmp.Serviceversion = valTmp;
                //Console.WriteLine("serviceversion:" + valTmp);
                break;

            case "guid":    //guid
                infoTmp.Guid = valTmp;
                break;

            case "bustype":
                infoTmp.Bustype = valTmp;
                break;

            case "orderid":
                infoTmp.Orderid = valTmp;
                break;
            }

            return(flag);
        }
Ejemplo n.º 3
0
        private static void GetAgent <T>(LogCondition logCondition,
                                         int tryCount, Func <logEntity, DbSet <T> > getSource) where T : errorInfoBase
        {
            using (var db = new logEntity())
            {
                var source = getSource(db);
                var tryTmp = source.Take(tryCount).ToList();
                //var tryTmp = source.Take(tryCount).ToList();
                if (tryTmp.Count() > 0)
                {
                    foreach (var moreGet in tryTmp)
                    {
                        var infoTmp = DoGetLogStrong(logCondition, moreGet.head, moreGet.url);

                        if (!infoTmp.LogErrorInfo.IsError)
                        {
                            db.Logs.AddRange(infoTmp.LogInfoList);//.AddRange(t.Result);

                            if (infoTmp.LogErrorInfo.HasMore)
                            {
                                //infoTmp.LogErrorInfo.ErrorMoreInfo.head = moreGet.head;
                                db.MoreInfos.Add(infoTmp.LogErrorInfo.ErrorMoreInfo);
                                //}
                            }
                            //db.MoreInfos
                            source.Remove(moreGet);
                            //db.MoreInfos.Remove(moreGet);
                            try {
                                db.SaveChanges();
                            }
                            catch (Exception e)
                            {
                                Console.WriteLine(e);
                            }
                        }
                    }
                }
            }
        }
Ejemplo n.º 4
0
        private static List <LogInfo> dataProcessor(LogCondition logCondition, DateTime head, long b, Dictionary <string, object> rsObj, Action act)
        {
            var infoTmpList = new List <LogInfo>();

            if (rsObj != null && int.Parse(rsObj["size"].ToString()) > 0)
            {
                #region db
                //using (var db = new clogEntity())
                //{
                Console.ForegroundColor = ConsoleColor.Green;
                Console.WriteLine(rsObj["size"]);

                var logs = rsObj["logs"] as List <object>;
                foreach (var log in logs)
                {
                    var logTmp      = log as Dictionary <string, object>;
                    var appIdTmp    = logTmp["appId"] as string;
                    var logLevelTmp = logTmp["logLevel"] as string;
                    var hostTmp     = logTmp["hostName"] as string;
                    //var logTyptTmp = logTmp["logType"] as string;
                    var logTimeTmp    = logTmp["timestamp"] as string;
                    var logTitleTmp   = logTmp["title"] as string;
                    var logMessageTmp = logTmp["message"] as string;

                    if (logCondition.appidLs.Contains(appIdTmp) && logCondition.LogLevelLs.Contains(logLevelTmp))
                    {
                        if (int.Parse(rsObj["size"].ToString()) == 100)
                        {
                            Console.ForegroundColor = ConsoleColor.Red;
                            Console.WriteLine(rsObj["size"]);

                            act();
                        }

                        var infoTmp = new LogInfo();
                        infoTmp.AppId     = appIdTmp;
                        infoTmp.HostIp    = (hostIp)Enum.Parse(typeof(hostIp), hostTmp);
                        infoTmp.TimeStamp = ulong.Parse(logTimeTmp);
                        infoTmp.Title     = logTitleTmp;
                        infoTmp.message   = logMessageTmp;
                        infoTmp.head      = head;

                        infoTmp.Level = (logLevel)Enum.Parse(typeof(logLevel), logLevelTmp);
                        var attrArrTmp = logTmp["attributes"] as List <object>;
                        foreach (var attr in attrArrTmp)
                        {
                            var attrTmp = attr as Dictionary <string, object>;
                            var keyTmp  = attrTmp["key"] as string;
                            var valTmp  = attrTmp["value"] as string;

                            var isMatch = takeValueWhenMatch(logCondition, infoTmpList, infoTmp, keyTmp, valTmp);
                            if (!isMatch)
                            {
                                break;
                            }
                        }
                    }
                }

                //    db.SaveChanges();
                //}
                #endregion
            }
            //string id = dyn.Id;
            //string name = dyn.Name;
            //string dob = dyn.DateOfBirth;
            var c = DateTime.Now.Ticks;
            Console.WriteLine("self:" + (c - b) / 10000);

            return(infoTmpList);
        }
Ejemplo n.º 5
0
        private static List <LogInfo> dataProcessorStrong(LogCondition logCondition, DateTime head, long b, logData rsObj, Action act)
        {
            var infoTmpList = new List <LogInfo>();

            if (rsObj != null && rsObj.size > 0)
            {
                #region db
                //using (var db = new clogEntity())
                //{
                Console.ForegroundColor = ConsoleColor.Green;
                Console.WriteLine("size=" + rsObj.size);

                var logs = rsObj.logs;
                foreach (var log in logs)
                {
                    var appIdTmp    = log.appId;       // logTmp["appId"] as string;
                    var logLevelTmp = log.logLevel;    // logTmp["logLevel"] as string;
                    var hostTmp     = log.hostName;    // logTmp["hostName"] as string;
                    //var logTyptTmp = logTmp["logType"] as string;
                    var logTimeTmp    = log.timestamp; // logTmp["timestamp"] as string;
                    var logTitleTmp   = log.title;     // logTmp["title"] as string;
                    var logMessageTmp = log.message;   // logTmp["message"] as string;

                    if (logCondition.appidLs.Contains(appIdTmp) && logCondition.LogLevelLs.Contains(logLevelTmp))
                    {
                        if (rsObj.size == 100)
                        {
                            Console.ForegroundColor = ConsoleColor.Red;
                            Console.WriteLine(rsObj.size);

                            act();
                        }

                        var attrArrTmp = log.attributes;// logTmp["attributes"] as List<object>;

                        //attrArrTmp.AsEnumerable().Aggregate(infoTmpList, (r, n) =>
                        //    {
                        //        takeValueWhenMatch(logCondition,infoTmpList , infoTmp, n.key, n.value);
                        //        return r;
                        //    });

                        Dictionary <string, string> attrDic =
                            attrArrTmp.AsEnumerable().
                            Aggregate(new Dictionary <string, string>(), (r, n) =>
                        {
                            r.Add(n.key.ToLower(), n.value);
                            return(r);
                        });

                        //Func<string, string> helpTmp = (str) => attrDic.Keys.Contains(str) ? attrDic[str] : string.Empty;
                        Func <string, string> helpExTmp = (str) =>
                        {
                            string res = string.Empty;
                            try
                            {
                                res = attrDic[str];
                            }
                            catch (Exception)
                            {
                                res = string.Empty;
                            }

                            return(res);
                        };
                        var typeTmp = helpExTmp("logtype");
                        if (logCondition.TagValueArr.Contains(typeTmp))
                        {
                            var infoTmp = new LogInfo();

                            //var infoTmp = new LogInfo();
                            infoTmp.AppId     = appIdTmp;
                            infoTmp.HostIp    = (hostIp)Enum.Parse(typeof(hostIp), hostTmp);
                            infoTmp.TimeStamp = ulong.Parse(logTimeTmp);
                            infoTmp.Title     = logTitleTmp;
                            infoTmp.message   = logMessageTmp;
                            infoTmp.head      = head;
                            infoTmp.Level     = (logLevel)Enum.Parse(typeof(logLevel), logLevelTmp);

                            infoTmp.Logtype        = helpExTmp("logtype");        // attrDic.Keys.Contains("logtype") ? attrDic["logtype"] : string.Empty;
                            infoTmp.Uid            = helpExTmp("uid");            // attrDic.Keys.Contains("uid") ? attrDic["uid"] : string.Empty;
                            infoTmp.Platform       = helpExTmp("platform");       // attrDic.Keys.Contains("platform") ? attrDic["platform"] : string.Empty;
                            infoTmp.Servicecode    = helpExTmp("servicecode");    // attrDic.Keys.Contains("servicecode") ? attrDic["servicecode"] : string.Empty;
                            infoTmp.Servicetype    = helpExTmp("servicetype");    // attrDic.Keys.Contains("servicetype") ? attrDic["servicetype"] : string.Empty;
                            infoTmp.Serviceversion = helpExTmp("serviceversion"); // attrDic.Keys.Contains("serviceversion") ? attrDic["serviceversion"] : string.Empty;
                            infoTmp.Guid           = helpExTmp("guid");           // attrDic.Keys.Contains("guid") ? attrDic["Guid"] : string.Empty;
                            infoTmp.Bustype        = helpExTmp("bustype");        // attrDic.Keys.Contains("bustype") ? attrDic["bustype"] : string.Empty;
                            infoTmp.Orderid        = helpExTmp("orderid");        // attrDic.Keys.Contains("orderid") ? attrDic["orderid"] : string.Empty;

                            infoTmpList.Add(infoTmp);
                        }
                    }
                }
                #endregion
            }
            var c = DateTime.Now.Ticks;
            Console.WriteLine("self:" + (c - b) / 10000);

            return(infoTmpList);
        }
Ejemplo n.º 6
0
        //private static void DoRetryToGetData(List<string> logLevelLs, List<string> appidLs, string[] tagValueArr, int tryCount)
        //{
        //    using (var db = new clogEntity())
        //    {
        //        var tryTmp = db.RetryInfos.Take(tryCount).ToList();
        //        if (tryTmp.Count() > 0)
        //        {
        //            foreach (var toTry in tryTmp)
        //            {
        //                var infoTmp = DoGetLog(logLevelLs, appidLs, tagValueArr, toTry.head, toTry.url);

        //                if (!infoTmp.LogErrorInfo.IsError)
        //                {
        //                    db.Logs.AddRange(infoTmp.LogInfoList);//.AddRange(t.Result);

        //                    if (infoTmp.LogErrorInfo.HasMore)
        //                    {
        //                        infoTmp.LogErrorInfo.ErrorMoreInfo.head = toTry.head;
        //                        db.MoreInfos.Add(infoTmp.LogErrorInfo.ErrorMoreInfo);
        //                        //}
        //                    }
        //                    db.RetryInfos.Remove(toTry);
        //                    db.SaveChanges();
        //                }
        //            }
        //        }
        //    }
        //}

        private static void GetData(LogCondition logCondition)
        {
            #region prepare for time
            TimeTable endTmp = null;
            using (var db = new logEntity())
            {
                if (db.Times.Count() < 1)
                {
                    endTmp       = new TimeTable();
                    endTmp.start = new DateTime(2015, 1, 16, 16, 0, 0);// headTime.end;
                    endTmp.end   = endTmp.start.AddSeconds(30);
                    endTmp.head  = new DateTime(2015, 1, 16, 16, 0, 0);
                    //db.Times.Add(endTmp);
                    //db.SaveChanges();
                }
                else
                {
                    //var headTime =
                    //    (from ht in db.Times
                    //    group ht by ht.id into grp
                    //    select grp.OrderByDescending(g => g.end).FirstOrDefault()).First();

                    var headTime = db.Times.OrderByDescending(g => g.end).First();
                    endTmp       = new TimeTable();
                    endTmp.start = headTime.end;
                    endTmp.end   = headTime.end.AddSeconds(30);
                    endTmp.head  = endTmp.start.Month != headTime.start.Month ? endTmp.start : headTime.head;
                    if (endTmp.end > DateTime.Now.AddMinutes(-1))
                    {
                        return;
                    }

                    //db.Times.Add(endTmp);
                    //db.SaveChanges();
                }
            }
            #endregion
            //var from = endTmp.start;// new DateTime(2014, 8, 26);
            var newFrom = endTmp.start;
            var newTo   = endTmp.end;

            if (newTo > DateTime.Now.AddMinutes(-5))
            {
                Thread.Sleep(new TimeSpan(0, 5, 0));
                return;
            }

            var urlLs = getUrlListByMinutes(logCondition.HostList, newFrom, newTo, logCondition.appidLs); //getUrlListByMinutes(hostList, logLevelLs, newFrom, newTo, appidLs, tagValueLs);
            //urlLs = getUrlListByMinutes(hostList, "2014-09-03 10:50:13", "2014-09-03 10:52:13", appidArr);
            Random rnd = new Random();

            List <Task <getTaskResult> > tasks = new List <Task <getTaskResult> >();
            // Execute the task 10 times.
            foreach (var url in urlLs)
            {
                tasks.Add(Task <getTaskResult> .Factory.StartNew(() =>
                {
                    //RetryInfo retryTmp = null;
                    var taskRes = DoGetLogStrong(logCondition, endTmp.head, url);

                    return(taskRes);// new getTaskResult { LogInfoList = taskRes.LogInfoList, LogErrorInfo = new ErrorInfo { ErrorRetryInfo = retryTmp } };
                }));
            }

            var t1 = DateTime.Now.Ticks;

            Task.WaitAll(tasks.ToArray());

            var t2 = DateTime.Now.Ticks;

            Console.ForegroundColor = ConsoleColor.DarkYellow;
            Console.WriteLine("Task totle time = " + ((t2 - t1) / 10000000).ToString());

            using (var db = new logEntity())
            {
                foreach (var t in tasks)
                {
                    db.Logs.AddRange(t.Result.LogInfoList);
                    //if(t.Result.retryInfo != null)
                    //{

                    if (t.Result.LogErrorInfo.IsError)
                    {
                        t.Result.LogErrorInfo.ErrorRetryInfo.head = endTmp.head;
                        db.RetryInfos.Add(t.Result.LogErrorInfo.ErrorRetryInfo);
                        //}
                    }
                    if (t.Result.LogErrorInfo.HasMore)
                    {
                        t.Result.LogErrorInfo.ErrorMoreInfo.head = endTmp.head;
                        db.MoreInfos.Add(t.Result.LogErrorInfo.ErrorMoreInfo);
                        //}
                    }
                }
                db.Times.Add(endTmp);
                try {
                    db.SaveChanges();
                }
                catch (DbEntityValidationException e)
                {
                    Console.WriteLine(e);
                }
            }
        }