Ejemplo n.º 1
0
        public void InsertUseAppConfig()
        {
            ClearDb();

            using (MySqlConnection conn = new MySqlConnection(ConnectString))
            {
                ISpider spider = new DefaultSpider("test", new Site());

                MySqlEntityPipeline insertPipeline = new MySqlEntityPipeline();
                var metadata = EntityDefine.Parse <ProductInsert>();
                insertPipeline.AddEntity(metadata);
                insertPipeline.InitPipeline(spider);

                // Common data
                DataObject data1 = new DataObject {
                    { "Sku", "110" }, { "Category", "3C" }, { "Url", "http://jd.com/110" }, { "CDate", "2016-08-13" }
                };
                DataObject data2 = new DataObject {
                    { "Sku", "111" }, { "Category", "3C" }, { "Url", "http://jd.com/111" }, { "CDate", "2016-08-13" }
                };
                // Value is null
                DataObject data3 = new DataObject {
                    { "Sku", "112" }, { "Category", null }, { "Url", "http://jd.com/111" }, { "CDate", "2016-08-13" }
                };
                insertPipeline.Process(metadata.Name, new List <DataObject> {
                    data1, data2, data3
                });

                var list = conn.Query <ProductInsert>($"select * from test.sku_{DateTime.Now.ToString("yyyy_MM_dd")}").ToList();
                Assert.Equal(3, list.Count);
                Assert.Equal("110", list[0].Sku);
                Assert.Equal("111", list[1].Sku);
                Assert.Null(list[2].Category);
            }

            ClearDb();
        }
Ejemplo n.º 2
0
        public void InsertUseAppConfig()
        {
            ClearDb();

            using (MySqlConnection conn = new MySqlConnection(ConnectString))
            {
                ISpider spider = new DefaultSpider("test", new Site());

                MySqlEntityPipeline insertPipeline = new MySqlEntityPipeline();
                var metadata = new EntityDefine <ProductInsert>();
                insertPipeline.AddEntity(metadata);
                insertPipeline.InitPipeline(spider);

                var data1 = new ProductInsert {
                    Sku = "110", Category = "3C", Url = "http://jd.com/110", CDate = new DateTime(2016, 8, 13)
                };
                var data2 = new ProductInsert {
                    Sku = "111", Category = "3C", Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
                };
                var data3 = new ProductInsert {
                    Sku = "112", Category = null, Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
                };

                insertPipeline.Process(metadata.Name, new List <dynamic> {
                    data1, data2, data3
                });

                var list = conn.Query <ProductInsert>($"select * from test.sku_{DateTime.Now.ToString("yyyy_MM_dd")}").ToList();
                Assert.Equal(3, list.Count);
                Assert.Equal("110", list[0].Sku);
                Assert.Equal("111", list[1].Sku);
                Assert.Null(list[2].Category);
            }

            ClearDb();
        }
Ejemplo n.º 3
0
        public void Extract()
        {
            var             entityMetadata = EntityDefine.Parse <Product>();
            EntityExtractor extractor      = new EntityExtractor("test", null, entityMetadata);
            var             results        = extractor.Extract(new Page(new Request("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, dynamic>
            {
                { "cat", "手机" },
                { "cat3", "110" }
            }), null)
            {
                Content = File.ReadAllText(Path.Combine(Env.BaseDirectory, "Jd.html"))
            });

            Assert.Equal(60, results.Count);
            Assert.Equal("手机", results[0]["CategoryName"]);
            Assert.Equal("110", results[0]["CategoryId"]);
            Assert.Equal("http://item.jd.com/3031737.html", results[0]["Url"]);
            Assert.Equal("3031737", results[0]["Sku"]);
            Assert.Equal("荣耀官方旗舰店", results[0]["ShopName"]);
            Assert.Equal("荣耀 NOTE 8 4GB+32GB 全网通版 冰河银", results[0]["Name"]);
            Assert.Equal("1000000904", results[0]["VenderId"]);
            Assert.Equal("1000000904", results[0]["JdzyShopId"]);
            Assert.Equal(DateTime.Now.ToString("yyyy-MM-dd"), results[0]["RunId"]);
        }
Ejemplo n.º 4
0
        public static void Update(this EntitySyncComponent self)
        {
            if (TimeHelper.Now() - self.Timer > self.Interval)
            {
                self.Timer = TimeHelper.Now();
                var transform = self.Parent.GetComponent <TransformComponent>();
                var lp        = transform.lastPosition;
                var p         = transform.position;
                if (Vector3.Distance(lp, p) < 0.1f)
                {
                    return;
                }
                transform.lastPosition = p;

                var msg = new M2C_OnEntityChanged();
                msg.EntityId   = self.Id;
                msg.EntityType = EntityDefine.GetTypeId(self.Parent.GetType());

                msg.X = (int)(p.x * 100);
                msg.Y = (int)(p.y * 100);
                msg.Z = (int)(p.z * 100);
                MessageHelper.Broadcast(self.Domain, msg);
            }
        }
 public override void AddEntity(EntityDefine metadata)
 {
 }
Ejemplo n.º 6
0
        public static void OnPropertyChanged(Entity entity, string propertyName, byte[] valueBytes)
        {
            var    type      = entity.GetType();
            Entity component = null;

            if (EntityDefine.IsComponent(type))
            {
                component = entity;
                entity    = entity.GetParent <Entity>();
            }

            var entityTypeId = EntityDefine.GetTypeId(entity.GetType());
            //if (component != null)
            //{

            //}
            //if (!EntityDefine.PropertyDefineCollectionMap.ContainsKey(entityTypeId))
            //{
            //    Log.Error($"EntityDefine.PropertyDefineCollectionMap has no key {entityTypeId}");
            //    return;
            //}
            //if (!EntityDefine.PropertyDefineCollectionMap[entityTypeId].ContainsKey(propertyName))
            //{
            //    Log.Error($"EntityDefine.PropertyDefineCollectionMap[{entityTypeId}] has no key {propertyName}");
            //    return;
            //}
            var msg = new M2C_OnEntityChanged();
            PropertyDefineAttribute attr;

            if (component != null)
            {
                msg.ComponentType = EntityDefine.GetTypeId(component.GetType());
                attr = EntityDefine.PropertyDefineCollectionMap[msg.ComponentType][propertyName];
            }
            else
            {
                attr = EntityDefine.PropertyDefineCollectionMap[entityTypeId][propertyName];
            }
            msg.EntityId            = entity.Id;
            msg.EntityType          = entityTypeId;
            msg.PropertyId          = attr.Id;
            msg.PropertyValue.bytes = valueBytes;

            Log.Error($"MessageHelper OnPropertyChanged {type} {propertyName} flag={attr.Flag} ");

            if (attr.Flag == SyncFlag.AllClients)
            {
                Broadcast(entity.Domain, msg);
            }
            if (attr.Flag == SyncFlag.OtherClients)
            {
                if (entity is Unit u)
                {
                    BroadcastToOther(u, msg);
                }
            }
            if (attr.Flag == SyncFlag.OwnClient)
            {
                if (entity is Unit u)
                {
                    Send(u, msg);
                }
            }
        }
Ejemplo n.º 7
0
        public void InsertAndIgnoreDuplicate()
        {
#if NETSTANDARD
            if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
            {
                return;
            }
#endif
            ClearDb();

            ISpider spider = new DefaultSpider("test", new Site());

            CassandraEntityPipeline insertPipeline = new CassandraEntityPipeline(connectString);
            var metadata = new EntityDefine <ProductInsert>();
            insertPipeline.AddEntity(metadata);
            insertPipeline.Init();

            // Common data
            var data1 = new ProductInsert {
                Sku = "110", Category = "3C", Url = "http://jd.com/110", CDate = new DateTime(2016, 8, 13)
            };
            var data2 = new ProductInsert {
                Sku = "111", Category = "3C", Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
            };
            var data3 = new ProductInsert {
                Sku = "112", Category = null, Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
            };
            // Value is null
            insertPipeline.Process(metadata.Name, new List <dynamic> {
                data1, data2, data3
            }, spider);

            var cluster = CassandraUtil.CreateCluster(connectString);

            var session = cluster.Connect();
            session.ChangeKeyspace("test");
            var rows    = session.Execute($"SELECT * FROM test.sku_cassandra_{DateTime.Now.ToString("yyyy_MM_dd")}").GetRows().ToList();
            var results = new List <ProductInsert>();
            foreach (var row in rows)
            {
                results.Add(new ProductInsert
                {
                    Sku      = row.GetValue <string>("sku"),
                    Category = row.GetValue <string>("category"),
                    Id       = row.GetValue <Guid>("id")
                });
            }
            insertPipeline.DefaultPipelineModel = PipelineMode.InsertAndIgnoreDuplicate;
            var sku   = results.First().Sku;
            var data4 = new ProductInsert {
                Id = results.First().Id, Sku = "113", Category = "asdfasf", Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
            };
            insertPipeline.Process(metadata.Name, new List <dynamic> {
                data4
            }, spider);

            rows    = session.Execute($"SELECT * FROM test.sku_cassandra_{DateTime.Now.ToString("yyyy_MM_dd")}").GetRows().ToList();
            results = new List <ProductInsert>();
            foreach (var row in rows)
            {
                results.Add(new ProductInsert
                {
                    Sku      = row.GetValue <string>("sku"),
                    Category = row.GetValue <string>("category")
                });
            }
            Assert.Equal(3, results.Count);
            Assert.DoesNotContain(results, r => r.Sku == sku);

            Assert.Contains(results, r => r.Sku == "113");
            Assert.Contains(results, r => r.Category == "asdfasf");

            ClearDb();
        }
Ejemplo n.º 8
0
        /// <summary>
        ///  1, 匹配 TargetUrl,判断是否需要解析HTML
        ///  2,匹配 HelperUrl,判断是否需要加入到请求列表
        /// </summary>
        /// <typeparam name="TEntity"></typeparam>
        /// <param name="entity"></param>
        /// <param name="entityDefine"></param>
        /// <param name="page"></param>
        /// <returns></returns>
        public TEntity Process <TEntity>(TEntity entity, EntityDefine entityDefine, Page page) where TEntity : IEntity
        {
            if (string.IsNullOrEmpty(page.Content))
            {
                return(entity);
            }

            // 查找所有链接
            var allLinkEle = page.Selector?.SelectorAll("a", Html.HtmlSelectorPathType.Css);

            List <string> links = new List <string>();

            if (allLinkEle != null)
            {
                var r = allLinkEle.Select(t => t.GetAttribute("href")).Where(t => !string.IsNullOrEmpty(t)).Distinct().ToList();

                if (r.Count > 0)
                {
                    links.AddRange(r);
                }
            }

            // 匹配新请求
            if (entityDefine.HelperUrls?.Length > 0)
            {
                foreach (var url in entityDefine.HelperUrls)
                {
                    foreach (var link in links)
                    {
                        if (url == link)
                        {
                            continue;
                        }

                        Regex regex = new Regex(url);

                        if (regex.IsMatch(link))
                        {
                            page.AddTargetRequest(link, page.Response.Request.Uri.ToString());
                        }
                    }
                }
            }

            if (entityDefine.TargetUrls == null || entityDefine.TargetUrls.Length == 0)
            {
                return(entity);
            }


            foreach (var url in entityDefine.TargetUrls)
            {
                foreach (var link in links)
                {
                    if (url == link)
                    {
                        continue;
                    }

                    Regex regex = new Regex(url);

                    if (regex.IsMatch(link))
                    {
                        page.AddTargetRequest(link, page.Response.Request.Uri.ToString());
                    }
                }
            }

            // 匹配内容

            var requestUrl = page.Response.Request.Uri.ToString();

            foreach (var url in entityDefine.TargetUrls)
            {
                Regex regex = new Regex(url);

                if (regex.IsMatch(requestUrl))
                {
                    MatchSelector(entityDefine, entity, page);
                }
            }

            return(entity);
        }
Ejemplo n.º 9
0
        public override async ETTask G2M_CreateUnitHandler(Scene scene, G2M_CreateUnit request, M2G_CreateUnit response, Action reply)
        {
            var copyMap = Game.Scene.Children.Values.ToList().Find((x) =>
            {
                if (x is Scene s)
                {
                    return(s.Name == "CopyMap1");
                }
                return(false);
            });

            if (copyMap == null)
            {
                var copyMapConfig = StartConfigComponent.Instance.GetByName("CopyMap1");
                copyMap = await SceneFactory.Create(Game.Scene, copyMapConfig.GetComponent <SceneConfig>().Name, SceneType.Map);
            }

            Unit unit = null;

            if (request.UnitId != 0)
            {
                unit = await DBComponent.Instance.Query <Unit>(request.UnitId);

                unit.Domain = copyMap;
            }
            else
            {
                unit          = EntityFactory.CreateWithId <Unit>(copyMap, IdGenerater.GenerateId());
                unit.PlayerId = request.PlayerId;
                unit.Setup();
                unit.Save().Coroutine();
            }

            unit.AddComponent <MoveComponent>();
            unit.AddComponent <Body2dComponent>().CreateBody(.5f, .5f);
            unit.AddComponent <MailBoxComponent>();
            await unit.AddLocation();

            unit.AddComponent <UnitGateComponent, long>(request.GateSessionId);
            copyMap.GetComponent <UnitComponent>().Add(unit);
            response.UnitId = unit.Id;

            // 广播创建的unit
            var inViewUnitsMsg = new M2C_InViewUnits();
            var enterViewMsg   = new M2C_OnEnterView();

            Unit[] units = copyMap.GetComponent <UnitComponent>().GetAll();
            foreach (Unit u in units)
            {
                var entityInfo = new EntiyInfo();
                entityInfo.BsonBytes       = new Google.Protobuf.ByteString();
                entityInfo.BsonBytes.bytes = MongoHelper.ToBson(u);
                entityInfo.Type            = EntityDefine.GetTypeId <Unit>();
                if (u.Id == unit.Id)
                {
                    enterViewMsg.EnterEntity = entityInfo;
                    inViewUnitsMsg.SelfUnit  = entityInfo.BsonBytes;
                    continue;
                }
                inViewUnitsMsg.InViewEntitys.Add(entityInfo);
            }
            var monsters = copyMap.GetComponent <MonsterComponent>().GetAll();

            foreach (var u in monsters)
            {
                var entityInfo = new EntiyInfo();
                entityInfo.BsonBytes       = new Google.Protobuf.ByteString();
                entityInfo.BsonBytes.bytes = MongoHelper.ToBson(u);
                entityInfo.Type            = EntityDefine.GetTypeId <Monster>();
                inViewUnitsMsg.InViewEntitys.Add(entityInfo);
            }
            MessageHelper.BroadcastToOther(unit, enterViewMsg);
            MessageHelper.Send(unit, inViewUnitsMsg);
            reply();
        }
Ejemplo n.º 10
0
        public static void Start()
        {
#if ILRuntime
            if (!Define.IsILRuntime)
            {
                Log.Error("mono层是mono模式, 但是Hotfix层是ILRuntime模式");
            }
#else
            if (Define.IsILRuntime)
            {
                Log.Error("mono层是ILRuntime模式, Hotfix层是mono模式");
            }
#endif

            try
            {
                // 注册热更层回调
                ETModel.Game.Hotfix.Update            = () => { Update(); };
                ETModel.Game.Hotfix.LateUpdate        = () => { LateUpdate(); };
                ETModel.Game.Hotfix.OnApplicationQuit = () => { OnApplicationQuit(); };

                OpcodeHelper.ignoreDebugLogMessageSet.Add(HotfixOpcode.UnitOperation);
                OpcodeHelper.ignoreDebugLogMessageSet.Add(HotfixOpcode.M2C_OnEntityChanged);

                Game.Scene.AddComponent <UIComponent>();
                Game.Scene.AddComponent <OpcodeTypeComponent>();
                Game.Scene.AddComponent <MessageDispatcherComponent>();

                // 加载热更配置
                ETModel.Game.Scene.GetComponent <ResourcesComponent>().LoadBundle("config.unity3d");
                Game.Scene.AddComponent <ConfigComponent>();
                ETModel.Game.Scene.GetComponent <ResourcesComponent>().UnloadBundle("config.unity3d");

                UnitConfig unitConfig = (UnitConfig)Game.Scene.GetComponent <ConfigComponent>().Get(typeof(UnitConfig), 1001);
                Log.Debug($"config {JsonHelper.ToJson(unitConfig)}");

                Game.EventSystem.Run(EventIdType.InitSceneStart);

                HandlersHelper.Instance         = new HandlersHelper();
                EntityDefine.OnPropertyChanged += (entity, name, value) =>
                {
                    Log.Debug($"OnPropertyChanged {name}");
                    var propertyDefineCollection = EntityDefine.PropertyDefineCollectionMap[EntityDefine.GetTypeId <Unit>()];
                    var attr = propertyDefineCollection[name];
                    var msg  = new C2M_SetEntityProperty();
                    msg.PropertyId          = attr.Id;
                    msg.PropertyValue.bytes = value;
                    SessionHelper.HotfixSend(msg);
                };
            }
            catch (Exception e)
            {
                Log.Error(e);
            }
        }