public void InsertUseAppConfig()
{
ClearDb();
using (MySqlConnection conn = new MySqlConnection(ConnectString))
{
ISpider spider = new DefaultSpider("test", new Site());
MySqlEntityPipeline insertPipeline = new MySqlEntityPipeline();
var metadata = EntityDefine.Parse <ProductInsert>();
insertPipeline.AddEntity(metadata);
insertPipeline.InitPipeline(spider);
// Common data
DataObject data1 = new DataObject {
{ "Sku", "110" }, { "Category", "3C" }, { "Url", "http://jd.com/110" }, { "CDate", "2016-08-13" }
};
DataObject data2 = new DataObject {
{ "Sku", "111" }, { "Category", "3C" }, { "Url", "http://jd.com/111" }, { "CDate", "2016-08-13" }
};
// Value is null
DataObject data3 = new DataObject {
{ "Sku", "112" }, { "Category", null }, { "Url", "http://jd.com/111" }, { "CDate", "2016-08-13" }
};
insertPipeline.Process(metadata.Name, new List <DataObject> {
data1, data2, data3
});
var list = conn.Query <ProductInsert>($"select * from test.sku_{DateTime.Now.ToString("yyyy_MM_dd")}").ToList();
Assert.Equal(3, list.Count);
Assert.Equal("110", list[0].Sku);
Assert.Equal("111", list[1].Sku);
Assert.Null(list[2].Category);
}
ClearDb();
}
public void InsertUseAppConfig()
{
ClearDb();
using (MySqlConnection conn = new MySqlConnection(ConnectString))
{
ISpider spider = new DefaultSpider("test", new Site());
MySqlEntityPipeline insertPipeline = new MySqlEntityPipeline();
var metadata = new EntityDefine <ProductInsert>();
insertPipeline.AddEntity(metadata);
insertPipeline.InitPipeline(spider);
var data1 = new ProductInsert {
Sku = "110", Category = "3C", Url = "http://jd.com/110", CDate = new DateTime(2016, 8, 13)
};
var data2 = new ProductInsert {
Sku = "111", Category = "3C", Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
};
var data3 = new ProductInsert {
Sku = "112", Category = null, Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
};
insertPipeline.Process(metadata.Name, new List <dynamic> {
data1, data2, data3
});
var list = conn.Query <ProductInsert>($"select * from test.sku_{DateTime.Now.ToString("yyyy_MM_dd")}").ToList();
Assert.Equal(3, list.Count);
Assert.Equal("110", list[0].Sku);
Assert.Equal("111", list[1].Sku);
Assert.Null(list[2].Category);
}
ClearDb();
}
public void Extract()
{
var entityMetadata = EntityDefine.Parse <Product>();
EntityExtractor extractor = new EntityExtractor("test", null, entityMetadata);
var results = extractor.Extract(new Page(new Request("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, dynamic>
{
{ "cat", "手机" },
{ "cat3", "110" }
}), null)
{
Content = File.ReadAllText(Path.Combine(Env.BaseDirectory, "Jd.html"))
});
Assert.Equal(60, results.Count);
Assert.Equal("手机", results[0]["CategoryName"]);
Assert.Equal("110", results[0]["CategoryId"]);
Assert.Equal("http://item.jd.com/3031737.html", results[0]["Url"]);
Assert.Equal("3031737", results[0]["Sku"]);
Assert.Equal("荣耀官方旗舰店", results[0]["ShopName"]);
Assert.Equal("荣耀 NOTE 8 4GB+32GB 全网通版 冰河银", results[0]["Name"]);
Assert.Equal("1000000904", results[0]["VenderId"]);
Assert.Equal("1000000904", results[0]["JdzyShopId"]);
Assert.Equal(DateTime.Now.ToString("yyyy-MM-dd"), results[0]["RunId"]);
}
public static void Update(this EntitySyncComponent self)
{
if (TimeHelper.Now() - self.Timer > self.Interval)
{
self.Timer = TimeHelper.Now();
var transform = self.Parent.GetComponent <TransformComponent>();
var lp = transform.lastPosition;
var p = transform.position;
if (Vector3.Distance(lp, p) < 0.1f)
{
return;
}
transform.lastPosition = p;
var msg = new M2C_OnEntityChanged();
msg.EntityId = self.Id;
msg.EntityType = EntityDefine.GetTypeId(self.Parent.GetType());
msg.X = (int)(p.x * 100);
msg.Y = (int)(p.y * 100);
msg.Z = (int)(p.z * 100);
MessageHelper.Broadcast(self.Domain, msg);
}
}
public override void AddEntity(EntityDefine metadata)
{
}
public static void OnPropertyChanged(Entity entity, string propertyName, byte[] valueBytes)
{
var type = entity.GetType();
Entity component = null;
if (EntityDefine.IsComponent(type))
{
component = entity;
entity = entity.GetParent <Entity>();
}
var entityTypeId = EntityDefine.GetTypeId(entity.GetType());
//if (component != null)
//{
//}
//if (!EntityDefine.PropertyDefineCollectionMap.ContainsKey(entityTypeId))
//{
// Log.Error($"EntityDefine.PropertyDefineCollectionMap has no key {entityTypeId}");
// return;
//}
//if (!EntityDefine.PropertyDefineCollectionMap[entityTypeId].ContainsKey(propertyName))
//{
// Log.Error($"EntityDefine.PropertyDefineCollectionMap[{entityTypeId}] has no key {propertyName}");
// return;
//}
var msg = new M2C_OnEntityChanged();
PropertyDefineAttribute attr;
if (component != null)
{
msg.ComponentType = EntityDefine.GetTypeId(component.GetType());
attr = EntityDefine.PropertyDefineCollectionMap[msg.ComponentType][propertyName];
}
else
{
attr = EntityDefine.PropertyDefineCollectionMap[entityTypeId][propertyName];
}
msg.EntityId = entity.Id;
msg.EntityType = entityTypeId;
msg.PropertyId = attr.Id;
msg.PropertyValue.bytes = valueBytes;
Log.Error($"MessageHelper OnPropertyChanged {type} {propertyName} flag={attr.Flag} ");
if (attr.Flag == SyncFlag.AllClients)
{
Broadcast(entity.Domain, msg);
}
if (attr.Flag == SyncFlag.OtherClients)
{
if (entity is Unit u)
{
BroadcastToOther(u, msg);
}
}
if (attr.Flag == SyncFlag.OwnClient)
{
if (entity is Unit u)
{
Send(u, msg);
}
}
}
public void InsertAndIgnoreDuplicate()
{
#if NETSTANDARD
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
#endif
ClearDb();
ISpider spider = new DefaultSpider("test", new Site());
CassandraEntityPipeline insertPipeline = new CassandraEntityPipeline(connectString);
var metadata = new EntityDefine <ProductInsert>();
insertPipeline.AddEntity(metadata);
insertPipeline.Init();
// Common data
var data1 = new ProductInsert {
Sku = "110", Category = "3C", Url = "http://jd.com/110", CDate = new DateTime(2016, 8, 13)
};
var data2 = new ProductInsert {
Sku = "111", Category = "3C", Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
};
var data3 = new ProductInsert {
Sku = "112", Category = null, Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
};
// Value is null
insertPipeline.Process(metadata.Name, new List <dynamic> {
data1, data2, data3
}, spider);
var cluster = CassandraUtil.CreateCluster(connectString);
var session = cluster.Connect();
session.ChangeKeyspace("test");
var rows = session.Execute($"SELECT * FROM test.sku_cassandra_{DateTime.Now.ToString("yyyy_MM_dd")}").GetRows().ToList();
var results = new List <ProductInsert>();
foreach (var row in rows)
{
results.Add(new ProductInsert
{
Sku = row.GetValue <string>("sku"),
Category = row.GetValue <string>("category"),
Id = row.GetValue <Guid>("id")
});
}
insertPipeline.DefaultPipelineModel = PipelineMode.InsertAndIgnoreDuplicate;
var sku = results.First().Sku;
var data4 = new ProductInsert {
Id = results.First().Id, Sku = "113", Category = "asdfasf", Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
};
insertPipeline.Process(metadata.Name, new List <dynamic> {
data4
}, spider);
rows = session.Execute($"SELECT * FROM test.sku_cassandra_{DateTime.Now.ToString("yyyy_MM_dd")}").GetRows().ToList();
results = new List <ProductInsert>();
foreach (var row in rows)
{
results.Add(new ProductInsert
{
Sku = row.GetValue <string>("sku"),
Category = row.GetValue <string>("category")
});
}
Assert.Equal(3, results.Count);
Assert.DoesNotContain(results, r => r.Sku == sku);
Assert.Contains(results, r => r.Sku == "113");
Assert.Contains(results, r => r.Category == "asdfasf");
ClearDb();
}
/// <summary>
/// 1, 匹配 TargetUrl，判断是否需要解析HTML
/// 2，匹配 HelperUrl，判断是否需要加入到请求列表
/// </summary>
/// <typeparam name="TEntity"></typeparam>
/// <param name="entity"></param>
/// <param name="entityDefine"></param>
/// <param name="page"></param>
/// <returns></returns>
public TEntity Process <TEntity>(TEntity entity, EntityDefine entityDefine, Page page) where TEntity : IEntity
{
if (string.IsNullOrEmpty(page.Content))
{
return(entity);
}
// 查找所有链接
var allLinkEle = page.Selector?.SelectorAll("a", Html.HtmlSelectorPathType.Css);
List <string> links = new List <string>();
if (allLinkEle != null)
{
var r = allLinkEle.Select(t => t.GetAttribute("href")).Where(t => !string.IsNullOrEmpty(t)).Distinct().ToList();
if (r.Count > 0)
{
links.AddRange(r);
}
}
// 匹配新请求
if (entityDefine.HelperUrls?.Length > 0)
{
foreach (var url in entityDefine.HelperUrls)
{
foreach (var link in links)
{
if (url == link)
{
continue;
}
Regex regex = new Regex(url);
if (regex.IsMatch(link))
{
page.AddTargetRequest(link, page.Response.Request.Uri.ToString());
}
}
}
}
if (entityDefine.TargetUrls == null || entityDefine.TargetUrls.Length == 0)
{
return(entity);
}
foreach (var url in entityDefine.TargetUrls)
{
foreach (var link in links)
{
if (url == link)
{
continue;
}
Regex regex = new Regex(url);
if (regex.IsMatch(link))
{
page.AddTargetRequest(link, page.Response.Request.Uri.ToString());
}
}
}
// 匹配内容
var requestUrl = page.Response.Request.Uri.ToString();
foreach (var url in entityDefine.TargetUrls)
{
Regex regex = new Regex(url);
if (regex.IsMatch(requestUrl))
{
MatchSelector(entityDefine, entity, page);
}
}
return(entity);
}
public override async ETTask G2M_CreateUnitHandler(Scene scene, G2M_CreateUnit request, M2G_CreateUnit response, Action reply)
{
var copyMap = Game.Scene.Children.Values.ToList().Find((x) =>
{
if (x is Scene s)
{
return(s.Name == "CopyMap1");
}
return(false);
});
if (copyMap == null)
{
var copyMapConfig = StartConfigComponent.Instance.GetByName("CopyMap1");
copyMap = await SceneFactory.Create(Game.Scene, copyMapConfig.GetComponent <SceneConfig>().Name, SceneType.Map);
}
Unit unit = null;
if (request.UnitId != 0)
{
unit = await DBComponent.Instance.Query <Unit>(request.UnitId);
unit.Domain = copyMap;
}
else
{
unit = EntityFactory.CreateWithId <Unit>(copyMap, IdGenerater.GenerateId());
unit.PlayerId = request.PlayerId;
unit.Setup();
unit.Save().Coroutine();
}
unit.AddComponent <MoveComponent>();
unit.AddComponent <Body2dComponent>().CreateBody(.5f, .5f);
unit.AddComponent <MailBoxComponent>();
await unit.AddLocation();
unit.AddComponent <UnitGateComponent, long>(request.GateSessionId);
copyMap.GetComponent <UnitComponent>().Add(unit);
response.UnitId = unit.Id;
// 广播创建的unit
var inViewUnitsMsg = new M2C_InViewUnits();
var enterViewMsg = new M2C_OnEnterView();
Unit[] units = copyMap.GetComponent <UnitComponent>().GetAll();
foreach (Unit u in units)
{
var entityInfo = new EntiyInfo();
entityInfo.BsonBytes = new Google.Protobuf.ByteString();
entityInfo.BsonBytes.bytes = MongoHelper.ToBson(u);
entityInfo.Type = EntityDefine.GetTypeId <Unit>();
if (u.Id == unit.Id)
{
enterViewMsg.EnterEntity = entityInfo;
inViewUnitsMsg.SelfUnit = entityInfo.BsonBytes;
continue;
}
inViewUnitsMsg.InViewEntitys.Add(entityInfo);
}
var monsters = copyMap.GetComponent <MonsterComponent>().GetAll();
foreach (var u in monsters)
{
var entityInfo = new EntiyInfo();
entityInfo.BsonBytes = new Google.Protobuf.ByteString();
entityInfo.BsonBytes.bytes = MongoHelper.ToBson(u);
entityInfo.Type = EntityDefine.GetTypeId <Monster>();
inViewUnitsMsg.InViewEntitys.Add(entityInfo);
}
MessageHelper.BroadcastToOther(unit, enterViewMsg);
MessageHelper.Send(unit, inViewUnitsMsg);
reply();
}
public static void Start()
{
#if ILRuntime
if (!Define.IsILRuntime)
{
Log.Error("mono层是mono模式, 但是Hotfix层是ILRuntime模式");
}
#else
if (Define.IsILRuntime)
{
Log.Error("mono层是ILRuntime模式, Hotfix层是mono模式");
}
#endif
try
{
// 注册热更层回调
ETModel.Game.Hotfix.Update = () => { Update(); };
ETModel.Game.Hotfix.LateUpdate = () => { LateUpdate(); };
ETModel.Game.Hotfix.OnApplicationQuit = () => { OnApplicationQuit(); };
OpcodeHelper.ignoreDebugLogMessageSet.Add(HotfixOpcode.UnitOperation);
OpcodeHelper.ignoreDebugLogMessageSet.Add(HotfixOpcode.M2C_OnEntityChanged);
Game.Scene.AddComponent <UIComponent>();
Game.Scene.AddComponent <OpcodeTypeComponent>();
Game.Scene.AddComponent <MessageDispatcherComponent>();
// 加载热更配置
ETModel.Game.Scene.GetComponent <ResourcesComponent>().LoadBundle("config.unity3d");
Game.Scene.AddComponent <ConfigComponent>();
ETModel.Game.Scene.GetComponent <ResourcesComponent>().UnloadBundle("config.unity3d");
UnitConfig unitConfig = (UnitConfig)Game.Scene.GetComponent <ConfigComponent>().Get(typeof(UnitConfig), 1001);
Log.Debug($"config {JsonHelper.ToJson(unitConfig)}");
Game.EventSystem.Run(EventIdType.InitSceneStart);
HandlersHelper.Instance = new HandlersHelper();
EntityDefine.OnPropertyChanged += (entity, name, value) =>
{
Log.Debug($"OnPropertyChanged {name}");
var propertyDefineCollection = EntityDefine.PropertyDefineCollectionMap[EntityDefine.GetTypeId <Unit>()];
var attr = propertyDefineCollection[name];
var msg = new C2M_SetEntityProperty();
msg.PropertyId = attr.Id;
msg.PropertyValue.bytes = value;
SessionHelper.HotfixSend(msg);
};
}
catch (Exception e)
{
Log.Error(e);
}
}