public async Task CreateTableNoSchema()
{
using (var conn = CreateConnection())
{
// 如果实体的 Schema 没有配置表名，则使用类名
await conn.ExecuteAsync("drop table if exists createtableentity1;");
var services = SpiderFactory.CreateScopeServiceProvider();
var storage = CreateStorage(StorageType.Insert);
var dfc = new DataFlowContext(null, services);
var typeName = typeof(CreateTableEntity1).FullName;
var entity = new CreateTableEntity1();
dfc.Add(typeName, entity.GetTableMetadata());
var items = new ParseResult <CreateTableEntity1>
{
entity
};
dfc.AddParseItem(typeName, items);
await storage.HandleAsync(dfc);
var list = (await conn.QueryAsync <CreateTableEntity1>("SELECT * FROM createtableentity1")).ToList();
Assert.Single(list);
entity = list.First();
Assert.Equal("xxx", entity.Str1);
Assert.Equal("yyy", entity.Str2);
Assert.Equal(655, entity.Required);
Assert.Equal(0, entity.Decimal);
Assert.Equal(600, entity.Long);
Assert.Equal(400, entity.Double);
Assert.Equal(200.0F, entity.Float);
await conn.ExecuteAsync("drop table if exists createtableentity1;");
}
}
public void ParseEntity()
{
var service = SpiderFactory.CreateScopeServiceProvider();
var dataContext = new DataFlowContext(new Response
{
Request = new Request("https://list.jd.com/list.html?cat=9987,653,655",
new Dictionary <string, string>
{
{ "cat", "手机" },
{ "cat3", "110" }
}),
RawText = File.ReadAllText("Jd.html")
}, service);
DataParser <Product> extractor = new DataParser <Product>();
extractor.HandleAsync(dataContext).GetAwaiter().GetResult();
var results = (ParseResult <Product>)dataContext.GetParseItem(typeof(Product).FullName);
Assert.Equal(60, results.Count);
Assert.Equal("手机", results[0].CategoryName);
Assert.Equal(110, results[0].CategoryId);
Assert.Equal("https://item.jd.com/3031737.html", results[0].Url);
Assert.Equal("3031737", results[0].Sku);
Assert.Equal("荣耀官方旗舰店", results[0].ShopName);
Assert.Equal("荣耀 NOTE 8 4GB+32GB 全网通版 冰河银", results[0].Name);
Assert.Equal("1000000904", results[0].VenderId);
Assert.Equal("1000000904", results[0].JdzyShopId);
Assert.Equal(DateTime.Now.ToString("yyyy-MM-dd"), results[0].RunId.ToString("yyyy-MM-dd"));
}
public async Task UpdatePartColumns()
{
using (var conn = CreateConnection())
{
// 如果实体的 Schema 没有配置表名，则使用类名
await conn.ExecuteAsync(
$"drop table if exists {Escape}test{Escape}.{Escape}updatepartcolumns{Escape};");
var services = SpiderFactory.CreateScopeServiceProvider();
var storage = CreateStorage(StorageType.InsertIgnoreDuplicate);
var dfc = new DataFlowContext(null, services);
var typeName = typeof(CreateTableEntity6).FullName;
var entity = new CreateTableEntity6();
dfc.Add(typeName, entity.GetTableMetadata());
var items = new ParseResult<CreateTableEntity6>
{
entity
};
dfc.AddParseItem(typeName, items);
await storage.HandleAsync(dfc);
var dfc2 = new DataFlowContext(null, services);
dfc2.Add(typeName, entity.GetTableMetadata());
var now = DateTime.Now;
dfc2.AddParseItem(typeName, new ParseResult<CreateTableEntity6>
{
new CreateTableEntity6
{
Str1 = "TTT",
DateTime = now,
DateTimeOffset = now,
Double = 888,
Float = 999F,
Required = 888
}
});
var storage2 = CreateStorage(StorageType.Update);
await storage2.HandleAsync(dfc2);
var list = (await conn.QueryAsync<CreateTableEntity6>(
$"SELECT * FROM {Escape}test{Escape}.{Escape}updatepartcolumns{Escape}"))
.ToList();
Assert.Single(list);
entity = list.First();
Assert.Equal("TTT", entity.Str1);
Assert.Equal("yyy", entity.Str2);
Assert.Equal(655, entity.Required);
Assert.Equal(0, entity.Decimal);
Assert.Equal(600, entity.Long);
Assert.Equal(888, entity.Double);
Assert.Equal(200.0F, entity.Float);
await conn.ExecuteAsync(
$"drop table if exists {Escape}test{Escape}.{Escape}updatepartcolumns{Escape};");
}
}
public void XpathFollow()
{
var services = SpiderFactory.CreateScopeServiceProvider();
var dataContext = new DataFlowContext(new Response
{
Request = new Request("http://cnblogs.com"),
RawText = File.ReadAllText("cnblogs.html")
}, services);
var xpathFollow = DataParserHelper.QueryFollowRequestsByXPath(".//div[@class='pager']");
var requests = xpathFollow.Invoke(dataContext);
Assert.Equal(12, requests.Count);
Assert.Contains(requests, r => r == "http://cnblogs.com/sitehome/p/2");
}
public async Task CreateTableAutoIncPrimary()
{
using (var conn = CreateConnection())
{
// 如果实体的 Schema 没有配置表名，则使用类名
await conn.ExecuteAsync("drop table if exists test.dbo.createtableautoincprimay;");
var services = SpiderFactory.CreateScopeServiceProvider();
var storage = CreateStorage(StorageType.Insert);
var dfc = new DataFlowContext(null, services);
var typeName = typeof(CreateTableEntity5).FullName;
var entity = new CreateTableEntity5();
dfc.Add(typeName, entity.GetTableMetadata());
var items = new ParseResult <CreateTableEntity5>
{
entity,
entity
};
dfc.AddParseItem(typeName, items);
await storage.HandleAsync(dfc);
var list =
(await conn.QueryAsync <CreateTableEntity5>("SELECT * FROM test.dbo.createtableautoincprimay"))
.ToList();
Assert.Equal(2, list.Count);
entity = list.First();
Assert.Equal("xxx", entity.Str1);
Assert.Equal("yyy", entity.Str2);
Assert.Equal(655, entity.Required);
Assert.Equal(0, entity.Decimal);
Assert.Equal(600, entity.Long);
Assert.Equal(400, entity.Double);
Assert.Equal(200.0F, entity.Float);
Assert.Equal(1, entity.Id);
Assert.Equal(2, list[1].Id);
var primaries = (await conn.QueryAsync <IndexInfo>
(@"USE test; EXEC sp_pkeys @table_name='createtableautoincprimay'")
).ToList();
Assert.Single(primaries);
Assert.Equal("id", primaries[0].COLUMN_NAME);
Assert.Equal(1, primaries[0].KEY_SEQ);
await conn.ExecuteAsync("drop table if exists test.dbo.createtableautoincprimay;");
}
}
public async Task AutoIncPrimary()
{
using (var conn = CreateConnection())
{
// 如果实体的 Schema 没有配置表名，则使用类名
await conn.ExecuteAsync(
$"drop table if exists {Escape}test{Escape}.{Escape}createtableautoincprimay{Escape};");
var services = SpiderFactory.CreateScopeServiceProvider();
var storage = CreateStorage(StorageType.Insert);
var dfc = new DataFlowContext(null, services);
var typeName = typeof(CreateTableEntity5).FullName;
var entity = new CreateTableEntity5();
dfc.Add(typeName, entity.GetTableMetadata());
var items = new ParseResult<CreateTableEntity5>
{
entity,
entity
};
dfc.AddParseItem(typeName, items);
await storage.HandleAsync(dfc);
var list =
(await conn.QueryAsync<CreateTableEntity5>(
$"SELECT * FROM {Escape}test{Escape}.{Escape}createtableautoincprimay{Escape}"))
.ToList();
Assert.Equal(2, list.Count);
entity = list.First();
Assert.Equal("xxx", entity.Str1);
Assert.Equal("yyy", entity.Str2);
Assert.Equal(655, entity.Required);
Assert.Equal(0, entity.Decimal);
Assert.Equal(600, entity.Long);
Assert.Equal(400, entity.Double);
Assert.Equal(200.0F, entity.Float);
Assert.Equal(1, entity.Id);
Assert.Equal(2, list[1].Id);
var primaries = (await conn.QueryAsync<PrimaryInfo>(
$"SELECT t.CONSTRAINT_TYPE, c.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS t, INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS c WHERE t.TABLE_NAME = c.TABLE_NAME AND t.TABLE_SCHEMA = 'test' AND t.CONSTRAINT_TYPE = 'PRIMARY KEY' AND t.TABLE_NAME='createtableautoincprimay';")
).ToList();
Assert.Single(primaries);
Assert.Equal("id", primaries[0].COLUMN_NAME);
await conn.ExecuteAsync(
$"drop table if exists {Escape}test{Escape}.{Escape}createtableautoincprimay{Escape};");
}
}
public void MultiEntitySelector()
{
var service = SpiderFactory.CreateScopeServiceProvider();
var dataContext = new DataFlowContext(new Response
{
Request = new Request("http://abcd.com"),
RawText = Html
}, service);
var parser = new DataParser <E>();
parser.HandleAsync(dataContext).GetAwaiter().GetResult();
var results = (ParseResult <E>)dataContext.GetParseItem(typeof(E).FullName);
Assert.Equal("a", results[0].title);
Assert.Equal("b", results[1].title);
}
public async Task Indexes()
{
using (var conn = CreateConnection())
{
await conn.ExecuteAsync(
$"drop table if exists {Escape}test{Escape}.{Escape}createtableindexes{Escape};");
var services = SpiderFactory.CreateScopeServiceProvider();
var storage = CreateStorage(StorageType.Insert);
var dfc = new DataFlowContext(null, services);
var typeName = typeof(CreateTableEntity5).FullName;
var entity = new CreateTableEntity9();
dfc.Add(typeName, entity.GetTableMetadata());
var items = new ParseResult<CreateTableEntity9>
{
entity
};
dfc.AddParseItem(typeName, items);
await storage.HandleAsync(dfc);
var indexes = (await conn.QueryAsync<IndexInfo>
($"show index from test.createtableindexes")
).ToList();
Assert.Equal(6, indexes.Count);
Assert.Contains(indexes,
x => x.Key_name == "INDEX_STR1" && x.Non_unique == 1 && x.Column_name == "str1");
Assert.Contains(indexes, x =>
x.Key_name == "INDEX_STR1_STR2" && x.Non_unique == 1 && x.Column_name == "str1");
Assert.Contains(indexes, x =>
x.Key_name == "INDEX_STR1_STR2" && x.Non_unique == 1 && x.Column_name == "str2");
Assert.Contains(indexes,
x => x.Key_name == "UNIQUE_STR3" && x.Non_unique == 0 && x.Column_name == "str3");
Assert.Contains(indexes, x =>
x.Key_name == "UNIQUE_STR3_STR4" && x.Non_unique == 0 && x.Column_name == "str3");
Assert.Contains(indexes, x =>
x.Key_name == "UNIQUE_STR3_STR4" && x.Non_unique == 0 && x.Column_name == "str4");
await conn.ExecuteAsync(
$"drop table if exists {Escape}test{Escape}.{Escape}createtableindexes{Escape};");
}
}
public async Task UseTransaction()
{
using (var conn = CreateConnection())
{
// 如果实体的 Schema 没有配置表名，则使用类名
await conn.ExecuteAsync(
$"drop table if exists {Escape}test{Escape}.{Escape}createtableprimay{Escape};");
var services = SpiderFactory.CreateScopeServiceProvider();
var storage = (RelationalDatabaseEntityStorageBase) CreateStorage(StorageType.InsertIgnoreDuplicate);
storage.UseTransaction = true;
var dfc = new DataFlowContext(null, services);
var typeName = typeof(CreateTableEntity4).FullName;
var entity = new CreateTableEntity4();
dfc.Add(typeName, entity.GetTableMetadata());
var items = new ParseResult<CreateTableEntity4>
{
entity,
entity,
entity
};
dfc.AddParseItem(typeName, items);
await storage.HandleAsync(dfc);
var list = (await conn.QueryAsync<CreateTableEntity4>(
$"SELECT * FROM {Escape}test{Escape}.{Escape}createtableprimay{Escape}"))
.ToList();
Assert.Single(list);
entity = list.First();
Assert.Equal("xxx", entity.Str1);
Assert.Equal("yyy", entity.Str2);
Assert.Equal(655, entity.Required);
Assert.Equal(0, entity.Decimal);
Assert.Equal(600, entity.Long);
Assert.Equal(400, entity.Double);
Assert.Equal(200.0F, entity.Float);
await conn.ExecuteAsync(
$"drop table if exists {Escape}test{Escape}.{Escape}createtableprimay{Escape};");
}
}