private List <IPipeline> GetPipepines(List <JObject> pipelines)
{
if (pipelines == null || pipelines.Count == 0)
{
throw new SpiderException("Missing Pipeline.");
}
List <IPipeline> results = new List <IPipeline>();
foreach (var pipeline in pipelines)
{
IPipeline tmp = null;
var pipelineType = pipeline.SelectToken("$.Type")?.ToString();
if (pipelineType == null)
{
throw new SpiderException("Missing PrepareStartUrls type: " + pipeline);
}
switch (pipelineType)
{
case "MongoDb":
{
tmp = new MongoDbEntityPipeline(pipeline.SelectToken("$.ConnectString").ToString());
break;
}
case "SqlServer":
{
tmp = new SqlServerEntityPipeline(pipeline.SelectToken("$.ConnectString").ToString());
break;
}
case "MySql":
{
tmp = new MySqlEntityPipeline(pipeline.SelectToken("$.ConnectString").ToString());
break;
}
case "MySqlFile":
{
tmp = new MySqlFileEntityPipeline
{
DataFolder = pipeline.SelectToken("$.DataFolder").ToString()
};
break;
}
}
if (tmp == null)
{
throw new SpiderException("UNSPORT PIPELINE: " + pipeline);
}
else
{
results.Add(tmp);
}
}
return(results);
}
public void DefineUpdateEntity()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
SqlServerEntityPipeline insertPipeline = new SqlServerEntityPipeline(ConnectString);
try
{
insertPipeline.AddEntity(EntitySpider.GenerateEntityDefine(typeof(UpdateEntity1).GetTypeInfo()));
throw new SpiderException("TEST FAILED.");
}
catch (SpiderException e)
{
Assert.Equal("Columns set as primary is not a property of your entity.", e.Message);
}
try
{
insertPipeline.AddEntity(EntitySpider.GenerateEntityDefine(typeof(UpdateEntity2).GetTypeInfo()));
throw new SpiderException("TEST FAILED.");
}
catch (SpiderException e)
{
Assert.Equal("Columns set as update is not a property of your entity.", e.Message);
}
try
{
insertPipeline.AddEntity(EntitySpider.GenerateEntityDefine(typeof(UpdateEntity3).GetTypeInfo()));
throw new SpiderException("TEST FAILED.");
}
catch (SpiderException e)
{
Assert.Equal("There is no column need update.", e.Message);
}
var metadata = EntitySpider.GenerateEntityDefine(typeof(UpdateEntity4).GetTypeInfo());
insertPipeline.AddEntity(EntitySpider.GenerateEntityDefine(typeof(UpdateEntity4).GetTypeInfo()));
Assert.Single(insertPipeline.GetUpdateColumns(metadata.Name));
Assert.Equal("Value", insertPipeline.GetUpdateColumns(metadata.Name).First());
SqlServerEntityPipeline insertPipeline2 = new SqlServerEntityPipeline(ConnectString);
var metadata2 = EntitySpider.GenerateEntityDefine(typeof(UpdateEntity5).GetTypeInfo());
insertPipeline2.AddEntity(metadata2);
Assert.Single(insertPipeline2.GetUpdateColumns(metadata2.Name));
Assert.Equal("Value", insertPipeline2.GetUpdateColumns(metadata2.Name).First());
}
public void Update()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
ClearDb();
using (SqlConnection conn = new SqlConnection(ConnectString))
{
ISpider spider = new DefaultSpider("test", new Site());
SqlServerEntityPipeline insertPipeline = new SqlServerEntityPipeline(ConnectString);
var metadata = EntityDefine.Parse <ProductInsert>();
insertPipeline.AddEntity(metadata);
insertPipeline.InitPipeline(spider);
DataObject data1 = new DataObject {
{ "Sku", "110" }, { "Category", "3C" }, { "Url", "http://jd.com/110" }, { "CDate", "2016-08-13" }
};
DataObject data2 = new DataObject {
{ "Sku", "111" }, { "Category", "3C" }, { "Url", "http://jd.com/111" }, { "CDate", "2016-08-13" }
};
insertPipeline.Process(metadata.Name, new List <DataObject> {
data1, data2
});
SqlServerEntityPipeline updatePipeline = new SqlServerEntityPipeline(ConnectString);
var metadat2 = EntityDefine.Parse <ProductUpdate>();
updatePipeline.AddEntity(metadat2);
updatePipeline.InitPipeline(spider);
DataObject data3 = new DataObject {
{ "Sku", "110" }, { "Category", "4C" }, { "Url", "http://jd.com/110" }, { "CDate", "2016-08-13" }
};
updatePipeline.Process(metadat2.Name, new List <DataObject> {
data3
});
var list = conn.Query <ProductInsert>($"use test;select * from sku_{DateTime.Now.ToString("yyyy_MM_dd")}").ToList();
Assert.Equal(2, list.Count);
Assert.Equal("110", list[0].Sku);
Assert.Equal("4C", list[0].Category);
}
ClearDb();
}
public void UpdateWhenUnionPrimaryCheckIfSameBeforeUpdate()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
ClearDb();
using (SqlConnection conn = new SqlConnection(ConnectString))
{
ISpider spider = new DefaultSpider("test", new Site());
SqlServerEntityPipeline insertPipeline = new SqlServerEntityPipeline(ConnectString);
var metadata = EntitySpider.GenerateEntityDefine(typeof(Product2).GetTypeInfo());
insertPipeline.AddEntity(metadata);
insertPipeline.InitPipeline(spider);
var data1 = new DataObject {
{ "Sku", "110" }, { "Category1", "4C" }, { "Category", "3C" }, { "Url", "http://jd.com/110" }, { "CDate", "2016-08-13" }
};
var data2 = new DataObject {
{ "Sku", "111" }, { "Category1", "4C" }, { "Category", "3C" }, { "Url", "http://jd.com/111" }, { "CDate", "2016-08-13" }
};
insertPipeline.Process(metadata.Name, new List <DataObject> {
data1, data2
});
SqlServerEntityPipeline updatePipeline = new SqlServerEntityPipeline(ConnectString, true);
var metadata2 = EntitySpider.GenerateEntityDefine(typeof(Product2Update).GetTypeInfo());
updatePipeline.AddEntity(EntitySpider.GenerateEntityDefine(typeof(Product2Update).GetTypeInfo()));
updatePipeline.InitPipeline(spider);
var data3 = new DataObject {
{ "Sku", "110" }, { "Category1", "4C" }, { "Category", "AAAA" }, { "Url", "http://jd.com/110" }, { "CDate", "2016-08-13" }
};
updatePipeline.Process(metadata2.Name, new List <DataObject> {
data3
});
var list = conn.Query <Product2>($"use test;select * from sku2_{DateTime.Now.ToString("yyyy_MM_dd")}").ToList();
Assert.Equal(2, list.Count);
Assert.Equal("110", list[0].Sku);
Assert.Equal("AAAA", list[0].Category);
}
ClearDb();
}
public void Insert()
{
#if NETSTANDARD
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
#endif
ClearDb();
using (SqlConnection conn = new SqlConnection(ConnectString))
{
ISpider spider = new DefaultSpider("test", new Site());
SqlServerEntityPipeline insertPipeline = new SqlServerEntityPipeline(ConnectString);
var metadata = new EntityDefine <ProductInsert>();
insertPipeline.AddEntity(metadata);
insertPipeline.Init();
// Common data
var data1 = new ProductInsert {
Sku = "110", Category = "3C", Url = "http://jd.com/110", CDate = new DateTime(2016, 8, 13)
};
var data2 = new ProductInsert {
Sku = "111", Category = "3C", Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
};
var data3 = new ProductInsert {
Sku = "112", Category = null, Url = "http://jd.com/111", CDate = new DateTime(2016, 8, 13)
};
// Value is null
insertPipeline.Process(metadata.Name, new List <dynamic> {
data1, data2, data3
}, spider);
var list = conn.Query <ProductInsert>($"use test;select * from sku_{DateTime.Now.ToString("yyyy_MM_dd")}").ToList();
Assert.Equal(3, list.Count);
Assert.Equal("110", list[0].Sku);
Assert.Equal("111", list[1].Sku);
Assert.Null(list[2].Category);
}
ClearDb();
}
public void Insert()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
return;
}
ClearDb();
using (SqlConnection conn = new SqlConnection(ConnectString))
{
ISpider spider = new DefaultSpider("test", new Site());
SqlServerEntityPipeline insertPipeline = new SqlServerEntityPipeline(ConnectString);
var metadata = EntitySpider.GenerateEntityDefine(typeof(ProductInsert).GetTypeInfo());
insertPipeline.AddEntity(EntitySpider.GenerateEntityDefine(typeof(ProductInsert).GetTypeInfo()));
insertPipeline.InitPipeline(spider);
// Common data
var data1 = new DataObject {
{ "Sku", "110" }, { "Category", "3C" }, { "Url", "http://jd.com/110" }, { "CDate", "2016-08-13" }
};
var data2 = new DataObject {
{ "Sku", "111" }, { "Category", "3C" }, { "Url", "http://jd.com/111" }, { "CDate", "2016-08-13" }
};
// Value is null
var data3 = new DataObject {
{ "Sku", "112" }, { "Category", null }, { "Url", "http://jd.com/111" }, { "CDate", "2016-08-13" }
};
insertPipeline.Process(metadata.Name, new List <DataObject> {
data1, data2, data3
});
var list = conn.Query <ProductInsert>($"use test;select * from sku_{DateTime.Now.ToString("yyyy_MM_dd")}").ToList();
Assert.Equal(3, list.Count);
Assert.Equal("110", list[0].Sku);
Assert.Equal("111", list[1].Sku);
Assert.Null(list[2].Category);
}
ClearDb();
}
public override void DataTypes()
{
if (!Env.IsWindows)
{
return;
}
using (var conn = new SqlConnection("Server=.\\SQLEXPRESS;Database=master;Trusted_Connection=True;MultipleActiveResultSets=true"))
{
try
{
conn.Execute("create database test;");
}
catch
{
}
try
{
conn.Execute("USE [test]; drop table [test].dbo.[table15]");
}
catch
{
}
var spider = new DefaultSpider();
EntityProcessor <Entity15> processor = new EntityProcessor <Entity15>();
var pipeline = new SqlServerEntityPipeline("Server=.\\SQLEXPRESS;Database=master;Trusted_Connection=True;MultipleActiveResultSets=true");
var resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems.AddOrUpdateResultItem(processor.Model.Identity, new Tuple <IModel, IEnumerable <dynamic> >(processor.Model, new dynamic[] {
new Dictionary <string, dynamic>
{
{ "int", "1" },
{ "bool", "1" },
{ "bigint", "11" },
{ "string", "aaa" },
{ "time", "2018-06-12" },
{ "float", "1" },
{ "double", "1" },
{ "string1", "abc" },
{ "string2", "abcdd" },
{ "decimal", "1" }
}
}));
pipeline.Process(new ResultItems[] { resultItems }, spider);
var columns = conn.Query <ColumnInfo>("USE [test];select b.name Name,c.name+'(' + cast(c.length as varchar)+')' [Type] from sysobjects a,syscolumns b,systypes c where a.id=b.id and a.name='table15' and a.xtype='U'and b.xtype=c.xtype").ToList();
Assert.Equal(15, columns.Count);
Assert.Equal("creation_date".ToLower(), columns[0].Name);
Assert.Equal("int".ToLower(), columns[1].Name);
Assert.Equal("time".ToLower(), columns[2].Name);
Assert.Equal("creation_time".ToLower(), columns[3].Name);
Assert.Equal("float".ToLower(), columns[4].Name);
Assert.Equal("double".ToLower(), columns[5].Name);
Assert.Equal("bool".ToLower(), columns[6].Name);
Assert.Equal("decimal".ToLower(), columns[7].Name);
Assert.Equal("bigint".ToLower(), columns[8].Name);
Assert.Equal("string".ToLower(), columns[9].Name);
Assert.Equal("string1".ToLower(), columns[10].Name);
Assert.Equal("string2".ToLower(), columns[11].Name);
Assert.Equal("date(3)", columns[0].Type);
Assert.Equal("int(4)", columns[1].Type);
Assert.Equal("datetime(8)", columns[2].Type);
Assert.Equal("datetime(8)", columns[3].Type);
Assert.Equal("float(8)", columns[4].Type);
Assert.Equal("float(8)", columns[5].Type);
Assert.Equal("bit(1)", columns[6].Type);
Assert.Equal("decimal(17)", columns[7].Type);
Assert.Equal("bigint(8)", columns[8].Type);
Assert.Equal("nvarchar(8000)", columns[9].Type);
Assert.Equal("nvarchar(8000)", columns[10].Type);
Assert.Equal("nvarchar(8000)", columns[11].Type);
conn.Execute("USE [test]; drop table [test].dbo.[table15]");
}
}