private async Task DumpToDB(IEnumerable <T> data, string destTable, string connectionString, string destLabel) { using (var bulkReader = new BulkDataReader <T>(TypeAccessorManager <T> .GetAccessorByDestLabel(destLabel, connectionString, destTable), data)) { using (var conn = new SqlConnection(connectionString)) { await conn.OpenAsync(); using (var bulkCopy = new SqlBulkCopy(conn)) { foreach (SqlBulkCopyColumnMapping map in bulkReader.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = destTable; bulkCopy.BulkCopyTimeout = 0; bulkCopy.BatchSize = m_bulkSize; // Write from the source to the destination. await bulkCopy.WriteToServerAsync(bulkReader); } } } }
public void TestReferenceLoopDepth() { AppDomain.CurrentDomain.SetData("DataDirectory", AppDomain.CurrentDomain.BaseDirectory); var connectString = @"Data Source=(LocalDB)\v11.0;AttachDbFilename=|DataDirectory|\loop.mdf;Initial Catalog=loop;Integrated Security=True;Connect Timeout=30"; //init database System.Data.Entity.Database.SetInitializer(new DropCreateDatabaseAlways <LoopContext>()); var context = new LoopContext(connectString); context.LoopInits.Add(new LoopInit()); context.SaveChanges(); var loopA = new LoopA { Name = "loopA", LoopB = new LoopB { Name = "loopB", LoopC = new LoopC { LoopA = new LoopA { Name = "loopA2" } } } }; using (var connection = new SqlConnection(connectString)) { connection.Open(); var tableName = "dbo.LoopEntities"; var accessor = TypeAccessorManager <LoopA> .GetAccessorByDestLabel(GSProduct.AD, connectString, tableName); var reader = new BulkDataReader <LoopA>(accessor, new[] { loopA }); using (var bulkCopy = new SqlBulkCopy(connection)) { foreach (SqlBulkCopyColumnMapping map in reader.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = tableName; bulkCopy.WriteToServer(reader); } } Assert.AreEqual("loopA", context.LoopEntities.FirstOrDefault().Name); }
public void TestNestClass() { var connectString = TestUtils.GetLocalDBConnectionString(); Database.SetInitializer(new DropCreateDatabaseAlways <NestContext>()); var context = new NestContext(connectString); context.NestInits.Add(new NestInit()); context.SaveChanges(); var a1 = new ABulk { }; var a2 = new ABulk { B = new B { } }; var a3 = new ABulk { B = new B { Value2 = "a" } }; var a4 = new ABulk { Value1 = "a", B = new B { Value2 = "a" } }; using (var connection = new SqlConnection(connectString)) { connection.Open(); var aTypeAccessor = TypeAccessorManager <ABulk> .GetAccessorForTable(new TargetTable("NestClassTest", connectString, "dbo.AEFs")); var aReader = new BulkDataReader <ABulk>(aTypeAccessor, new[] { a1, a2, a3, a4 }); using (var aCopy = new SqlBulkCopy(connection)) { foreach (var map in aReader.ColumnMappings) { aCopy.ColumnMappings.Add(map); } aCopy.DestinationTableName = "dbo.AEFs"; aCopy.WriteToServer(aReader); } } Assert.AreEqual(1, context.AAs.FirstOrDefault().Id); Assert.AreEqual(1, context.AAs.Count(a => a.Value1 == "a")); Assert.AreEqual(3, context.AAs.Count(a => a.Value1 == "-")); Assert.AreEqual(2, context.AAs.Count(a => a.Value2 == "a")); Assert.AreEqual(2, context.AAs.Count(a => a.Value2 == "--")); }
public void TestReferenceLoopDepth() { var connectString = TestUtils.GetLocalDBConnectionString(); //init database Database.SetInitializer(new DropCreateDatabaseAlways <LoopContext>()); var context = new LoopContext(connectString); context.LoopInits.Add(new LoopInit()); context.SaveChanges(); var loopA = new LoopA { Name = "loopA", LoopB = new LoopB { Name = "loopB", LoopC = new LoopC { LoopA = new LoopA { Name = "loopA2" } } } }; using (var connection = new SqlConnection(connectString)) { connection.Open(); var tableName = "dbo.LoopEntities"; var accessor = TypeAccessorManager <LoopA> .GetAccessorForTable(new TargetTable(GSProduct.AD, connectString, tableName)); var reader = new BulkDataReader <LoopA>(accessor, new[] { loopA }); using (var bulkCopy = new SqlBulkCopy(connection)) { foreach (SqlBulkCopyColumnMapping map in reader.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = tableName; bulkCopy.WriteToServer(reader); } } Assert.AreEqual("loopA", context.LoopEntities.FirstOrDefault().Name); }
/// <summary> /// Constructs a DbDataJoiner instance /// </summary> /// <param name="joinOn">Property path from the root object down to the lookup key</param> /// <param name="dimTableTarget">Table information of the remote dimension table</param> /// <param name="option">Option to use for this dataflow</param> /// <param name="batchSize">The batch size for a batched remote look up</param> /// <param name="cacheSize">The local cache item count (part of the remote table)</param> public DbDataJoiner(Expression <Func <TIn, TLookupKey> > joinOn, TargetTable dimTableTarget, DataflowOptions option, int batchSize = 8 * 1024, int cacheSize = 1024 * 1024) : base(option) { m_dimTableTarget = dimTableTarget; m_batchSize = batchSize; m_batcher = new BatchBlock <TIn>(batchSize, option.ToGroupingBlockOption()).ToDataflow(option); m_batcher.Name = "Batcher"; m_lookupNode = new TransformManyDataflow <JoinBatch <TIn>, TIn>(this.JoinBatch, option); m_lookupNode.Name = "LookupNode"; m_typeAccessor = TypeAccessorManager <TIn> .GetAccessorForTable(dimTableTarget); m_keyComparer = typeof(TLookupKey) == typeof(byte[]) ? (IEqualityComparer <TLookupKey>)((object)new ByteArrayEqualityComparer()) : EqualityComparer <TLookupKey> .Default; m_rowCache = new RowCache <TLookupKey>(cacheSize, m_keyComparer); m_logger = Utils.GetNamespaceLogger(); m_joinOnMapping = m_typeAccessor.DbColumnMappings.First(m => m.Host.PropertyInfo == this.ExtractPropertyInfo(joinOn)); var transformer = new TransformBlock <TIn[], JoinBatch <TIn> >( array => new JoinBatch <TIn>(array, CacheLookupStrategy.RemoteLookup), option.ToExecutionBlockOption()).ToDataflow(option); transformer.Name = "ArrayToJoinBatchConverter"; m_batcher.LinkTo(transformer); transformer.LinkTo(m_lookupNode); RegisterChild(m_batcher); RegisterChild(transformer); RegisterChild(m_lookupNode); m_dimInserter = new DimTableInserter(this, dimTableTarget, joinOn, option) { Name = "DimInserter" }; var hb = new HeartbeatNode <JoinBatch <TIn> >(option); m_dimInserter.RegisterDependency(m_lookupNode); m_dimInserter.LinkTo(hb); hb.LinkTo(m_lookupNode); RegisterChild(m_dimInserter); RegisterChild(hb); RegisterChildRing(transformer.CompletionTask, m_lookupNode, m_dimInserter, hb); }
public void TestReader() { AppDomain.CurrentDomain.SetData("DataDirectory", AppDomain.CurrentDomain.BaseDirectory); var connectString = @"Data Source=(LocalDB)\v11.0;AttachDbFilename=|DataDirectory|\TestReader.mdf;Initial Catalog=mytest;Integrated Security=True;Connect Timeout=30"; //init database System.Data.Entity.Database.SetInitializer(new DropCreateDatabaseAlways <MyContext>()); var context = new MyContext(connectString); context.Inits.Add(new Init()); context.SaveChanges(); #region init sample data var entities = new List <Entity> { new Entity { Id = 1, PersonName = "zhang san", PhoneNumber = "110", Location = new Location { RoadNumber = 1, RoadName = "first" } }, new Entity { Id = 2, PersonName = "li si", PhoneNumber = "119", // Location = new Location{RoadNumber = 2, RoadName = "second", RoadMap = new RoadMap{Id=2}} Address = new Address { AddressNumber = 10 } } }; var entities2 = new List <Entity> { new Entity { Id = 3, PersonName = "wang er", PhoneNumber = "120" } }; #endregion #region add to db using (var connection = new SqlConnection(connectString)) { connection.Open(); // Wd: var wdAccessor = TypeAccessorManager <Entity> .GetAccessorByDestLabel(GSProduct.WD, connectString, "dbo.Wds"); var wdReader2 = new BulkDataReader <Entity>(wdAccessor, entities); using (var bulkCopy = new SqlBulkCopy(connection)) { foreach (SqlBulkCopyColumnMapping map in wdReader2.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = "dbo.Wds"; bulkCopy.WriteToServer(wdReader2); } //Cd: var cdAccessor = TypeAccessorManager <Entity> .GetAccessorByDestLabel(GSProduct.CD, connectString, "dbo.Cds"); var cdReader2 = new BulkDataReader <Entity>(cdAccessor, entities); using (var bulkCopy = new SqlBulkCopy(connection)) { foreach (SqlBulkCopyColumnMapping map in cdReader2.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = "dbo.Cds"; bulkCopy.WriteToServer(cdReader2); } //Wd 2: use debug model to test whether this new WdAccessor is new build or not //right: this WdAccessor should just get from TypeAccessorManager, it can not be rebuild. var wdAccessor2 = TypeAccessorManager <Entity> .GetAccessorByDestLabel(GSProduct.WD, connectString, "dbo.Wds"); var wdReader22 = new BulkDataReader <Entity>(wdAccessor2, entities2); using (var bulkCopy = new SqlBulkCopy(connection)) { foreach (var map in wdReader22.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = "dbo.Wds"; bulkCopy.WriteToServer(wdReader22); } } #endregion #region reader from db and test whether data is exist. //wd Assert.AreEqual(3, context.Wds.Count()); Assert.AreEqual("li si", context.Wds.FirstOrDefault(w => w.WdId == 2).Name); Assert.AreEqual("110", context.Wds.FirstOrDefault(w => w.WdId == 1).PhoneNumber); var entity = context.Wds.FirstOrDefault(); Assert.AreEqual(entity.PhoneNumber, entity.PhoneNumber2); //Cd Assert.AreEqual(2, context.Cds.Count()); Assert.AreEqual("110", context.Cds.FirstOrDefault(c => c.CdId == 1).PhoneNum); Assert.AreEqual("first", context.Cds.FirstOrDefault(c => c.CdId == 1).RoadName); Assert.AreEqual(null, context.Cds.FirstOrDefault(c => c.CdId == 2).RoadName); //Cd default value,the roadnum's default value is -1 Assert.AreEqual(-1, context.Cds.FirstOrDefault(c => c.CdId == 2).RoadNum); Assert.AreEqual(null, context.Cds.FirstOrDefault(c => c.CdId == 2).RoadName); //Cd not Product Property Assert.AreEqual(10, context.Cds.FirstOrDefault(c => c.CdId == 2).AddressNumber); #endregion context.Dispose(); }
public void TestReader() { var connectString = TestUtils.GetLocalDBConnectionString("TestReader2"); //init database var context = new MyContext(connectString); context.Inits.Add(new Init()); context.SaveChanges(); #region init sample data var entities = new List <Entity> { new Entity { Id = 1, PersonName = "zhang san", PhoneNumber = "110", Location = new Location { RoadNumber = 1, RoadName = "first" } }, new Entity { Id = 2, PersonName = "li si", PhoneNumber = "119", // Location = new Location{RoadNumber = 2, RoadName = "second", RoadMap = new RoadMap{Id=2}} Address = new Address { AddressNumber = 10 } } }; var entities2 = new List <Entity> { new Entity { Id = 3, PersonName = "wang er", PhoneNumber = "120" } }; #endregion #region add to db using (var connection = new SqlConnection(connectString)) { connection.Open(); // Wd: var wdAccessor = TypeAccessorManager <Entity> .GetAccessorForTable(new TargetTable(GSProduct.WD, connectString, "dbo.Wds")); var wdReader2 = new BulkDataReader <Entity>(wdAccessor, entities); using (var bulkCopy = new SqlBulkCopy(connection)) { foreach (SqlBulkCopyColumnMapping map in wdReader2.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = "dbo.Wds"; bulkCopy.WriteToServer(wdReader2); } //Cd: var cdAccessor = TypeAccessorManager <Entity> .GetAccessorForTable(new TargetTable(GSProduct.CD, connectString, "dbo.Cds")); var cdReader2 = new BulkDataReader <Entity>(cdAccessor, entities); using (var bulkCopy = new SqlBulkCopy(connection)) { foreach (SqlBulkCopyColumnMapping map in cdReader2.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = "dbo.Cds"; bulkCopy.WriteToServer(cdReader2); } //Wd 2: use debug model to test whether this new WdAccessor is new build or not //right: this WdAccessor should just get from TypeAccessorManager, it can not be rebuild. var wdAccessor2 = TypeAccessorManager <Entity> .GetAccessorForTable( new TargetTable(GSProduct.WD, connectString, "dbo.Wds")); var wdReader22 = new BulkDataReader <Entity>(wdAccessor2, entities2); using (var bulkCopy = new SqlBulkCopy(connection)) { foreach (var map in wdReader22.ColumnMappings) { bulkCopy.ColumnMappings.Add(map); } bulkCopy.DestinationTableName = "dbo.Wds"; bulkCopy.WriteToServer(wdReader22); } } #endregion #region reader from db and test whether data is exist. //wd Assert.AreEqual(3, context.Wds.Count()); Assert.AreEqual("li si", context.Wds.FirstOrDefault(w => w.WdId == 2).Name); Assert.AreEqual("110", context.Wds.FirstOrDefault(w => w.WdId == 1).PhoneNumber); var entity = context.Wds.FirstOrDefault(); Assert.AreEqual(entity.PhoneNumber, entity.PhoneNumber2); //Cd Assert.AreEqual(2, context.Cds.Count()); Assert.AreEqual("110", context.Cds.FirstOrDefault(c => c.CdId == 1).PhoneNum); Assert.AreEqual("first", context.Cds.FirstOrDefault(c => c.CdId == 1).RoadName); Assert.AreEqual(null, context.Cds.FirstOrDefault(c => c.CdId == 2).RoadName); //Cd default value,the roadnum's default value is -1 Assert.AreEqual(-1, context.Cds.FirstOrDefault(c => c.CdId == 2).RoadNum); Assert.AreEqual(null, context.Cds.FirstOrDefault(c => c.CdId == 2).RoadName); //Cd not Product Property Assert.AreEqual(10, context.Cds.FirstOrDefault(c => c.CdId == 2).AddressNumber); #endregion context.Dispose(); }
public void TestNestClass() { AppDomain.CurrentDomain.SetData("DataDirectory", AppDomain.CurrentDomain.BaseDirectory); var connectString = @"Data Source=(LocalDB)\v11.0;AttachDbFilename=|DataDirectory|\nestClassTest.mdf;Initial Catalog=nestClassTest;Integrated Security=True;Connect Timeout=30"; System.Data.Entity.Database.SetInitializer(new DropCreateDatabaseAlways <NestContext>()); var context = new NestContext(connectString); context.NestInits.Add(new NestInit()); context.SaveChanges(); var a = new A { AId = 1, B = new B { BId = 2, A = new A { AId = 3 } } }; var b = new B { BId = 1, A = new A { AId = 2, B = new B { BId = 3 } } }; using (var connection = new SqlConnection(connectString)) { connection.Open(); var aTypeAccessor = TypeAccessorManager <A> .GetAccessorByDestLabel("test1", connectString, "dbo.AAs"); var aReader = new BulkDataReader <A>(aTypeAccessor, new[] { a }); using (var aCopy = new SqlBulkCopy(connection)) { foreach (var map in aReader.ColumnMappings) { aCopy.ColumnMappings.Add(map); } aCopy.DestinationTableName = "dbo.AAs"; aCopy.WriteToServer(aReader); } var bTypeAccessor = TypeAccessorManager <B> .GetAccessorByDestLabel("test2", connectString, "dbo.BBs"); var bReader = new BulkDataReader <B>(bTypeAccessor, new[] { b }); using (var bCopy = new SqlBulkCopy(connection)) { foreach (var map in bReader.ColumnMappings) { bCopy.ColumnMappings.Add(map); } bCopy.DestinationTableName = "dbo.BBs"; bCopy.WriteToServer(aReader); } } Assert.AreEqual(1, context.AAs.FirstOrDefault().AId); Assert.AreEqual(1, context.BBs.FirstOrDefault().BId); }