Example #1
0
        private async Task HandleResponseAsync(Request request, Response response, byte[] responseBytes)
        {
            try
            {
                using var scope = _services.ServiceProvider.CreateScope();
                var context = new DataContext(scope.ServiceProvider, Options, request, response);
                context.AddData(Consts.ResponseBytes, responseBytes);
                context.AddData(Consts.SpiderId, Id);

                foreach (var dataFlow in _dataFlows)
                {
                    await dataFlow.HandleAsync(context);
                }

                var count = await AddRequestsAsync(context.FollowRequests);

                await _services.StatisticsClient.IncreaseTotalAsync(Id, count);

                await _services.StatisticsClient.IncreaseSuccessAsync(Id);
            }
            catch (ExitException ee)
            {
                Logger.LogError($"Exit by: {ee.Message}");
                await ExitAsync();
            }
            catch (Exception e)
            {
                // if download correct content, parser or storage failed by network or something else
                // retry it until trigger retryTimes limitation
                await AddRequestsAsync(request);

                Logger.LogError($"{Id} handle {JsonConvert.SerializeObject(request)} failed: {e}");
            }
        }
Example #2
0
        protected override Task Parse(DataContext context)
        {
            if (context.Response != null)
            {
                context.AddData("RequestUri", context.Request.RequestUri.ToString());
                context.AddData("Content", context.Response.ReadAsString());
                context.AddData("TargetUri", context.Response.TargetUri);
                context.AddData("Success", context.Response.StatusCode == HttpStatusCode.OK);
                context.AddData("ElapsedMilliseconds", context.Response.ElapsedMilliseconds);
            }

            return(Task.CompletedTask);
        }
        public async Task CreateTableWhenNoTableNameInSchema()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync($"drop table if exists createtablenotablename;");

                var storage = CreateStorage(StorageMode.Insert);
                var context = new DataContext(null, new SpiderOptions(),
                                              new Request(), new Response());
                var typeName = typeof(CreateTableEntity2);
                var entity   = new CreateTableEntity2();

                var items = new List <CreateTableEntity2> {
                    entity
                };
                context.AddData(typeName, items);
                await storage.HandleAsync(context);

                var list = (await conn.QueryAsync <CreateTableEntity2>($"SELECT * FROM createtablenotablename"))
                           .ToList();
                Assert.Single(list);
                entity = list.First();
                Assert.Equal("xxx", entity.Str1);
                Assert.Equal("yyy", entity.Str2);
                Assert.Equal(655, entity.Required);
                Assert.Equal(0, entity.Decimal);
                Assert.Equal(600, entity.Long);
                Assert.Equal(400, entity.Double);
                Assert.Equal(200.0F, entity.Float);
                await conn.ExecuteAsync($"drop table if exists createtablenotablename;");
            }
        }
        public async Task CreateTableNoSchema()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync("drop table if exists createtableentity1;");

                {
                    var storage  = CreateStorage(StorageMode.Insert);
                    var dfc      = new DataContext(null, null, null, null);
                    var typeName = typeof(CreateTableEntity1);
                    var entity   = new CreateTableEntity1();

                    var items = new List <CreateTableEntity1> {
                        entity
                    };
                    dfc.AddData(typeName, items);
                    await storage.HandleAsync(dfc);

                    var list = (await conn.QueryAsync <CreateTableEntity1>("SELECT * FROM createtableentity1")).ToList();
                    Assert.Single(list);
                    entity = list.First();
                    Assert.Equal("xxx", entity.Str1);
                    Assert.Equal("yyy", entity.Str2);
                    Assert.Equal(655, entity.Required);
                    Assert.Equal(0, entity.Decimal);
                    Assert.Equal(600, entity.Long);
                    Assert.Equal(400, entity.Double);
                    Assert.Equal(200.0F, entity.Float);
                    await conn.ExecuteAsync("drop table if exists createtableentity1;");
                }
            }
        }
        public async Task Store_Should_Success()
        {
            var mongoCollection = new Mock <IMongoCollection <BsonDocument> >();

            var mongoDatabase = new Mock <IMongoDatabase>();

            mongoDatabase.Setup(d =>
                                d.GetCollection <BsonDocument>(It.IsAny <string>(), It.IsAny <MongoCollectionSettings>()))
            .Returns(mongoCollection.Object);

            var mongoClient = new Mock <IMongoClient>();

            mongoClient.Setup(d => d.GetDatabase(It.IsAny <string>(), It.IsAny <MongoDatabaseSettings>()))
            .Returns(mongoDatabase.Object);

            var mongoEntityStorage = new MongoEntityStorage(mongoClient.Object);

            var dfc      = new DataContext(null, null, null, null);
            var typeName = typeof(CreateTableEntity1);
            var entity   = new CreateTableEntity1();

            var items = new List <CreateTableEntity1>
            {
                entity
            };

            dfc.AddData(typeName, items);
            await mongoEntityStorage.HandleAsync(dfc);
        }
        public async Task UpdatePartColumns()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync(
                    $"drop table if exists {Escape}test{Escape}.{Escape}updatepartcolumns{Escape};");

                {
                    var storage = CreateStorage(StorageMode.InsertIgnoreDuplicate);
                    var context = new DataContext(null, new SpiderOptions(),
                                                  new Request(), new Response());
                    var typeName = typeof(CreateTableEntity6);
                    var entity   = new CreateTableEntity6();

                    var items = new List <CreateTableEntity6> {
                        entity
                    };
                    context.AddData(typeName, items);
                    await storage.HandleAsync(context);

                    var dfc2 = new DataContext(null, new SpiderOptions(),
                                               new Request(), new Response());
                    var now = DateTime.Now;
                    dfc2.AddData(typeName,
                                 new List <CreateTableEntity6>
                    {
                        new CreateTableEntity6
                        {
                            Str1           = "TTT",
                            DateTime       = now,
                            DateTimeOffset = now,
                            Double         = 888,
                            Float          = 999F,
                            Required       = 888
                        }
                    });
                    var storage2 = CreateStorage(StorageMode.Update);
                    await storage2.HandleAsync(dfc2);

                    var list = (await conn.QueryAsync <CreateTableEntity6>(
                                    $"SELECT * FROM {Escape}test{Escape}.{Escape}updatepartcolumns{Escape}"))
                               .ToList();
                    Assert.Single(list);
                    entity = list.First();
                    Assert.Equal("TTT", entity.Str1);
                    Assert.Equal("yyy", entity.Str2);
                    Assert.Equal(655, entity.Required);
                    Assert.Equal(0, entity.Decimal);
                    Assert.Equal(600, entity.Long);
                    Assert.Equal(888, entity.Double);
                    Assert.Equal(200.0F, entity.Float);

                    await conn.ExecuteAsync(
                        $"drop table if exists {Escape}test{Escape}.{Escape}updatepartcolumns{Escape};");
                }
            }
        }
        public async Task UpdateAllColumns()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync("drop table if exists test.dbo.createtableprimay;");

                {
                    var storage  = CreateStorage(StorageMode.InsertIgnoreDuplicate);
                    var dfc      = new DataContext(null, null, null, null);
                    var typeName = typeof(CreateTableEntity4);
                    var entity   = new CreateTableEntity4();

                    var items = new List <CreateTableEntity4> {
                        entity
                    };
                    dfc.AddData(typeName, items);
                    await storage.HandleAsync(dfc);

                    var dfc2 = new DataContext(null, null, null, null);

                    var now = DateTime.Now;
                    dfc2.AddData(typeName,
                                 new List <CreateTableEntity4>
                    {
                        new CreateTableEntity4
                        {
                            Str1 = "TTT", DateTime = now, DateTimeOffset = now, Double = 888
                        }
                    });
                    var storage2 = CreateStorage(StorageMode.Update);
                    await storage2.HandleAsync(dfc2);

                    var list = (await conn.QueryAsync <CreateTableEntity4>("SELECT * FROM test.dbo.createtableprimay"))
                               .ToList();
                    Assert.Single(list);
                    entity = list.First();
                    Assert.Equal("TTT", entity.Str1);
                    Assert.Equal("yyy", entity.Str2);
                    Assert.Equal(655, entity.Required);
                    Assert.Equal(0, entity.Decimal);
                    Assert.Equal(600, entity.Long);
                    Assert.Equal(888, entity.Double);
                    Assert.Equal(200.0F, entity.Float);

                    await conn.ExecuteAsync("drop table if exists test.dbo.createtableprimay;");
                }
            }
        }
Example #8
0
            protected override Task Parse(DataContext context)
            {
                var typeName = typeof(News).FullName;

                context.AddData(typeName,
                                new News
                {
                    Url     = context.Request.RequestUri.ToString(),
                    Title   = context.Request.Properties["title"]?.Trim(),
                    Summary = context.Request.Properties["summary"]?.Trim(),
                    Views   = int.Parse(context.Request.Properties["views"]),
                    Content = context.Selectable.Select(Selectors.XPath(".//div[@id='news_body']")).Value
                              ?.Trim()
                });
                return(Task.CompletedTask);
            }
        public async Task AutoIncPrimary()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync(
                    $"drop table if exists {Escape}test{Escape}.{Escape}createtableautoincprimay{Escape};");

                {
                    var storage = CreateStorage(StorageMode.Insert);
                    var context = new DataContext(null, new SpiderOptions(),
                                                  new Request(), new Response());
                    var typeName = typeof(CreateTableEntity5);
                    var entity   = new CreateTableEntity5();
                    var items    = new List <CreateTableEntity5> {
                        entity, entity
                    };
                    context.AddData(typeName, items);
                    await storage.HandleAsync(context);

                    var list =
                        (await conn.QueryAsync <CreateTableEntity5>(
                             $"SELECT * FROM {Escape}test{Escape}.{Escape}createtableautoincprimay{Escape}"))
                        .ToList();
                    Assert.Equal(2, list.Count);
                    entity = list.First();
                    Assert.Equal("xxx", entity.Str1);
                    Assert.Equal("yyy", entity.Str2);
                    Assert.Equal(655, entity.Required);
                    Assert.Equal(0, entity.Decimal);
                    Assert.Equal(600, entity.Long);
                    Assert.Equal(400, entity.Double);
                    Assert.Equal(200.0F, entity.Float);
                    Assert.Equal(1, entity.Id);
                    Assert.Equal(2, list[1].Id);

                    var primaries = (await conn.QueryAsync <PrimaryInfo>(
                                         $"SELECT t.CONSTRAINT_TYPE, c.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS t, INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS c WHERE t.TABLE_NAME = c.TABLE_NAME AND t.TABLE_SCHEMA = 'test' AND t.CONSTRAINT_TYPE = 'PRIMARY KEY' AND t.TABLE_NAME='createtableautoincprimay';")
                                     ).ToList();
                    Assert.Single(primaries);
                    Assert.Equal("id", primaries[0].COLUMN_NAME);
                    await conn.ExecuteAsync(
                        $"drop table if exists {Escape}test{Escape}.{Escape}createtableautoincprimay{Escape};");
                }
            }
        }
Example #10
0
        public async Task MultiPrimary()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync(
                    $"drop table if exists {Escape}test{Escape}.{Escape}createtablemultiprimay{Escape};");

                var storage = CreateStorage(StorageMode.Insert);
                var context = new DataContext(null, new SpiderOptions(),
                                              new Request(), new Response());
                var typeName = typeof(CreateTableEntity8);
                var entity   = new CreateTableEntity8();

                var items = new List <CreateTableEntity8> {
                    entity
                };
                context.AddData(typeName, items);
                await storage.HandleAsync(context);

                var list = (await conn.QueryAsync <CreateTableEntity8>(
                                $"SELECT * FROM {Escape}test{Escape}.{Escape}createtablemultiprimay{Escape}"))
                           .ToList();
                Assert.Single(list);
                entity = list.First();
                Assert.Equal("xxx", entity.Str1);
                Assert.Equal("yyy", entity.Str2);
                Assert.Equal(655, entity.Required);
                Assert.Equal(0, entity.Decimal);
                Assert.Equal(600, entity.Long);
                Assert.Equal(400, entity.Double);
                Assert.Equal(200.0F, entity.Float);

                var primaries = (await conn.QueryAsync <PrimaryInfo>(
                                     $"SELECT t.CONSTRAINT_TYPE, c.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS t, INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS c WHERE t.TABLE_NAME = c.TABLE_NAME AND t.TABLE_SCHEMA = 'test' AND t.CONSTRAINT_TYPE = 'PRIMARY KEY' AND t.TABLE_NAME='createtablemultiprimay';")
                                 ).ToList();
                _testOutputHelper.WriteLine(JsonConvert.SerializeObject(primaries));
                var columnNames = primaries.Select(x => x.COLUMN_NAME).ToList();
                Assert.Equal(2, primaries.Count);
                Assert.Contains("str2", columnNames);
                Assert.Contains("decimal", columnNames);
                await conn.ExecuteAsync(
                    $"drop table if exists {Escape}test{Escape}.{Escape}createtablemultiprimay{Escape};");
            }
        }
        public async Task CreateTableAutoIncPrimary()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync("drop table if exists test.dbo.createtableautoincprimay;");

                {
                    var storage  = CreateStorage(StorageMode.Insert);
                    var dfc      = new DataContext(null, null, null, null);
                    var typeName = typeof(CreateTableEntity5);
                    var entity   = new CreateTableEntity5();

                    var items = new List <CreateTableEntity5> {
                        entity, entity
                    };
                    dfc.AddData(typeName, items);
                    await storage.HandleAsync(dfc);

                    var list =
                        (await conn.QueryAsync <CreateTableEntity5>("SELECT * FROM test.dbo.createtableautoincprimay"))
                        .ToList();
                    Assert.Equal(2, list.Count);
                    entity = list.First();
                    Assert.Equal("xxx", entity.Str1);
                    Assert.Equal("yyy", entity.Str2);
                    Assert.Equal(655, entity.Required);
                    Assert.Equal(0, entity.Decimal);
                    Assert.Equal(600, entity.Long);
                    Assert.Equal(400, entity.Double);
                    Assert.Equal(200.0F, entity.Float);
                    Assert.Equal(1, entity.Id);
                    Assert.Equal(2, list[1].Id);

                    var primaries = (await conn.QueryAsync <IndexInfo>
                                         (@"USE test; EXEC sp_pkeys @table_name='createtableautoincprimay'")
                                     ).ToList();
                    Assert.Single(primaries);
                    Assert.Equal("id", primaries[0].COLUMN_NAME);
                    Assert.Equal(1, primaries[0].KEY_SEQ);
                    await conn.ExecuteAsync("drop table if exists test.dbo.createtableautoincprimay;");
                }
            }
        }
        public async Task Indexes()
        {
            using (var conn = CreateConnection())
            {
                await conn.ExecuteAsync(
                    $"drop table if exists {Escape}test{Escape}.{Escape}createtableindexes{Escape};");

                {
                    var storage = CreateStorage(StorageMode.Insert);

                    var context = new DataContext(null, new SpiderOptions(),
                                                  new Request(), new Response());
                    var typeName = typeof(CreateTableEntity5);
                    var entity   = new CreateTableEntity9();

                    var items = new List <CreateTableEntity9> {
                        entity
                    };
                    context.AddData(typeName, items);
                    await storage.HandleAsync(context);

                    var indexes = (await conn.QueryAsync <IndexInfo>
                                       ("show index from test.createtableindexes")
                                   ).ToList();
                    Assert.Equal(6, indexes.Count);
                    Assert.Contains(indexes,
                                    x => x.Key_name == "INDEX_STR1" && x.Non_unique == 1 && x.Column_name == "str1");
                    Assert.Contains(indexes, x =>
                                    x.Key_name == "INDEX_STR1_STR2" && x.Non_unique == 1 && x.Column_name == "str1");
                    Assert.Contains(indexes, x =>
                                    x.Key_name == "INDEX_STR1_STR2" && x.Non_unique == 1 && x.Column_name == "str2");
                    Assert.Contains(indexes,
                                    x => x.Key_name == "UNIQUE_STR3" && x.Non_unique == 0 && x.Column_name == "str3");
                    Assert.Contains(indexes, x =>
                                    x.Key_name == "UNIQUE_STR3_STR4" && x.Non_unique == 0 && x.Column_name == "str3");
                    Assert.Contains(indexes, x =>
                                    x.Key_name == "UNIQUE_STR3_STR4" && x.Non_unique == 0 && x.Column_name == "str4");

                    await conn.ExecuteAsync(
                        $"drop table if exists {Escape}test{Escape}.{Escape}createtableindexes{Escape};");
                }
            }
        }
Example #13
0
 protected virtual void AddParsedResult <T>(DataContext context, IEnumerable <T> results)
     where T : EntityBase <T>, new()
 {
     if (results != null)
     {
         var type  = typeof(T);
         var items = context.GetData(type);
         if (items == null)
         {
             var list = new List <T>();
             list.AddRange(results);
             context.AddData(type, list);
         }
         else
         {
             items.AddRange(results);
         }
     }
 }
        public async Task InsertAndUpdate()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync(
                    $"drop table if exists {Escape}test{Escape}.{Escape}createtableprimay{Escape};");

                {
                    var storage = CreateStorage(StorageMode.InsertAndUpdate);
                    var context = new DataContext(null, new SpiderOptions(),
                                                  new Request(), new Response());
                    var typeName = typeof(CreateTableEntity4);
                    var entity   = new CreateTableEntity4();

                    var items = new List <CreateTableEntity4> {
                        entity, new CreateTableEntity4 {
                            Str1 = "zzz"
                        }
                    };
                    context.AddData(typeName, items);

                    await storage.HandleAsync(context);

                    var list = (await conn.QueryAsync <CreateTableEntity4>(
                                    $"SELECT * FROM {Escape}test{Escape}.{Escape}createtableprimay{Escape}"))
                               .ToList();
                    Assert.Single(list);
                    entity = list.First();
                    Assert.Equal("zzz", entity.Str1);
                    Assert.Equal("yyy", entity.Str2);
                    Assert.Equal(655, entity.Required);
                    Assert.Equal(0, entity.Decimal);
                    Assert.Equal(600, entity.Long);
                    Assert.Equal(400, entity.Double);
                    Assert.Equal(200.0F, entity.Float);

                    await conn.ExecuteAsync(
                        $"drop table if exists {Escape}test{Escape}.{Escape}createtableprimay{Escape};");
                }
            }
        }
Example #15
0
        public void RightPath_Test()
        {
            const string path      = "/randomString";
            const string originUrl = "https://bitly.com";
            var          urls      = new[]
            {
                new Url {
                    OriginUrl = originUrl, ShortUrl = Settings.GetShortUrl(path)
                }
            };

            DataContext.AddData(urls);

            string redirectUrl = null;

            MiddlewareInvoke(path, s => { redirectUrl = s; });
            Assert.AreEqual(redirectUrl, originUrl);

            DataContext.Clear();
        }
        public async Task UseTransaction()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync("drop table if exists test.dbo.createtableprimay;");

                {
                    var storage =
                        (RelationalDatabaseEntityStorageBase)CreateStorage(StorageMode.InsertIgnoreDuplicate);
                    storage.UseTransaction = true;
                    var dfc      = new DataContext(null, null, null, null);
                    var typeName = typeof(CreateTableEntity4);
                    var entity   = new CreateTableEntity4();

                    var items = new List <CreateTableEntity4> {
                        entity, entity, entity
                    };
                    dfc.AddData(typeName, items);
                    await storage.HandleAsync(dfc);

                    var list = (await conn.QueryAsync <CreateTableEntity4>("SELECT * FROM test.dbo.createtableprimay"))
                               .ToList();
                    Assert.Single(list);
                    entity = list.First();
                    Assert.Equal("xxx", entity.Str1);
                    Assert.Equal("yyy", entity.Str2);
                    Assert.Equal(655, entity.Required);
                    Assert.Equal(0, entity.Decimal);
                    Assert.Equal(600, entity.Long);
                    Assert.Equal(400, entity.Double);
                    Assert.Equal(200.0F, entity.Float);

                    await conn.ExecuteAsync("drop table if exists test.dbo.createtableprimay;");
                }
            }
        }
        public async Task IgnoreCase()
        {
            using (var conn = CreateConnection())
            {
                // 如果实体的 Schema 没有配置表名,则使用类名
                await conn.ExecuteAsync($"drop table if exists {Escape}test{Escape}.{Escape}IgnoreCase{Escape};");

                {
                    var storage = (RelationalDatabaseEntityStorageBase)CreateStorage(StorageMode.Insert);
                    storage.IgnoreCase = false;
                    var context = new DataContext(null, new SpiderOptions(),
                                                  new Request(), new Response());
                    var typeName = typeof(CreateTableEntity7);
                    var entity   = new CreateTableEntity7();

                    var items = new List <CreateTableEntity7> {
                        entity
                    };
                    context.AddData(typeName, items);
                    await storage.HandleAsync(context);

                    var list = (await conn.QueryAsync <CreateTableEntity7>(
                                    $"SELECT * FROM {Escape}test{Escape}.{Escape}IgnoreCase{Escape}")).ToList();
                    Assert.Single(list);
                    entity = list.First();
                    Assert.Equal("xxx", entity.Str1);
                    Assert.Equal("yyy", entity.Str2);
                    Assert.Equal(655, entity.Required);
                    Assert.Equal(0, entity.Decimal);
                    Assert.Equal(600, entity.Long);
                    Assert.Equal(400, entity.Double);
                    Assert.Equal(200.0F, entity.Float);
                    await conn.ExecuteAsync($"drop table if exists {Escape}test{Escape}.{Escape}IgnoreCase{Escape};");
                }
            }
        }
Example #18
0
 protected override Task Parse(DataContext context)
 {
     context.AddData("URL", context.Request.RequestUri);
     context.AddData("Title", context.Selectable.XPath(".//title")?.Value);
     return(Task.CompletedTask);
 }
Example #19
0
        /// <summary>
        /// This method will load all basic properties for the specified model object
        /// </summary>
        public static void LoadAssociations <TModel>(this TModel me, DataContext context, IPrincipal principal, params String[] loadProperties) where TModel : IIdentifiedEntity
        {
            if (me == null || me.LoadState >= context.LoadState)
            {
                return;
            }
            else if (context.Transaction != null)
            {
                return;
            }

#if DEBUG
            Stopwatch sw = new Stopwatch();
            sw.Start();
#endif

            // Cache get classification property - thiz makez us fasters
            PropertyInfo classProperty = null;
            if (!s_classificationProperties.TryGetValue(typeof(TModel), out classProperty))
            {
                classProperty = typeof(TModel).GetRuntimeProperty(typeof(TModel).GetTypeInfo().GetCustomAttribute <ClassifierAttribute>()?.ClassifierProperty ?? "____XXX");
                if (classProperty != null)
                {
                    classProperty = typeof(TModel).GetRuntimeProperty(classProperty.GetCustomAttribute <SerializationReferenceAttribute>()?.RedirectProperty ?? classProperty.Name);
                }
                lock (s_lockObject)
                    if (!s_classificationProperties.ContainsKey(typeof(TModel)))
                    {
                        s_classificationProperties.Add(typeof(TModel), classProperty);
                    }
            }

            // Classification property?
            String classValue = classProperty?.GetValue(me)?.ToString();

            // Cache the props so future kitties can call it
            IEnumerable <PropertyInfo> properties = null;
            var propertyCacheKey = $"{me.GetType()}.FullName[{classValue}]";
            if (!s_runtimeProperties.TryGetValue(propertyCacheKey, out properties))
            {
                lock (s_runtimeProperties)
                {
                    properties = me.GetType().GetRuntimeProperties().Where(o => o.GetCustomAttribute <DataIgnoreAttribute>() == null && o.GetCustomAttributes <AutoLoadAttribute>().Any(p => p.ClassCode == classValue || p.ClassCode == null) && typeof(IdentifiedData).IsAssignableFrom(o.PropertyType.StripGeneric())).ToList();

                    if (!s_runtimeProperties.ContainsKey(propertyCacheKey))
                    {
                        s_runtimeProperties.Add(propertyCacheKey, properties);
                    }
                }
            }

            // Load fast or lean mode only root associations which will appear on the wire
            if (context.LoadState == LoadState.PartialLoad)
            {
                if (me.LoadState == LoadState.PartialLoad) // already partially loaded :/
                {
                    return;
                }
                else
                {
                    me.LoadState = LoadState.PartialLoad;
                    properties   = properties.Where(o => o.GetCustomAttribute <XmlAttributeAttribute>() != null || o.GetCustomAttribute <XmlElementAttribute>() != null).ToList();
                }
            }

            // Iterate over the properties and load the properties
            foreach (var pi in properties)
            {
                if (loadProperties.Length > 0 &&
                    !loadProperties.Contains(pi.Name))
                {
                    continue;
                }

                // Map model type to domain
                var adoPersister = ApplicationServiceContext.Current.GetService(pi.PropertyType.StripGeneric()) as IAdoPersistenceService;

                // Loading associations, so what is the associated type?
                if (typeof(IList).IsAssignableFrom(pi.PropertyType) &&
                    adoPersister is IAdoAssociativePersistenceService &&
                    me.Key.HasValue) // List so we select from the assoc table where we are the master table
                {
                    // Is there not a value?
                    var assocPersister = adoPersister as IAdoAssociativePersistenceService;

                    // We want to query based on our PK and version if applicable
                    decimal?        versionSequence = (me as IBaseEntityData)?.ObsoletionTime.HasValue == true ? (me as IVersionedEntity)?.VersionSequence : null;
                    var             assoc           = assocPersister.GetFromSource(context, me.Key.Value, versionSequence);
                    ConstructorInfo ci = null;
                    if (!m_constructors.TryGetValue(pi.PropertyType, out ci))
                    {
                        var type = pi.PropertyType.StripGeneric();
                        while (type != typeof(Object) && ci == null)
                        {
                            ci   = pi.PropertyType.GetConstructor(new Type[] { typeof(IEnumerable <>).MakeGenericType(type) });
                            type = type.BaseType;
                        }
                        if (ci != null)
                        {
                            lock (s_lockObject)
                                if (!m_constructors.ContainsKey(pi.PropertyType))
                                {
                                    m_constructors.Add(pi.PropertyType, ci);
                                }
                        }
                        else
                        {
                            throw new InvalidOperationException($"This is odd, you seem to have a list with no constructor -> {pi.PropertyType}");
                        }
                    }
                    var listValue = ci.Invoke(new object[] { assoc });
                    pi.SetValue(me, listValue);
                }
                else if (typeof(IIdentifiedEntity).IsAssignableFrom(pi.PropertyType)) // Single
                {
                    // Single property, we want to execute a get on the key property
                    var pid = pi;
                    if (pid.Name.EndsWith("Xml")) // Xml purposes only
                    {
                        pid = pi.DeclaringType.GetProperty(pi.Name.Substring(0, pi.Name.Length - 3));
                    }
                    var redirectAtt = pid.GetCustomAttribute <SerializationReferenceAttribute>();

                    // We want to issue a query
                    var keyProperty = redirectAtt != null?pi.DeclaringType.GetProperty(redirectAtt.RedirectProperty) : pi.DeclaringType.GetProperty(pid.Name + "Key");

                    if (keyProperty == null)
                    {
                        continue;
                    }
                    var keyValue = keyProperty?.GetValue(me);
                    if (keyValue == null ||
                        Guid.Empty.Equals(keyValue))
                    {
                        continue; // No key specified
                    }
                    // This is kinda messy.. maybe iz to be changez
                    object value = null;
                    if (!context.Data.TryGetValue(keyValue.ToString(), out value))
                    {
                        value = adoPersister.Get(context, (Guid)keyValue);
                        context.AddData(keyValue.ToString(), value);
                    }
                    pid.SetValue(me, value);
                }
            }
#if DEBUG
            sw.Stop();
            s_traceSource.TraceVerbose("Load associations for {0} took {1} ms", me, sw.ElapsedMilliseconds);
#endif

            if (me.LoadState == LoadState.New)
            {
                me.LoadState = LoadState.FullLoad;
            }
        }