public override void BulkUpsertIndexValues(string objectFullName, ObjectMetadata metadata, IEnumerable<object[]> indexValues) { using (var db = OpenSchema()) { // first, stage the values _StageBulkIndexValues(db, objectFullName, metadata, indexValues); try { // second, merge them into the primary index table _MergeBulkIndexValues(db, objectFullName, metadata); } finally { _CleanStageBulkIndexValues(db, objectFullName); } } }
public override void BulkUpsertIndexValues(string objectFullName, ObjectMetadata metadata, IEnumerable<object[]> indexValues) { using (var db = OpenData()) { var tableName = db.MakeQuotedName(_CreateTableName(db, objectFullName)); var colNames = new List<string>(); colNames.Add(db.MakeQuotedName(IDColumn)); foreach (var idx in metadata.Indexes) { colNames.Add(idx.Name); } db.ExecuteBulkInsert( indexValues, tableName, colNames.ToArray()); } }
public override void ProvisionIndex(ObjectMetadata metadata) { using (var db = OpenSchema()) { var tableName = _CreateTableName(db, metadata.ObjectFullName); string idColName = db.MakeQuotedName(IDColumn); string colDefs = idColName + " INT NOT NULL PRIMARY KEY"; string colIndexNames = idColName; if (null != metadata.Indexes && 0 < metadata.Indexes.Length) { colDefs += "," + string.Join(",", metadata.Indexes.Select(i => _CreateColumnDef(db, i)).ToArray()); colIndexNames += "," + string.Join(",", metadata.Indexes.Select(i => db.MakeQuotedName(i.Name)).ToArray()); } var createTableSQL = string.Format(SQLStatements.CreateTableIfNotExists, tableName, colDefs); db.ExecuteNonQuery(createTableSQL); var indexTableSQL = string.Format(SQLStatements.CreateIndex, tableName, colIndexNames); db.ExecuteNonQuery(indexTableSQL); } }
private void _StageBulkIndexValues(IDatabaseService db, string objectFullName, ObjectMetadata metadata, IEnumerable<object[]> records) { var stagingTableName = _CreateStagingTableName(db, objectFullName); if (!_TableExists(db, stagingTableName)) { _ProvisionIndexStaging(db, metadata); } var colNames = new List<string>(); colNames.Add(db.MakeQuotedName(IDColumn)); foreach (var idx in metadata.Indexes) { colNames.Add(db.MakeQuotedName(idx.Name)); } db.ExecuteBulkInsert( records, "[ZeroG]." + db.MakeQuotedName(stagingTableName), colNames.ToArray()); }
private void _MergeBulkIndexValues(IDatabaseService db, string objectFullName, ObjectMetadata metadata) { string tableName = _CreateTableName(db, objectFullName); string stagingTableName = _CreateStagingTableName(db, objectFullName); var colNameList = new List<string>(); var idCol = db.MakeQuotedName(IDColumn); colNameList.Add(idCol); string setStatement = idCol + " = source." + idCol; string valuesStatement = "source." + idCol; foreach (var idx in metadata.Indexes) { var idxName = db.MakeQuotedName(idx.Name); colNameList.Add(idxName); setStatement += "," + idxName + " = source." + idxName; valuesStatement += ",source." + idxName; } var sql = string.Format(@"MERGE [ZeroG].{0} AS target USING (SELECT {1} FROM [ZeroG].{2}) AS source ({1}) ON (target.{3} = source.{3}) WHEN MATCHED THEN UPDATE SET {4} WHEN NOT MATCHED THEN INSERT ({1}) VALUES ({5});", db.MakeQuotedName(tableName), // 0 string.Join(",", colNameList.ToArray()), // 1 db.MakeQuotedName(stagingTableName), // 2 idCol, // 3 setStatement, // 4 valuesStatement // 5 ); db.ExecuteNonQuery(sql); }
private void _ProvisionIndexStaging(IDatabaseService db, ObjectMetadata metadata) { var tableName = _CreateStagingTableName(db, metadata.ObjectFullName); string idColName = db.MakeQuotedName(IDColumn); string colDefs = idColName + " [int] NOT NULL"; string colIndexNames = idColName; if (null != metadata.Indexes && 0 < metadata.Indexes.Length) { colDefs += "," + string.Join(",", metadata.Indexes.Select(i => _CreateColumnDef(db, i)).ToArray()); colIndexNames += "," + string.Join(",", metadata.Indexes.Select(i => db.MakeQuotedName(i.Name)).ToArray()); } var createTableSQL = string.Format(SQLStatements.CreateStagingTableIfNotExists, tableName, colDefs, _FileGroup); db.ExecuteNonQuery(createTableSQL); }
public void CacheWithObjectRemovedEvent() { Config config = ObjectTestHelper.GetConfigWithCaching(); string ns = ObjectTestHelper.NameSpace1; string obj = ObjectTestHelper.ObjectName1; string objectFullName = ObjectNaming.CreateFullObjectName(ns, obj); ObjectMetadata objectMetadata = new ObjectMetadata(ns, obj, new ObjectIndexMetadata[] { new ObjectIndexMetadata("IntIndex1", ObjectIndexType.Integer), new ObjectIndexMetadata("StrIndex1", ObjectIndexType.String, 15) }); // Temporarily use the ObjectService to provision the Object's Metadata. using (var svc = new ObjectService(config)) { svc.CreateNameSpace(new ObjectNameSpaceConfig(ns, "ZeroG Test", "Unit Test", DateTime.Now)); svc.ProvisionObjectStore(objectMetadata); } ObjectMetadataStore metadata = new ObjectMetadataStore(config); ObjectVersionStore versions = new ObjectVersionStore(config, metadata); ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions); string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString(); string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString(); string strVal2 = ObjectIndex.Create("StrIdx", "Val3").ToString(); object[] cacheParams = new object[] { objectFullName, intVal1, strVal1 }; object[] cacheParams2 = new object[] { objectFullName, intVal1, strVal2 }; int[] objectIds = new int[] { 1000, 2000 }; int[] objectIds2 = new int[] { 3000, 4000 }; try { // add a couple items to cache - they should not be cleared // when the object's metadata is removed cache.Set(objectIds, cacheParams); cache.Set(objectIds2, cacheParams2); Assert.AreEqual(2, cache.EnumerateCache().Count()); CacheTotals totals = cache.Totals; Assert.AreEqual(2, totals.TotalQueries); Assert.AreEqual(4, totals.TotalValues); Assert.IsNotNull(cache.Get(cacheParams)); Assert.IsNotNull(cache.Get(cacheParams2)); metadata.Remove(objectFullName); // NOTE: Dirty cache entries are not removed from cache as soon as // they become dirty. They are removed lazily when Get/Set // are called. // CacheTotals and Enumerate will return Dirty objects and are // only meant to be used by the Cache Cleaner and not directly // by the Object Indexer. Assert.AreEqual(0, cache.EnumerateCache().Count()); totals = cache.Totals; Assert.AreEqual(0, totals.TotalQueries); Assert.AreEqual(0, totals.TotalValues); Assert.IsNull(cache.Get(cacheParams)); Assert.IsNull(cache.Get(cacheParams2)); // The metadata needs to be added back so TestCleanup completes metadata.StoreMetadata(objectMetadata); } finally { cache.Dispose(); versions.Dispose(); metadata.Dispose(); } }
public void CacheWithDependencyVersionChangeEvent() { Config config = ObjectTestHelper.GetConfigWithCaching(); string ns = ObjectTestHelper.NameSpace1; string obj = ObjectTestHelper.ObjectName1; string obj2 = ObjectTestHelper.ObjectName2; string objectFullName = ObjectNaming.CreateFullObjectName(ns, obj); string objectFullName2 = ObjectNaming.CreateFullObjectName(ns, obj2); ObjectMetadata objectMetadata = new ObjectMetadata(ns, obj, new ObjectIndexMetadata[] { new ObjectIndexMetadata("IntIndex1", ObjectIndexType.Integer), new ObjectIndexMetadata("StrIndex1", ObjectIndexType.String, 15) }); ObjectMetadata objectMetadata2 = new ObjectMetadata(ns, obj2, null, new string[] { obj }); // Make Object1 a dependency of Object2. // Now whenever Object2's version changes, Object1's change event will fire as well. // NOTE: Do not supply Full Object Name as objects can only depend on other objects // within their namespace. // Temporarily use the ObjectService to provision the Object's Metadata. using (var svc = new ObjectService(config)) { svc.CreateNameSpace(new ObjectNameSpaceConfig(ns, "ZeroG Test", "Unit Test", DateTime.Now)); svc.ProvisionObjectStore(objectMetadata); svc.ProvisionObjectStore(objectMetadata2); } ObjectMetadataStore metadata = new ObjectMetadataStore(config); ObjectVersionStore versions = new ObjectVersionStore(config, metadata); ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions); string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString(); string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString(); string strVal2 = ObjectIndex.Create("StrIdx", "Val3").ToString(); object[] cacheParams = new object[] { objectFullName, intVal1, strVal1 }; object[] cacheParams2 = new object[] { objectFullName, intVal1, strVal2 }; int[] objectIds = new int[] { 1000, 2000 }; int[] objectIds2 = new int[] { 3000, 4000 }; try { // add a couple items to cache - they should not be returned // once the objects dependency object version changes cache.Set(objectIds, cacheParams); cache.Set(objectIds2, cacheParams2); Assert.AreEqual(2, cache.EnumerateCache().Count()); CacheTotals totals = cache.Totals; Assert.AreEqual(2, totals.TotalQueries); Assert.AreEqual(4, totals.TotalValues); Assert.IsNotNull(cache.Get(cacheParams)); Assert.IsNotNull(cache.Get(cacheParams2)); versions.Update(objectFullName2); Assert.AreEqual(2, cache.EnumerateCache().Count()); totals = cache.Totals; Assert.AreEqual(2, totals.TotalQueries); Assert.AreEqual(4, totals.TotalValues); Assert.IsNull(cache.Get(cacheParams)); Assert.IsNull(cache.Get(cacheParams2)); cache.Set(objectIds, cacheParams); cache.Set(objectIds2, cacheParams2); Assert.AreEqual(2, cache.EnumerateCache().Count()); totals = cache.Totals; Assert.AreEqual(2, totals.TotalQueries); Assert.AreEqual(4, totals.TotalValues); Assert.IsNotNull(cache.Get(cacheParams)); Assert.IsNotNull(cache.Get(cacheParams2)); } finally { cache.Dispose(); versions.Dispose(); metadata.Dispose(); } }
public void BulkUpsert() { var provider = IndexProvider; var metadata = new ObjectMetadata(NameSpace1, ObjectName1, new ObjectIndexMetadata[] { new ObjectIndexMetadata("IntCol", ObjectIndexType.Integer), new ObjectIndexMetadata("TextCol", ObjectIndexType.String, 15), new ObjectIndexMetadata("DecCol", ObjectIndexType.Decimal, 7, 2), new ObjectIndexMetadata("DateTimeCol", ObjectIndexType.DateTime), new ObjectIndexMetadata("BinCol", ObjectIndexType.Binary, 16) }); provider.ProvisionIndex( metadata); Int32 testInt = 3447; String testStr = "Test Value"; Decimal testDec = 156.12M; DateTime testDate = new DateTime(2011, 2, 14, 3, 10, 0); Guid testGuid = new Guid("76F5FB10BAEF4DE09578B3EB91FF6653"); provider.BulkUpsertIndexValues( ObjectFullName1, metadata, new object[][] { new object[] { 1000, testInt, testStr, testDec, testDate, testGuid.ToByteArray() }, new object[] { 500, 0, "asdf", new Decimal(5.4), DateTime.UtcNow, Guid.NewGuid().ToByteArray() } }); Assert.AreEqual(2, provider.CountObjects(ObjectFullName1)); int[] ids = provider.Find(ObjectFullName1, ObjectIndex.Create("ID", 1000)); Assert.IsNotNull(ids); Assert.AreEqual(1, ids.Length); Assert.AreEqual(1000, ids[0]); ids = provider.Find(ObjectFullName1, ObjectIndex.Create("IntCol", testInt)); Assert.IsNotNull(ids); Assert.AreEqual(1, ids.Length); Assert.AreEqual(1000, ids[0]); ids = provider.Find(ObjectFullName1, ObjectIndex.Create("TextCol", testStr)); Assert.IsNotNull(ids); Assert.AreEqual(1, ids.Length); Assert.AreEqual(1000, ids[0]); ids = provider.Find(ObjectFullName1, ObjectIndex.Create("DecCol", testDec)); Assert.IsNotNull(ids); Assert.AreEqual(1, ids.Length); Assert.AreEqual(1000, ids[0]); ids = provider.Find(ObjectFullName1, ObjectIndex.Create("DateTimeCol", testDate)); Assert.IsNotNull(ids); Assert.AreEqual(1, ids.Length); Assert.AreEqual(1000, ids[0]); ids = provider.Find(ObjectFullName1, ObjectIndex.Create("BinCol", testGuid.ToByteArray())); Assert.IsNotNull(ids); Assert.AreEqual(1, ids.Length); Assert.AreEqual(1000, ids[0]); // test that values are updated provider.BulkUpsertIndexValues( ObjectFullName1, metadata, new object[][] { new object[] { 1000, testInt, testStr, testDec, testDate, testGuid.ToByteArray() }, new object[] { 500, 5, "asdfupdated", new Decimal(5.7), DateTime.UtcNow, Guid.NewGuid().ToByteArray() } }); Assert.AreEqual(2, provider.CountObjects(ObjectFullName1)); Assert.AreEqual(0, provider.Count(ObjectFullName1, @"{""IntCol"" : 0}", metadata.Indexes)); Assert.AreEqual(1, provider.Count(ObjectFullName1, @"{""IntCol"" : 5}", metadata.Indexes)); Assert.AreEqual(0, provider.Count(ObjectFullName1, @"{""TextCol"" : ""asdf""}", metadata.Indexes)); Assert.AreEqual(1, provider.Count(ObjectFullName1, @"{""TextCol"" : ""asdfupdated""}", metadata.Indexes)); }
public void ProvisionObjectStore(ObjectMetadata metadata) { _service.ProvisionObjectStore(metadata); }
public void ProvisionObjectStore(ObjectMetadata metadata) { if (null == metadata) { throw new ArgumentNullException("metadata"); } else { if (string.IsNullOrEmpty(metadata.NameSpace)) { throw new ArgumentNullException("metadata.NameSpace"); } else if (string.IsNullOrEmpty(metadata.ObjectName)) { throw new ArgumentNullException("metadata.Name"); } else { if (!_objectNaming.NameSpaceExists(metadata.NameSpace)) { throw new ArgumentException("Unknown namespace: " + metadata.NameSpace); } // do not allow existing object store to be re-provisioned if (ObjectNameExists(metadata.NameSpace, metadata.ObjectName)) { throw new ArgumentException("Object store already exists: " + ObjectNaming.CreateFullObjectKey(metadata.NameSpace, metadata.ObjectName)); } using (var trans = new TransactionScope(TransactionScopeOption.Required, _DefaultTransactionOptions)) { // This call validates the format of the metadata and will throw and exception // if it is invalid. lock (_objectMetadata) { _objectMetadata.StoreMetadata(metadata); } if (null != metadata.Indexes && 0 < metadata.Indexes.Length) { lock (_objectIndexer) { _objectIndexer.ProvisionIndex(metadata); } } trans.Complete(); } } } }