public override void BulkUpsertIndexValues(string objectFullName, ObjectMetadata metadata, IEnumerable<object[]> indexValues)
{
using (var db = OpenSchema())
{
// first, stage the values
_StageBulkIndexValues(db, objectFullName, metadata, indexValues);
try
{
// second, merge them into the primary index table
_MergeBulkIndexValues(db, objectFullName, metadata);
}
finally
{
_CleanStageBulkIndexValues(db, objectFullName);
}
}
}
public override void BulkUpsertIndexValues(string objectFullName, ObjectMetadata metadata, IEnumerable<object[]> indexValues)
{
using (var db = OpenData())
{
var tableName = db.MakeQuotedName(_CreateTableName(db, objectFullName));
var colNames = new List<string>();
colNames.Add(db.MakeQuotedName(IDColumn));
foreach (var idx in metadata.Indexes)
{
colNames.Add(idx.Name);
}
db.ExecuteBulkInsert(
indexValues,
tableName,
colNames.ToArray());
}
}
public override void ProvisionIndex(ObjectMetadata metadata)
{
using (var db = OpenSchema())
{
var tableName = _CreateTableName(db, metadata.ObjectFullName);
string idColName = db.MakeQuotedName(IDColumn);
string colDefs = idColName + " INT NOT NULL PRIMARY KEY";
string colIndexNames = idColName;
if (null != metadata.Indexes && 0 < metadata.Indexes.Length)
{
colDefs += "," + string.Join(",", metadata.Indexes.Select(i => _CreateColumnDef(db, i)).ToArray());
colIndexNames += "," + string.Join(",", metadata.Indexes.Select(i => db.MakeQuotedName(i.Name)).ToArray());
}
var createTableSQL = string.Format(SQLStatements.CreateTableIfNotExists, tableName, colDefs);
db.ExecuteNonQuery(createTableSQL);
var indexTableSQL = string.Format(SQLStatements.CreateIndex, tableName, colIndexNames);
db.ExecuteNonQuery(indexTableSQL);
}
}
private void _StageBulkIndexValues(IDatabaseService db, string objectFullName, ObjectMetadata metadata, IEnumerable<object[]> records)
{
var stagingTableName = _CreateStagingTableName(db, objectFullName);
if (!_TableExists(db, stagingTableName))
{
_ProvisionIndexStaging(db, metadata);
}
var colNames = new List<string>();
colNames.Add(db.MakeQuotedName(IDColumn));
foreach (var idx in metadata.Indexes)
{
colNames.Add(db.MakeQuotedName(idx.Name));
}
db.ExecuteBulkInsert(
records,
"[ZeroG]." + db.MakeQuotedName(stagingTableName),
colNames.ToArray());
}
private void _MergeBulkIndexValues(IDatabaseService db, string objectFullName, ObjectMetadata metadata)
{
string tableName = _CreateTableName(db, objectFullName);
string stagingTableName = _CreateStagingTableName(db, objectFullName);
var colNameList = new List<string>();
var idCol = db.MakeQuotedName(IDColumn);
colNameList.Add(idCol);
string setStatement = idCol + " = source." + idCol;
string valuesStatement = "source." + idCol;
foreach (var idx in metadata.Indexes)
{
var idxName = db.MakeQuotedName(idx.Name);
colNameList.Add(idxName);
setStatement += "," + idxName + " = source." + idxName;
valuesStatement += ",source." + idxName;
}
var sql = string.Format(@"MERGE [ZeroG].{0} AS target
USING (SELECT {1} FROM [ZeroG].{2}) AS source ({1})
ON (target.{3} = source.{3})
WHEN MATCHED THEN
UPDATE SET {4}
WHEN NOT MATCHED THEN
INSERT ({1})
VALUES ({5});",
db.MakeQuotedName(tableName), // 0
string.Join(",", colNameList.ToArray()), // 1
db.MakeQuotedName(stagingTableName), // 2
idCol, // 3
setStatement, // 4
valuesStatement // 5
);
db.ExecuteNonQuery(sql);
}
private void _ProvisionIndexStaging(IDatabaseService db, ObjectMetadata metadata)
{
var tableName = _CreateStagingTableName(db, metadata.ObjectFullName);
string idColName = db.MakeQuotedName(IDColumn);
string colDefs = idColName + " [int] NOT NULL";
string colIndexNames = idColName;
if (null != metadata.Indexes && 0 < metadata.Indexes.Length)
{
colDefs += "," + string.Join(",", metadata.Indexes.Select(i => _CreateColumnDef(db, i)).ToArray());
colIndexNames += "," + string.Join(",", metadata.Indexes.Select(i => db.MakeQuotedName(i.Name)).ToArray());
}
var createTableSQL = string.Format(SQLStatements.CreateStagingTableIfNotExists, tableName, colDefs, _FileGroup);
db.ExecuteNonQuery(createTableSQL);
}
public void CacheWithObjectRemovedEvent()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
string ns = ObjectTestHelper.NameSpace1;
string obj = ObjectTestHelper.ObjectName1;
string objectFullName = ObjectNaming.CreateFullObjectName(ns, obj);
ObjectMetadata objectMetadata = new ObjectMetadata(ns, obj,
new ObjectIndexMetadata[]
{
new ObjectIndexMetadata("IntIndex1", ObjectIndexType.Integer),
new ObjectIndexMetadata("StrIndex1", ObjectIndexType.String, 15)
});
// Temporarily use the ObjectService to provision the Object's Metadata.
using (var svc = new ObjectService(config))
{
svc.CreateNameSpace(new ObjectNameSpaceConfig(ns,
"ZeroG Test", "Unit Test", DateTime.Now));
svc.ProvisionObjectStore(objectMetadata);
}
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString();
string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString();
string strVal2 = ObjectIndex.Create("StrIdx", "Val3").ToString();
object[] cacheParams = new object[] { objectFullName, intVal1, strVal1 };
object[] cacheParams2 = new object[] { objectFullName, intVal1, strVal2 };
int[] objectIds = new int[] { 1000, 2000 };
int[] objectIds2 = new int[] { 3000, 4000 };
try
{
// add a couple items to cache - they should not be cleared
// when the object's metadata is removed
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
CacheTotals totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNotNull(cache.Get(cacheParams));
Assert.IsNotNull(cache.Get(cacheParams2));
metadata.Remove(objectFullName);
// NOTE: Dirty cache entries are not removed from cache as soon as
// they become dirty. They are removed lazily when Get/Set
// are called.
// CacheTotals and Enumerate will return Dirty objects and are
// only meant to be used by the Cache Cleaner and not directly
// by the Object Indexer.
Assert.AreEqual(0, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(0, totals.TotalQueries);
Assert.AreEqual(0, totals.TotalValues);
Assert.IsNull(cache.Get(cacheParams));
Assert.IsNull(cache.Get(cacheParams2));
// The metadata needs to be added back so TestCleanup completes
metadata.StoreMetadata(objectMetadata);
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
public void CacheWithDependencyVersionChangeEvent()
{
Config config = ObjectTestHelper.GetConfigWithCaching();
string ns = ObjectTestHelper.NameSpace1;
string obj = ObjectTestHelper.ObjectName1;
string obj2 = ObjectTestHelper.ObjectName2;
string objectFullName = ObjectNaming.CreateFullObjectName(ns, obj);
string objectFullName2 = ObjectNaming.CreateFullObjectName(ns, obj2);
ObjectMetadata objectMetadata = new ObjectMetadata(ns, obj,
new ObjectIndexMetadata[]
{
new ObjectIndexMetadata("IntIndex1", ObjectIndexType.Integer),
new ObjectIndexMetadata("StrIndex1", ObjectIndexType.String, 15)
});
ObjectMetadata objectMetadata2 = new ObjectMetadata(ns, obj2, null,
new string[] { obj }); // Make Object1 a dependency of Object2.
// Now whenever Object2's version changes, Object1's change event will fire as well.
// NOTE: Do not supply Full Object Name as objects can only depend on other objects
// within their namespace.
// Temporarily use the ObjectService to provision the Object's Metadata.
using (var svc = new ObjectService(config))
{
svc.CreateNameSpace(new ObjectNameSpaceConfig(ns,
"ZeroG Test", "Unit Test", DateTime.Now));
svc.ProvisionObjectStore(objectMetadata);
svc.ProvisionObjectStore(objectMetadata2);
}
ObjectMetadataStore metadata = new ObjectMetadataStore(config);
ObjectVersionStore versions = new ObjectVersionStore(config, metadata);
ObjectIndexerCache cache = new ObjectIndexerCache(metadata, versions);
string intVal1 = ObjectIndex.Create("IntIdx", 5).ToString();
string strVal1 = ObjectIndex.Create("StrIdx", "Val1").ToString();
string strVal2 = ObjectIndex.Create("StrIdx", "Val3").ToString();
object[] cacheParams = new object[] { objectFullName, intVal1, strVal1 };
object[] cacheParams2 = new object[] { objectFullName, intVal1, strVal2 };
int[] objectIds = new int[] { 1000, 2000 };
int[] objectIds2 = new int[] { 3000, 4000 };
try
{
// add a couple items to cache - they should not be returned
// once the objects dependency object version changes
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
CacheTotals totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNotNull(cache.Get(cacheParams));
Assert.IsNotNull(cache.Get(cacheParams2));
versions.Update(objectFullName2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNull(cache.Get(cacheParams));
Assert.IsNull(cache.Get(cacheParams2));
cache.Set(objectIds, cacheParams);
cache.Set(objectIds2, cacheParams2);
Assert.AreEqual(2, cache.EnumerateCache().Count());
totals = cache.Totals;
Assert.AreEqual(2, totals.TotalQueries);
Assert.AreEqual(4, totals.TotalValues);
Assert.IsNotNull(cache.Get(cacheParams));
Assert.IsNotNull(cache.Get(cacheParams2));
}
finally
{
cache.Dispose();
versions.Dispose();
metadata.Dispose();
}
}
public void BulkUpsert()
{
var provider = IndexProvider;
var metadata = new ObjectMetadata(NameSpace1, ObjectName1,
new ObjectIndexMetadata[]
{
new ObjectIndexMetadata("IntCol", ObjectIndexType.Integer),
new ObjectIndexMetadata("TextCol", ObjectIndexType.String, 15),
new ObjectIndexMetadata("DecCol", ObjectIndexType.Decimal, 7, 2),
new ObjectIndexMetadata("DateTimeCol", ObjectIndexType.DateTime),
new ObjectIndexMetadata("BinCol", ObjectIndexType.Binary, 16)
});
provider.ProvisionIndex(
metadata);
Int32 testInt = 3447;
String testStr = "Test Value";
Decimal testDec = 156.12M;
DateTime testDate = new DateTime(2011, 2, 14, 3, 10, 0);
Guid testGuid = new Guid("76F5FB10BAEF4DE09578B3EB91FF6653");
provider.BulkUpsertIndexValues(
ObjectFullName1,
metadata,
new object[][]
{
new object[] { 1000, testInt, testStr, testDec, testDate, testGuid.ToByteArray() },
new object[] { 500, 0, "asdf", new Decimal(5.4), DateTime.UtcNow, Guid.NewGuid().ToByteArray() }
});
Assert.AreEqual(2, provider.CountObjects(ObjectFullName1));
int[] ids = provider.Find(ObjectFullName1, ObjectIndex.Create("ID", 1000));
Assert.IsNotNull(ids);
Assert.AreEqual(1, ids.Length);
Assert.AreEqual(1000, ids[0]);
ids = provider.Find(ObjectFullName1, ObjectIndex.Create("IntCol", testInt));
Assert.IsNotNull(ids);
Assert.AreEqual(1, ids.Length);
Assert.AreEqual(1000, ids[0]);
ids = provider.Find(ObjectFullName1, ObjectIndex.Create("TextCol", testStr));
Assert.IsNotNull(ids);
Assert.AreEqual(1, ids.Length);
Assert.AreEqual(1000, ids[0]);
ids = provider.Find(ObjectFullName1, ObjectIndex.Create("DecCol", testDec));
Assert.IsNotNull(ids);
Assert.AreEqual(1, ids.Length);
Assert.AreEqual(1000, ids[0]);
ids = provider.Find(ObjectFullName1, ObjectIndex.Create("DateTimeCol", testDate));
Assert.IsNotNull(ids);
Assert.AreEqual(1, ids.Length);
Assert.AreEqual(1000, ids[0]);
ids = provider.Find(ObjectFullName1, ObjectIndex.Create("BinCol", testGuid.ToByteArray()));
Assert.IsNotNull(ids);
Assert.AreEqual(1, ids.Length);
Assert.AreEqual(1000, ids[0]);
// test that values are updated
provider.BulkUpsertIndexValues(
ObjectFullName1,
metadata,
new object[][]
{
new object[] { 1000, testInt, testStr, testDec, testDate, testGuid.ToByteArray() },
new object[] { 500, 5, "asdfupdated", new Decimal(5.7), DateTime.UtcNow, Guid.NewGuid().ToByteArray() }
});
Assert.AreEqual(2, provider.CountObjects(ObjectFullName1));
Assert.AreEqual(0, provider.Count(ObjectFullName1, @"{""IntCol"" : 0}", metadata.Indexes));
Assert.AreEqual(1, provider.Count(ObjectFullName1, @"{""IntCol"" : 5}", metadata.Indexes));
Assert.AreEqual(0, provider.Count(ObjectFullName1, @"{""TextCol"" : ""asdf""}", metadata.Indexes));
Assert.AreEqual(1, provider.Count(ObjectFullName1, @"{""TextCol"" : ""asdfupdated""}", metadata.Indexes));
}
public void ProvisionObjectStore(ObjectMetadata metadata)
{
_service.ProvisionObjectStore(metadata);
}
public void ProvisionObjectStore(ObjectMetadata metadata)
{
if (null == metadata)
{
throw new ArgumentNullException("metadata");
}
else
{
if (string.IsNullOrEmpty(metadata.NameSpace))
{
throw new ArgumentNullException("metadata.NameSpace");
}
else if (string.IsNullOrEmpty(metadata.ObjectName))
{
throw new ArgumentNullException("metadata.Name");
}
else
{
if (!_objectNaming.NameSpaceExists(metadata.NameSpace))
{
throw new ArgumentException("Unknown namespace: " + metadata.NameSpace);
}
// do not allow existing object store to be re-provisioned
if (ObjectNameExists(metadata.NameSpace, metadata.ObjectName))
{
throw new ArgumentException("Object store already exists: " + ObjectNaming.CreateFullObjectKey(metadata.NameSpace, metadata.ObjectName));
}
using (var trans = new TransactionScope(TransactionScopeOption.Required, _DefaultTransactionOptions))
{
// This call validates the format of the metadata and will throw and exception
// if it is invalid.
lock (_objectMetadata)
{
_objectMetadata.StoreMetadata(metadata);
}
if (null != metadata.Indexes && 0 < metadata.Indexes.Length)
{
lock (_objectIndexer)
{
_objectIndexer.ProvisionIndex(metadata);
}
}
trans.Complete();
}
}
}
}
