/// <summary>
/// delete an item given its catalog table
/// </summary>
/// <param name="item"></param>
/// <param name="table"></param>
public void DeleteItem(IDataItem item, bool HardDelete)
{
//the table to call delete item with
IReferenceDataTable delete_table = null;
//check if this is a reference table
if (TableMetadata.ReferenceTable)
{
//get the reference table
delete_table = (IReferenceDataTable)this;
}
else
{
//get the mod field
IColumnMetadata column = TableMetadata.ColumnsMetadata[TableMetadata.ModField];
//get the value to mode
object obj = item.GetType().GetField(column.FieldMapping).GetValue(item);
Int64 id = Convert.ToInt64(obj);
//mod to get the bucket index
Int64 bucket = id % TableMetadata.CatalogMetadata.Catalog.ComputeNode.GlobalBucketCount;
//get the bucket table
delete_table = TableMetadata.CatalogMetadata.Catalog.Buckets[bucket].BucketDataTables[TableMetadata.TableName];
}
//call delete
if (delete_table != null)
{
DeleteItem(item, delete_table, HardDelete);
}
}
/// <summary>
/// process an item
/// </summary>
/// <param name="item"></param>
/// <param name="table"></param>
private bool ProcessItem(IDataItem item, IReferenceDataTable table)
{
//check if item is deleted
if (table.IsDeleted(item))
{
table.DeleteItem(item, true);
return(false);
}
//add/update the list
else
{
return(table.AddItem(item));
}
}
/// <summary>
/// delete an item and its references
/// </summary>
/// <param name="item"></param>
/// <param name="table"></param>
private void DeleteItem(IDataItem item, IReferenceDataTable table, bool HardDelete)
{
//remove from table
object objTable;
object objCatalog;
IDataItem removed;
//check we need to handle relationship
if (table is RelationshipDataTable)
{
//parse the table's relationships
RelationshipDataTable rel_table = table as RelationshipDataTable;
//get the fields
long parent_id = Convert.ToInt64(item.GetType().GetField(rel_table.RelationshipMetadata.ParentRelationshipField).GetValue(item));
long child_id = Convert.ToInt64(item.GetType().GetField(rel_table.RelationshipMetadata.ChildRelationshipField).GetValue(item));
//lock the resource
lock (rel_table.RelationshipParentLookup)
{
//remove the item
if (table.TryRemove(item.PrimaryKey, table.TableMetadata.DataType, out objTable))
{
removed = (IDataItem)objTable;
}
//check if this is a hard delete
//HardDelete removes the relations
//SoftDelete leave the references
if (HardDelete)
{
//remove this relation
ConcurrentDictionary <long, long> lookup;
if (rel_table.RelationshipParentLookup.TryGetValue(parent_id, out lookup))
{
//remove all lookup entries for association
long id;
bool found = true;
while (found)
{
found = lookup.TryRemove(item.PrimaryKey, out id);
}
if (lookup.Count == 0)
{
//remove the lookup for this parent
if (rel_table.RelationshipParentLookup.TryRemove(parent_id, out lookup))
{
//lookup.Clear();
//lookup = null;
}
}
}
//get the parent item
object objParent;
IDataItem parent_item;
//dynamic parent_item;
IBucketDataTable parent_table = rel_table.Bucket.BucketDataTables[rel_table.RelationshipMetadata.ParentTable.TableName];
if (parent_table.TryGetValue(parent_id, parent_table.TableMetadata.DataType, out objParent))
{
parent_item = (IDataItem)objParent;
//parent_item.GetType().GetField(rel_table.RelationshipMetadata.ParentChildCollection).SetValue(parent_item, rel_table.GetChildList(parent_id));
var list = (List <Tuple <List <long>, long> >)parent_item.GetType().GetField(rel_table.RelationshipMetadata.ParentChildCollection).GetValue(parent_item);
if (rel_table.RelationshipMetadata.DistinctRelationship)
{
var rel_item = list.FirstOrDefault(t => t.Item2 == child_id);
if (rel_item != null)
{
if (rel_item.Item1.Contains(item.PrimaryKey))
{
rel_item.Item1.Remove(item.PrimaryKey);
}
if (rel_item.Item1.Count == 0)
{
list.Remove(rel_item);
}
}
}
else
{
var rel_item = list.FirstOrDefault(t => t.Item1.Any(l => l == item.PrimaryKey));
if (rel_item != null)
{
list.Remove(rel_item);
}
}
}
}
}
//if(rel_table.DataItems.Count % 1000 == 0)
//{
// System.Diagnostics.Trace.WriteLine(
// string.Format(
// "Lookup Status - Catalog:{0}, Table:{1}, TableCount:{2}, LookupCount:{3}",
// rel_table.TableMetadata.CatalogMetadata.Catalog.CatalogName,
// rel_table.TableMetadata.TableName,
// rel_table.DataItems.Count,
// rel_table.RelationshipParentLookup.Count));
//}
}
else if (table is BucketDataTable)
{
//remove the item
if (table.TryRemove(item.PrimaryKey, table.TableMetadata.DataType, out objTable))
{
removed = (IDataItem)objTable;
}
//parse the table's relationships
BucketDataTable bucket_table = table as BucketDataTable;
foreach (RelationshipMetadata rel in table.Catalog.CatalogMetadata.Relationships.Where(r => r.Value.ParentTable.TableName == table.TableMetadata.TableName).Select(r => r.Value))
{
//get the relationship table
RelationshipDataTable rel_table = bucket_table.Bucket.BucketDataTables[rel.RelationTable.TableName] as RelationshipDataTable;
//lock the resource
lock (rel_table.RelationshipParentLookup)
{
//remove the lookup for this parent
ConcurrentDictionary <long, long> lookup;
if (rel_table.RelationshipParentLookup.TryRemove(item.PrimaryKey, out lookup))
{
lookup.Clear();
lookup = null;
}
}
}
}
//remove from catalog
if (table.Catalog.CatalogDataTables[table.TableMetadata.TableName].TryRemove(item.PrimaryKey, table.TableMetadata.DataType, out objCatalog))
{
removed = (IDataItem)objCatalog;
}
}
/// <summary>
/// process an item
/// </summary>
/// <param name="item"></param>
/// <param name="table"></param>
private bool ProcessItem(IDataItem item, IReferenceDataTable table)
{
//check if item is deleted
if (table.IsDeleted(item))
{
table.DeleteItem(item, true);
return false;
}
//add/update the list
else
{
return table.AddItem(item);
}
}
/// <summary>
/// process a row
/// </summary>
/// <param name="dr"></param>
/// <param name="item"></param>
/// <param name="table_metadata"></param>
private void ProcessRow(IDataReader dr, IDataItem item, IReferenceDataTable table, bool update_last_update)
{
//get the data
List<Tuple<string,object>> data = new List<Tuple<string,object>>();
for (int x = 0; x < dr.FieldCount; x++)
{
//get the column metadata
string column_name = dr.GetName(x);
IColumnMetadata col = null;
if(table.TableMetadata.ColumnsMetadata.ContainsKey(column_name))
col = table.TableMetadata.ColumnsMetadata[column_name];
if (col != null)
{
//get the value
object value = dr[x];
data.Add(new Tuple<string, object>(column_name, value));
try
{
//check if the column is a collection
if (col.ColumnType == typeof(List<Tuple<List<long>, long>>))
{
string string_value = value.ToString();
List<Tuple<List<long>, long>> list = new List<Tuple<List<long>, long>>();
//get the relationship for this field
var rel = table.Catalog.CatalogMetadata.Relationships.Values.FirstOrDefault(r => r.ParentChildCollection == column_name);
//check if null
if (!dr.IsDBNull(x))
{
foreach (var s in string_value.Split(','))
{
string[] tuple = s.Split('|');
if (tuple.Length == 2)
{
long item1;
long item2;
if (Int64.TryParse(tuple[0], out item1))
{
if (Int64.TryParse(tuple[1], out item2))
{
if (rel != null && rel.DistinctRelationship)
{
var existing = list.FirstOrDefault(t => t.Item2 == item2);
if (existing == null)
{
List<long> sub_list = new List<long>();
sub_list.Add(item1);
list.Add(new Tuple<List<long>, long>(sub_list, item2));
}
else
{
existing.Item1.Add(item1);
}
}
else
{
List<long> sub_list = new List<long>();
sub_list.Add(item1);
list.Add(new Tuple<List<long>, long>(sub_list, item2));
}
}
}
}
}
}
//List<long> list = string_value.Split(',').Select(t => TryParseInt64(t))
// .Where(i => i.HasValue)
// .Select(i => i.Value)
// .ToList();
item.GetType().GetField(col.FieldMapping).SetValue(item, list);
}
else
{
//check if null
if (!dr.IsDBNull(x))
{
//set the value with reflection
item.GetType().GetField(col.FieldMapping).SetValue(item, value);
}
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine(ex.ToString());
}
//update the last update value
if (column_name == table.TableMetadata.SaturationUpdateField && update_last_update)
table.UpdateLastValue(value);
}
}
if (item is Mention)
{
if ((item as Mention).Tags == null)
{
System.Diagnostics.Trace.WriteLine("NO");
}
}
}
/// <summary>
/// delete an item and its references
/// </summary>
/// <param name="item"></param>
/// <param name="table"></param>
private void DeleteItem(IDataItem item, IReferenceDataTable table, bool HardDelete)
{
//remove from table
object objTable;
object objCatalog;
IDataItem removed;
//check we need to handle relationship
if (table is RelationshipDataTable)
{
//parse the table's relationships
RelationshipDataTable rel_table = table as RelationshipDataTable;
//get the fields
long parent_id = Convert.ToInt64(item.GetType().GetField(rel_table.RelationshipMetadata.ParentRelationshipField).GetValue(item));
long child_id = Convert.ToInt64(item.GetType().GetField(rel_table.RelationshipMetadata.ChildRelationshipField).GetValue(item));
//lock the resource
lock (rel_table.RelationshipParentLookup)
{
//remove the item
if (table.TryRemove(item.PrimaryKey, table.TableMetadata.DataType, out objTable))
{
removed = (IDataItem)objTable;
}
//check if this is a hard delete
//HardDelete removes the relations
//SoftDelete leave the references
if (HardDelete)
{
//remove this relation
ConcurrentDictionary<long, long> lookup;
if (rel_table.RelationshipParentLookup.TryGetValue(parent_id, out lookup))
{
//remove all lookup entries for association
long id;
bool found = true;
while (found)
found = lookup.TryRemove(item.PrimaryKey, out id);
if (lookup.Count == 0)
{
//remove the lookup for this parent
if (rel_table.RelationshipParentLookup.TryRemove(parent_id, out lookup))
{
//lookup.Clear();
//lookup = null;
}
}
}
//get the parent item
object objParent;
IDataItem parent_item;
//dynamic parent_item;
IBucketDataTable parent_table = rel_table.Bucket.BucketDataTables[rel_table.RelationshipMetadata.ParentTable.TableName];
if (parent_table.TryGetValue(parent_id, parent_table.TableMetadata.DataType, out objParent))
{
parent_item = (IDataItem)objParent;
//parent_item.GetType().GetField(rel_table.RelationshipMetadata.ParentChildCollection).SetValue(parent_item, rel_table.GetChildList(parent_id));
var list = (List<Tuple<List<long>,long>>)parent_item.GetType().GetField(rel_table.RelationshipMetadata.ParentChildCollection).GetValue(parent_item);
if (rel_table.RelationshipMetadata.DistinctRelationship)
{
var rel_item = list.FirstOrDefault(t => t.Item2 == child_id);
if (rel_item != null)
{
if (rel_item.Item1.Contains(item.PrimaryKey))
rel_item.Item1.Remove(item.PrimaryKey);
if (rel_item.Item1.Count == 0)
list.Remove(rel_item);
}
}
else
{
var rel_item = list.FirstOrDefault(t => t.Item1.Any(l => l == item.PrimaryKey));
if (rel_item != null)
list.Remove(rel_item);
}
}
}
}
//if(rel_table.DataItems.Count % 1000 == 0)
//{
// System.Diagnostics.Trace.WriteLine(
// string.Format(
// "Lookup Status - Catalog:{0}, Table:{1}, TableCount:{2}, LookupCount:{3}",
// rel_table.TableMetadata.CatalogMetadata.Catalog.CatalogName,
// rel_table.TableMetadata.TableName,
// rel_table.DataItems.Count,
// rel_table.RelationshipParentLookup.Count));
//}
}
else if (table is BucketDataTable)
{
//remove the item
if (table.TryRemove(item.PrimaryKey, table.TableMetadata.DataType, out objTable))
{
removed = (IDataItem)objTable;
}
//parse the table's relationships
BucketDataTable bucket_table = table as BucketDataTable;
foreach (RelationshipMetadata rel in table.Catalog.CatalogMetadata.Relationships.Where(r => r.Value.ParentTable.TableName == table.TableMetadata.TableName).Select(r => r.Value))
{
//get the relationship table
RelationshipDataTable rel_table = bucket_table.Bucket.BucketDataTables[rel.RelationTable.TableName] as RelationshipDataTable;
//lock the resource
lock (rel_table.RelationshipParentLookup)
{
//remove the lookup for this parent
ConcurrentDictionary<long, long> lookup;
if (rel_table.RelationshipParentLookup.TryRemove(item.PrimaryKey, out lookup))
{
lookup.Clear();
lookup = null;
}
}
}
}
//remove from catalog
if (table.Catalog.CatalogDataTables[table.TableMetadata.TableName].TryRemove(item.PrimaryKey, table.TableMetadata.DataType, out objCatalog))
{
removed = (IDataItem)objCatalog;
}
}
/// <summary>
/// process a row
/// </summary>
/// <param name="dr"></param>
/// <param name="item"></param>
/// <param name="table_metadata"></param>
private void ProcessRow(IDataReader dr, IDataItem item, IReferenceDataTable table, bool update_last_update)
{
//get the data
List <Tuple <string, object> > data = new List <Tuple <string, object> >();
for (int x = 0; x < dr.FieldCount; x++)
{
//get the column metadata
string column_name = dr.GetName(x);
IColumnMetadata col = null;
if (table.TableMetadata.ColumnsMetadata.ContainsKey(column_name))
{
col = table.TableMetadata.ColumnsMetadata[column_name];
}
if (col != null)
{
//get the value
object value = dr[x];
data.Add(new Tuple <string, object>(column_name, value));
try
{
//check if the column is a collection
if (col.ColumnType == typeof(List <Tuple <List <long>, long> >))
{
string string_value = value.ToString();
List <Tuple <List <long>, long> > list = new List <Tuple <List <long>, long> >();
//get the relationship for this field
var rel = table.Catalog.CatalogMetadata.Relationships.Values.FirstOrDefault(r => r.ParentChildCollection == column_name);
//check if null
if (!dr.IsDBNull(x))
{
foreach (var s in string_value.Split(','))
{
string[] tuple = s.Split('|');
if (tuple.Length == 2)
{
long item1;
long item2;
if (Int64.TryParse(tuple[0], out item1))
{
if (Int64.TryParse(tuple[1], out item2))
{
if (rel != null && rel.DistinctRelationship)
{
var existing = list.FirstOrDefault(t => t.Item2 == item2);
if (existing == null)
{
List <long> sub_list = new List <long>();
sub_list.Add(item1);
list.Add(new Tuple <List <long>, long>(sub_list, item2));
}
else
{
existing.Item1.Add(item1);
}
}
else
{
List <long> sub_list = new List <long>();
sub_list.Add(item1);
list.Add(new Tuple <List <long>, long>(sub_list, item2));
}
}
}
}
}
}
//List<long> list = string_value.Split(',').Select(t => TryParseInt64(t))
// .Where(i => i.HasValue)
// .Select(i => i.Value)
// .ToList();
item.GetType().GetField(col.FieldMapping).SetValue(item, list);
}
else
{
//check if null
if (!dr.IsDBNull(x))
{
//set the value with reflection
item.GetType().GetField(col.FieldMapping).SetValue(item, value);
}
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine(ex.ToString());
}
//update the last update value
if (column_name == table.TableMetadata.SaturationUpdateField && update_last_update)
{
table.UpdateLastValue(value);
}
}
}
if (item is Mention)
{
if ((item as Mention).Tags == null)
{
System.Diagnostics.Trace.WriteLine("NO");
}
}
}
/// <summary>
/// feed the thread pool for saturation
/// </summary>
private void Saturate()
{
//the currently executing threads and thier events
Thread[] threads = new Thread[SaturationThreadCount];
ManualResetEvent[] events = new ManualResetEvent[SaturationThreadCount];
for (int x = 0; x < SaturationThreadCount; x++)
{
events[x] = new ManualResetEvent(true);
}
//saturator is paused
bool Paused = false;
//main loop
while (true)
{
try
{
//check the memory usage
var memStatus = SystemInfo.GetMemoryStatusEx();
if (ComputeNode.Purging)
{
//mark as paused
Paused = true;
//report memory usage
System.Diagnostics.Trace.WriteLine(string.Format("Saturator has reach maximum memory threshold - TotalMemory:{0} MB, AvailableMemory:{1} MB, AvailablePercent:{2} %",
memStatus.ullTotalPhys / 1024 / 1024, memStatus.ullAvailPhys / 1024 / 1024, (double)memStatus.ullAvailPhys / (double)memStatus.ullTotalPhys * 100.0));
//sleep for 5 minutes
if (eventStop.WaitOne(1 * 60 * 1000))
{
return;
}
}
else
{
//report is we are unpausing
if (Paused)
{
//report memory usage
System.Diagnostics.Trace.WriteLine(string.Format("Saturator is starting after pause - TotalMemory:{0} MB, AvailableMemory:{1} MB, AvailablePercent:{2} %",
memStatus.ullTotalPhys / 1024 / 1024, memStatus.ullAvailPhys / 1024 / 1024, (double)memStatus.ullAvailPhys / (double)memStatus.ullTotalPhys * 100.0));
}
//mark as not paused
Paused = false;
//check to refresh catalogs
//if ((DateTime.UtcNow - LastCatalogRefresh).TotalMinutes > 5)
//{
// computeNode.RefreshCatalogs();
// LastCatalogRefresh = DateTime.UtcNow;
// PrintReport();
//}
//System.Diagnostics.Trace.WriteLine("Take Snapshot");
//Thread.Sleep(1 * 60 * 1000);
//wait at least on signaled event
int index = WaitHandle.WaitAny(events, 5000);
//check the return
if (index != WaitHandle.WaitTimeout)
{
//get the next bucket data table to service
IReferenceDataTable table = computeNode.SaturationTables.Where(t => t.Saturating == false).OrderBy(t => t.NextSaturation).FirstOrDefault();
//make sure we got one
if (table != null && DateTime.UtcNow > table.NextSaturation)
{
//create the thread
table.Saturating = true;
events[index].Reset();
threads[index] = new Thread(new ParameterizedThreadStart(SaturateDataTable));
threads[index].Start(new SaturateDataTableData()
{
eventDone = events[index],
table = table
});
//events[index].WaitOne();
////for testing remove all nodes
///////////////////////////////////
//foreach (var item in table.DataItems)
//{
// var args = new object[] { };
// object o = item.GetType().GetProperty("Value").GetValue(item, args);
// table.DeleteItem((IDataItem)o);
//}
//threads[index] = null;
///////////////////////////////////
////check profiler here
//GC.Collect(GC.MaxGeneration);
//GC.WaitForPendingFinalizers();
////foreach (IDataTable table in table.Catalog.ComputeNode.SaturationTables)
//// System.Diagnostics.Trace.WriteLine(string.Format("The status - Name:{0}, Count:{1}", table.TableMetadata.TableName, table.DataItems.Count));
//System.Diagnostics.Trace.WriteLine("Done");
//Thread.Sleep(1000000000);
//string test = "";
}
else
{
//sleep for little while
if (eventStop.WaitOne(5000))
{
return;
}
}
}
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine(ex.ToString());
}
if (eventStop.WaitOne(0))
{
return;
}
}
}
