private static void Main()
{
TTransport framedTransport = new TFramedTransport(new TSocket("localhost", 9160));
TTransport socketTransport = new TSocket("localhost", 9160);
TProtocol framedProtocol = new TBinaryProtocol(framedTransport);
TProtocol socketProtocol = new TBinaryProtocol(socketTransport);
var client = new Cassandra.Client(framedProtocol, framedProtocol); // all framed
//var client = new Cassandra.Client(socketProtocol, socketProtocol); // all socket
//var client = new Cassandra.Client(framedProtocol, socketProtocol); // in: framed out: socket
//var client = new Cassandra.Client(socketProtocol, framedProtocol); // in: socket out: framed
framedTransport.Open();
socketTransport.Open();
Console.WriteLine("Start");
client.set_keyspace("Keyspace1");
Console.WriteLine("Count Key");
var key = Encoding.ASCII.GetBytes("MyKey");
var columns = new List<byte[]>(new[] { Encoding.ASCII.GetBytes("MyColumn") });
var column_parent = new ColumnParent {
Column_family = "Standard1"
};
var predicate = new SlicePredicate {
Column_names = columns
};
client.get_count(key, column_parent, predicate, ConsistencyLevel.ALL);
Console.WriteLine("Done");
Console.Read();
}
// private string columnFamily = "users";
// private string keyspace = "test";
// private string key = "testKey";
// // Insert statement
//// private byte[] key = ByteEncoderHelper.LongEncoder.ToByteArray(i);
// private byte[] key = ByteEncoderHelper.LongEncoder.ToByteArray(2);
// private ColumnParent columnParent;
// private Column column;
// private ColumnPath columnPath;
// private ICluster cluster;
//
//
//
//cluster.Execute(new ExecutionBlock(delegate(CassandraClient client) {
// client.insert(key, columnParent, column, ConsistencyLevel.ONE);
// return null;
//}), keyspace);
// // Get statement
// byte[] key = ByteEncoderHelper.LongEncoder.ToByteArray(2);
//
// ICluster cluster = AquilesHelper.RetrieveCluster("Cassandra1");
// object rtnValue = cluster.Execute(new ExecutionBlock(delegate(CassandraClient client)
// {
// return client.get(key, columnPath, ConsistencyLevel.ONE);
// }), keyspace);
public void Run()
{
string columnFamily = "TestColumnFamily";
string keyspace = "TestKeyspace";
string key = "testKey";
string columnName = "testColumn";
string columnValue = "testValue";
// Insert statement
byte[] key2 = ByteEncoderHelper.LongEncoder.ToByteArray(1);
ColumnParent columnParent = new ColumnParent();
Column column = new Column()
{
Name = ByteEncoderHelper.UTF8Encoder.ToByteArray(columnName),
Timestamp = UnixHelper.UnixTimestamp,
Value = ByteEncoderHelper.UTF8Encoder.ToByteArray(columnValue),
};
columnParent.Column_family = columnFamily;
ICluster cluster = AquilesHelper.RetrieveCluster("Cassandra1");
cluster.Execute(new ExecutionBlock(delegate(CassandraClient client)
{
client.insert(key, columnParent, column, ConsistencyLevel.ONE);
return null;
}), keyspace);
}
/// <summary>
/// Biuld Thrift ColumnParent structure using ColumnFamily and SuperColumn information
/// </summary>
/// <param name="superColumn">name for the supercolumn (null in case there is not one)</param>
/// <returns>Thrift ColumnParent</returns>
protected ColumnParent BuildColumnParent(string superColumn)
{
ColumnParent columnParent = new ColumnParent();
bool isSuperColumnMissing = String.IsNullOrEmpty(superColumn);
columnParent.Column_family = this.ColumnFamily;
if (!isSuperColumnMissing)
{
columnParent.Super_column = ByteEncoderHelper.ToByteArray(superColumn);
}
return columnParent;
}
public int get_count(byte[] key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
{
send_get_count(key, column_parent, predicate, consistency_level);
return recv_get_count();
}
public void Read(TProtocol iprot)
{
TField field;
iprot.ReadStructBegin();
while (true)
{
field = iprot.ReadFieldBegin();
if (field.Type == TType.Stop) {
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.List) {
{
Keys = new List<byte[]>();
TList _list87 = iprot.ReadListBegin();
for( int _i88 = 0; _i88 < _list87.Count; ++_i88)
{
byte[] _elem89 = null;
_elem89 = iprot.ReadBinary();
Keys.Add(_elem89);
}
iprot.ReadListEnd();
}
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 2:
if (field.Type == TType.Struct) {
Column_parent = new ColumnParent();
Column_parent.Read(iprot);
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 3:
if (field.Type == TType.Struct) {
Predicate = new SlicePredicate();
Predicate.Read(iprot);
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 4:
if (field.Type == TType.I32) {
Consistency_level = (ConsistencyLevel)iprot.ReadI32();
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
default:
TProtocolUtil.Skip(iprot, field.Type);
break;
}
iprot.ReadFieldEnd();
}
iprot.ReadStructEnd();
}
public void Read(TProtocol iprot)
{
TField field;
iprot.ReadStructBegin();
while (true)
{
field = iprot.ReadFieldBegin();
if (field.Type == TType.Stop) {
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.Struct) {
Column_parent = new ColumnParent();
Column_parent.Read(iprot);
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 2:
if (field.Type == TType.Struct) {
Index_clause = new IndexClause();
Index_clause.Read(iprot);
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 3:
if (field.Type == TType.Struct) {
Column_predicate = new SlicePredicate();
Column_predicate.Read(iprot);
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 4:
if (field.Type == TType.I32) {
Consistency_level = (ConsistencyLevel)iprot.ReadI32();
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
default:
TProtocolUtil.Skip(iprot, field.Type);
break;
}
iprot.ReadFieldEnd();
}
iprot.ReadStructEnd();
}
public void send_insert(byte[] key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level)
{
oprot_.WriteMessageBegin(new TMessage("insert", TMessageType.Call, seqid_));
insert_args args = new insert_args();
args.Key = key;
args.Column_parent = column_parent;
args.Column = column;
args.Consistency_level = consistency_level;
args.Write(oprot_);
oprot_.WriteMessageEnd();
oprot_.Transport.Flush();
}
public void send_get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level)
{
oprot_.WriteMessageBegin(new TMessage("get_indexed_slices", TMessageType.Call, seqid_));
get_indexed_slices_args args = new get_indexed_slices_args();
args.Column_parent = column_parent;
args.Index_clause = index_clause;
args.Column_predicate = column_predicate;
args.Consistency_level = consistency_level;
args.Write(oprot_);
oprot_.WriteMessageEnd();
oprot_.Transport.Flush();
}
public Dictionary<byte[], List<ColumnOrSuperColumn>> multiget_slice(List<byte[]> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
{
send_multiget_slice(keys, column_parent, predicate, consistency_level);
return recv_multiget_slice();
}
public void insert(byte[] key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level)
{
send_insert(key, column_parent, column, consistency_level);
recv_insert();
}
/// <summary>
/// Returns a dictionary of parents with a list of items Dictionary[FolderID, List[Items]]
/// </summary>
/// <param name="itemIds"></param>
/// <returns></returns>
public Dictionary<UUID, List<UUID>> FindItemParentFolderIds(IEnumerable<UUID> itemIds)
{
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = ITEMPARENTS_CF;
SlicePredicate pred = new SlicePredicate();
pred.Column_names = new List<byte[]>();
pred.Column_names.Add(ByteEncoderHelper.UTF8Encoder.ToByteArray("parent"));
List<byte[]> allItemIdBytes = new List<byte[]>();
foreach (UUID id in itemIds)
{
allItemIdBytes.Add(ByteEncoderHelper.GuidEncoder.ToByteArray(id.Guid));
}
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
object val =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.multiget_slice(allItemIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
}), KEYSPACE);
Dictionary<byte[], List<ColumnOrSuperColumn>> itemParentCols = (Dictionary<byte[], List<ColumnOrSuperColumn>>)val;
Dictionary<UUID, List<UUID>> retParents = new Dictionary<UUID, List<UUID>>();
foreach (KeyValuePair<byte[], List<ColumnOrSuperColumn>> kvp in itemParentCols)
{
if (kvp.Value.Count == 1)
{
Column col = kvp.Value[0].Column;
UUID parentId = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Value));
if (!retParents.ContainsKey(parentId))
{
retParents.Add(parentId, new List<UUID>());
}
retParents[parentId].Add(new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(kvp.Key)));
}
}
return retParents;
}
public Guid FindItemParentFolderId(UUID itemId)
{
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = ITEMPARENTS_CF;
SlicePredicate pred = new SlicePredicate();
SliceRange range = new SliceRange();
range.Start = new byte[0];
range.Finish = new byte[0];
range.Reversed = false;
range.Count = int.MaxValue;
pred.Slice_range = range;
byte[] itemIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(itemId.Guid);
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
object val =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.get_slice(itemIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
}), KEYSPACE);
List<ColumnOrSuperColumn> indexCols = (List<ColumnOrSuperColumn>)val;
//no index means the item doesnt exist
if (indexCols.Count == 0)
{
return Guid.Empty;
}
var indexedColsByName = this.IndexColumnsByUTF8Name(indexCols);
return ByteEncoderHelper.GuidEncoder.FromByteArray(indexedColsByName["parent"].Value);
}
/// <summary>
/// Retrieves all the rows for the given list of keys in chunkSize chunks
/// </summary>
/// <param name="chunkSize"></param>
/// <param name="allKeys"></param>
/// <param name="colParent"></param>
/// <param name="pred"></param>
/// <param name="consistencyLevel"></param>
/// <returns></returns>
private Dictionary<byte[], List<ColumnOrSuperColumn>> RetrieveRowsInChunks(int chunkSize, List<byte[]> allKeys,
ColumnParent colParent, SlicePredicate pred, ConsistencyLevel consistencyLevel)
{
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
if (allKeys.Count <= chunkSize)
{
object val =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.multiget_slice(allKeys, colParent, pred, consistencyLevel);
}), KEYSPACE);
return (Dictionary<byte[], List<ColumnOrSuperColumn>>)val;
}
else
{
Dictionary<byte[], List<ColumnOrSuperColumn>> ret = new Dictionary<byte[], List<ColumnOrSuperColumn>>();
for (int i = 0; i < allKeys.Count; i += chunkSize)
{
int remaining = allKeys.Count - i;
List<byte[]> keys = allKeys.GetRange(i, remaining >= chunkSize ? chunkSize : remaining);
object val =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.multiget_slice(keys, colParent, pred, consistencyLevel);
}), KEYSPACE);
Dictionary<byte[], List<ColumnOrSuperColumn>> chunk = (Dictionary<byte[], List<ColumnOrSuperColumn>>)val;
foreach(KeyValuePair<byte[], List<ColumnOrSuperColumn>> kvp in chunk)
{
ret.Add(kvp.Key, kvp.Value);
}
}
return ret;
}
}
public InventoryItemBase GetItem(UUID itemId, UUID parentFolderHint)
{
//Retrieving an item requires a lookup of the parent folder followed by
//a retrieval of the item. This was a consious decision made since the
//inventory item data currently takes up the most space and a
//duplication of this data to prevent the index lookup
//would be expensive in terms of space required
try
{
Guid parentId;
if (parentFolderHint != UUID.Zero)
{
parentId = parentFolderHint.Guid;
}
else
{
parentId = FindItemParentFolderId(itemId);
}
if (parentId == Guid.Empty)
{
throw new InventoryObjectMissingException(String.Format("Item with ID {0} could not be found", itemId), "Item was not found in the index");
}
//try to retrieve the item. note that even though we have an index there is a chance we will
//not have the item data due to a race condition between index mutation and item mutation
byte[] itemIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(itemId.Guid);
byte[] folderIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(parentId);
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = FOLDERS_CF;
columnParent.Super_column = itemIdBytes;
SlicePredicate pred = new SlicePredicate();
SliceRange range = new SliceRange();
range.Start = new byte[0];
range.Finish = new byte[0];
range.Reversed = false;
range.Count = int.MaxValue;
pred.Slice_range = range;
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
object itemDataObj =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.get_slice(folderIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
}), KEYSPACE);
List<ColumnOrSuperColumn> itemCols = (List<ColumnOrSuperColumn>)itemDataObj;
if (itemCols.Count == 0)
{
throw new InventoryObjectMissingException(String.Format("Item with ID {0} could not be found", itemId), "Item was not found in its folder");
}
InventoryItemBase item = this.DecodeInventoryItem(itemCols, itemId.Guid, parentId);
return item;
}
catch (InventoryStorageException)
{
throw;
}
catch (Exception e)
{
_log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve item {0}: {1}", itemId, e);
throw new InventoryStorageException(e.Message, e);
}
}
public void Read(TProtocol iprot)
{
iprot.IncrementRecursionDepth();
try
{
TField field;
iprot.ReadStructBegin();
while (true)
{
field = iprot.ReadFieldBegin();
if (field.Type == TType.Stop)
{
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.String)
{
Key = iprot.ReadBinary();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 2:
if (field.Type == TType.Struct)
{
Column_parent = new ColumnParent();
Column_parent.Read(iprot);
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 3:
if (field.Type == TType.List)
{
{
Column_slices = new List <ColumnSlice>();
TList _list108 = iprot.ReadListBegin();
for (int _i109 = 0; _i109 < _list108.Count; ++_i109)
{
ColumnSlice _elem110;
_elem110 = new ColumnSlice();
_elem110.Read(iprot);
Column_slices.Add(_elem110);
}
iprot.ReadListEnd();
}
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 4:
if (field.Type == TType.Bool)
{
Reversed = iprot.ReadBool();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 5:
if (field.Type == TType.I32)
{
Count = iprot.ReadI32();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 6:
if (field.Type == TType.I32)
{
Consistency_level = (ConsistencyLevel)iprot.ReadI32();
}
else
{
TProtocolUtil.Skip(iprot, field.Type);
}
break;
default:
TProtocolUtil.Skip(iprot, field.Type);
break;
}
iprot.ReadFieldEnd();
}
iprot.ReadStructEnd();
}
finally
{
iprot.DecrementRecursionDepth();
}
}
public List<ColumnOrSuperColumn> get_slice(byte[] key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
{
send_get_slice(key, column_parent, predicate, consistency_level);
return recv_get_slice();
}
private List<UUID> GetActiveGestureItemIds(UUID userId)
{
try
{
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = USERACTIVEGESTURES_CF;
SlicePredicate pred = new SlicePredicate();
SliceRange range = new SliceRange();
range.Start = new byte[0];
range.Finish = new byte[0];
range.Reversed = false;
range.Count = int.MaxValue;
pred.Slice_range = range;
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
object retobj =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
byte[] userIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(userId.Guid);
return client.get_slice(userIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
}), KEYSPACE);
List<ColumnOrSuperColumn> cols = (List<ColumnOrSuperColumn>)retobj;
List<UUID> ret = new List<UUID>();
foreach (ColumnOrSuperColumn col in cols)
{
ret.Add(new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Column.Name)));
}
return ret;
}
catch (Exception e)
{
throw new InventoryStorageException(e.Message, e);
}
}
public Dictionary<byte[], int> multiget_count(List<byte[]> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
{
send_multiget_count(keys, column_parent, predicate, consistency_level);
return recv_multiget_count();
}
private List<ColumnOrSuperColumn> RetrieveAllColumnsInChunks(int chunkSize, byte[] key,
ColumnParent columnParent, SlicePredicate pred, ConsistencyLevel consistencyLevel)
{
pred.Slice_range.Count = chunkSize;
List<ColumnOrSuperColumn> retColumns = new List<ColumnOrSuperColumn>();
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
while (true)
{
object val =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.get_slice(key, columnParent, pred, consistencyLevel);
}), KEYSPACE);
List<ColumnOrSuperColumn> cols = (List<ColumnOrSuperColumn>)val;
if (pred.Slice_range.Start.Length == 0)
{
//no start specified, beginning of range
retColumns.AddRange(cols);
}
else
{
//omit the first returned item since it will be the last item
//in the previous get
for (int i = 1; i < cols.Count; i++)
{
retColumns.Add(cols[i]);
}
}
//if we didnt retrieve chunkSize rows, we finished
if (cols.Count < chunkSize)
{
break;
}
//else, we need to set the new start and continue
if (cols[cols.Count - 1].Column != null)
{
pred.Slice_range.Start = cols[cols.Count - 1].Column.Name;
}
else
{
pred.Slice_range.Start = cols[cols.Count - 1].Super_column.Name;
}
}
return retColumns;
}
public void send_get_count(byte[] key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
{
oprot_.WriteMessageBegin(new TMessage("get_count", TMessageType.Call, seqid_));
get_count_args args = new get_count_args();
args.Key = key;
args.Column_parent = column_parent;
args.Predicate = predicate;
args.Consistency_level = consistency_level;
args.Write(oprot_);
oprot_.WriteMessageEnd();
oprot_.Transport.Flush();
}
/// <summary>
/// Retrieves the index of all folders owned by this user and attempts to
/// find and repair any inconsistencies
/// </summary>
/// <param name="ownerId"></param>
/// <returns></returns>
public void Maint_RepairFolderIndex(UUID ownerId)
{
try
{
byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid);
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = USERFOLDERS_CF;
SlicePredicate pred = new SlicePredicate();
SliceRange range = new SliceRange();
range.Start = new byte[0];
range.Finish = new byte[0];
range.Reversed = false;
range.Count = int.MaxValue;
pred.Slice_range = range;
List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ,
ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
List<Guid> badIndexFolders = new List<Guid>();
foreach (ColumnOrSuperColumn col in cols)
{
Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns);
try
{
InventoryFolderBase folder = new InventoryFolderBase
{
ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)),
Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0],
Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value),
Owner = ownerId,
ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)),
Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value),
};
}
catch (KeyNotFoundException)
{
//there is a corruption, this folder can not be read
badIndexFolders.Add(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name));
}
}
List<Guid> destroyedFolders = new List<Guid>();
List<InventoryFolderBase> recoverableFolders = new List<InventoryFolderBase>();
//for each folder that has a bad index, try to read the folder.
//if we can read the folder, restore it from the data we have
//otherwise delete the index, the data is gone
foreach (Guid id in badIndexFolders)
{
try
{
InventoryFolderBase folder = this.GetFolderAttributes(new UUID(id));
//also verify the parent exists and is readable
InventoryFolderBase parentFolder = this.GetFolderAttributes(folder.ParentID);
recoverableFolders.Add(folder);
}
catch (KeyNotFoundException)
{
destroyedFolders.Add(id);
}
catch (InventoryObjectMissingException)
{
destroyedFolders.Add(id);
}
}
long timeStamp = Util.UnixTimeSinceEpochInMicroseconds();
foreach (InventoryFolderBase folder in recoverableFolders)
{
//recover anything recoverable
this.CreateFolderInternal(folder, timeStamp);
}
foreach (Guid id in destroyedFolders)
{
this.RemoveFromIndex(ownerId.Guid, id, timeStamp);
}
}
catch (Exception e)
{
_log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to recover folder index: {0}", e);
throw new InventoryStorageException(e.Message, e);
}
}
public void send_get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level)
{
oprot_.WriteMessageBegin(new TMessage("get_range_slices", TMessageType.Call, seqid_));
get_range_slices_args args = new get_range_slices_args();
args.Column_parent = column_parent;
args.Predicate = predicate;
args.Range = range;
args.Consistency_level = consistency_level;
args.Write(oprot_);
oprot_.WriteMessageEnd();
oprot_.Transport.Flush();
}
/// <summary>
/// Pulls down all folders and collects the list of subfolder
/// UUIDs for each. Then attempts to read each of the sub folders
/// listed in the subfolder index, and removes any indexed subfolders
/// that are no longer readable due to a partial deletion
/// </summary>
/// <param name="ownerId"></param>
/// <returns></returns>
public void Maint_RepairSubfolderIndexes(UUID ownerId)
{
try
{
byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid);
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = USERFOLDERS_CF;
SlicePredicate pred = new SlicePredicate();
SliceRange range = new SliceRange();
range.Start = new byte[0];
range.Finish = new byte[0];
range.Reversed = false;
range.Count = int.MaxValue;
pred.Slice_range = range;
List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ,
ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
List<InventoryFolderBase> goodIndexFolders = new List<InventoryFolderBase>();
foreach (ColumnOrSuperColumn col in cols)
{
Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns);
try
{
InventoryFolderBase folder = new InventoryFolderBase
{
ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)),
Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0],
Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value),
Owner = ownerId,
ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)),
Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value),
};
goodIndexFolders.Add(folder);
}
catch (KeyNotFoundException)
{
//there is a corruption, this folder can not be read. Ignore since there is
//another maint that can fix this that should be run first.
}
}
List<KeyValuePair<Guid, Guid>> invalidParentChild = new List<KeyValuePair<Guid, Guid>>();
//for each folder in the index, retrieve it and check for unreadable subfolders
foreach (InventoryFolderBase indexFolder in goodIndexFolders)
{
try
{
InventoryFolderBase folder = this.GetFolder(indexFolder.ID);
foreach (var subfolder in folder.SubFolders)
{
try
{
InventoryFolderBase subFolderDetails = this.GetFolderAttributes(subfolder.ID);
}
catch (InventoryObjectMissingException)
{
invalidParentChild.Add(new KeyValuePair<Guid, Guid>(folder.ID.Guid, subfolder.ID.Guid));
}
catch (InventoryStorageException e)
{
if (e.InnerException != null && e.InnerException is KeyNotFoundException)
{
invalidParentChild.Add(new KeyValuePair<Guid, Guid>(folder.ID.Guid, subfolder.ID.Guid));
}
}
}
}
catch (Exception e)
{
//we can't even get the folder, so no subfolders to fix
_log.ErrorFormat("[InWorldz.Data.Inventory.Cassandra] Indexed folder {0} could not be retrieved to look for children: {1}", indexFolder.ID, e);
}
}
_log.InfoFormat("[InWorldz.Data.Inventory.Cassandra][MAINT] Found {0} subfolder indexes to repair", invalidParentChild.Count);
long timeStamp = Util.UnixTimeSinceEpochInMicroseconds();
//we couldn't read the folder, pull it from the subfolder list of its parent
foreach (KeyValuePair<Guid, Guid> parentChildKvp in invalidParentChild)
{
byte[] childIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(parentChildKvp.Value);
Dictionary<byte[], Dictionary<string, List<Mutation>>> muts = new Dictionary<byte[], Dictionary<string, List<Mutation>>>();
GenerateSubfolderIndexDeletion(parentChildKvp.Key, timeStamp, childIdBytes, muts);
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
client.batch_mutate(muts, DEFAULT_CONSISTENCY_LEVEL);
return null;
}), KEYSPACE);
}
}
catch (Exception e)
{
_log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to repair subfolder index: {0}", e);
throw new InventoryStorageException(e.Message, e);
}
}
public void send_multiget_slice(List<byte[]> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level)
{
oprot_.WriteMessageBegin(new TMessage("multiget_slice", TMessageType.Call, seqid_));
multiget_slice_args args = new multiget_slice_args();
args.Keys = keys;
args.Column_parent = column_parent;
args.Predicate = predicate;
args.Consistency_level = consistency_level;
args.Write(oprot_);
oprot_.WriteMessageEnd();
oprot_.Transport.Flush();
}
/// <summary>
/// Retrieves the index of all folders owned by this user
/// </summary>
/// <param name="ownerId"></param>
/// <returns></returns>
private Dictionary<Guid, InventoryFolderBase> GetFolderIndex(UUID ownerId)
{
try
{
byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid);
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = USERFOLDERS_CF;
SlicePredicate pred = new SlicePredicate();
SliceRange range = new SliceRange();
range.Start = new byte[0];
range.Finish = new byte[0];
range.Reversed = false;
range.Count = int.MaxValue;
pred.Slice_range = range;
List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ,
ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
Dictionary<Guid, InventoryFolderBase> retIndex = new Dictionary<Guid, InventoryFolderBase>();
foreach (ColumnOrSuperColumn col in cols)
{
Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns);
try
{
InventoryFolderBase folder = new InventoryFolderBase
{
ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)),
Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0],
Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value),
Owner = ownerId,
ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)),
Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value),
};
retIndex.Add(folder.ID.Guid, folder);
}
catch (KeyNotFoundException)
{
//there is a corruption, this folder can not be read
_log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra]: Unable to read all columns from folder index item: {0}",
new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)).ToString());
}
}
return retIndex;
}
catch (Exception e)
{
_log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder index: {0}", e);
throw new InventoryStorageException(e.Message, e);
}
}
public void Read(TProtocol iprot)
{
TField field;
iprot.ReadStructBegin();
while (true)
{
field = iprot.ReadFieldBegin();
if (field.Type == TType.Stop) {
break;
}
switch (field.ID)
{
case 1:
if (field.Type == TType.String) {
Key = iprot.ReadBinary();
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 2:
if (field.Type == TType.Struct) {
Column_parent = new ColumnParent();
Column_parent.Read(iprot);
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 3:
if (field.Type == TType.Struct) {
Column = new Column();
Column.Read(iprot);
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
case 4:
if (field.Type == TType.I32) {
Consistency_level = (ConsistencyLevel)iprot.ReadI32();
} else {
TProtocolUtil.Skip(iprot, field.Type);
}
break;
default:
TProtocolUtil.Skip(iprot, field.Type);
break;
}
iprot.ReadFieldEnd();
}
iprot.ReadStructEnd();
}
public InventoryFolderBase GetFolder(UUID folderId)
{
if (folderId == UUID.Zero) throw new InventorySecurityException("Not returning folder with ID UUID.Zero");
try
{
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = FOLDERS_CF;
SlicePredicate pred = new SlicePredicate();
SliceRange range = new SliceRange();
range.Start = new byte[0];
range.Finish = new byte[0];
range.Reversed = false;
range.Count = int.MaxValue;
pred.Slice_range = range;
byte[] folderIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(folderId.Guid);
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_CONTENTS_CHUNK_SZ,
folderIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
if (cols.Count == 0)
{
throw new InventoryObjectMissingException(String.Format("Folder with ID {0} could not be found", folderId));
}
ParsedFolderData parsedData = new ParsedFolderData(cols);
InventoryFolderBase folder = DecodeFolderBase(folderId.Guid, parsedData.Properties.Super_column.Columns);
if (parsedData.SubFolders != null)
{
foreach (Column col in parsedData.SubFolders.Super_column.Columns)
{
SubFolderData data = SubFolderData.FromByteArray(col.Value);
InventorySubFolderBase folderStub = new InventorySubFolderBase
{
ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Name)),
Name = data.Name,
Owner = folder.Owner,
Type = data.Type
};
folder.SubFolders.Add(folderStub);
}
}
foreach (ColumnOrSuperColumn itemCol in parsedData.Items)
{
Guid itemId = ByteEncoderHelper.GuidEncoder.FromByteArray(itemCol.Super_column.Name);
InventoryItemBase item = this.DecodeInventoryItem(itemCol.Super_column.Columns, itemId, folder.ID.Guid);
folder.Items.Add(item);
}
//grab the folder version
try
{
ColumnPath path = new ColumnPath
{
Column = ByteEncoderHelper.UTF8Encoder.ToByteArray("count"),
Column_family = FOLDERVERSIONS_CF
};
object verVal =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.get(folderIdArray, path, DEFAULT_CONSISTENCY_LEVEL);
}), KEYSPACE);
ColumnOrSuperColumn verColumn = (ColumnOrSuperColumn)verVal;
folder.Version = (ushort)(verColumn.Counter_column.Value % (long)ushort.MaxValue);
}
catch (Exception e)
{
_log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Could not retrieve the version for folder {0} substituting 1: {1}", folderId, e);
//version column missing. this is either a partially deleted folder
//or the version mutation never happened. Return 1
folder.Version = 1;
}
return folder;
}
catch (InventoryObjectMissingException e)
{
_log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e);
throw;
}
catch (Exception e)
{
_log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e);
throw new InventoryStorageException(e.Message, e);
}
}
public void add(byte[] key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level)
{
send_add(key, column_parent, column, consistency_level);
recv_add();
}
public InventoryFolderBase GetFolderAttributes(UUID folderId)
{
if (folderId == UUID.Zero) throw new InventorySecurityException("Not returning folder with ID UUID.Zero");
try
{
ColumnParent columnParent = new ColumnParent();
columnParent.Column_family = FOLDERS_CF;
columnParent.Super_column = ByteEncoderHelper.UTF8Encoder.ToByteArray("properties");
SlicePredicate pred = new SlicePredicate();
SliceRange range = new SliceRange();
range.Start = new byte[0];
range.Finish = new byte[0];
range.Reversed = false;
range.Count = int.MaxValue;
pred.Slice_range = range;
byte[] folderIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(folderId.Guid);
ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName);
object val =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.get_slice(folderIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL);
}), KEYSPACE);
List<ColumnOrSuperColumn> cols = (List<ColumnOrSuperColumn>)val;
if (cols.Count == 0)
{
throw new InventoryObjectMissingException(String.Format("Folder with ID {0} could not be found", folderId));
}
InventoryFolderBase folder = DecodeFolderBase(folderId.Guid, cols);
//grab the folder version
ColumnPath path = new ColumnPath
{
Column = ByteEncoderHelper.UTF8Encoder.ToByteArray("count"),
Column_family = FOLDERVERSIONS_CF
};
object verVal =
cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client)
{
return client.get(folderIdArray, path, DEFAULT_CONSISTENCY_LEVEL);
}), KEYSPACE);
ColumnOrSuperColumn verColumn = (ColumnOrSuperColumn)verVal;
folder.Version = (ushort)(verColumn.Counter_column.Value % (long)ushort.MaxValue);
return folder;
}
catch (InventoryObjectMissingException e)
{
// _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e);
throw; // produces a duplicate error farther up with more context
}
catch (Exception e)
{
// _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e);
throw new InventoryStorageException(e.Message, e); // produces another error farther up with more context
}
}
public List<KeySlice> get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level)
{
send_get_indexed_slices(column_parent, index_clause, column_predicate, consistency_level);
return recv_get_indexed_slices();
}
public List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level)
{
send_get_range_slices(column_parent, predicate, range, consistency_level);
return recv_get_range_slices();
}