private static void Main() { TTransport framedTransport = new TFramedTransport(new TSocket("localhost", 9160)); TTransport socketTransport = new TSocket("localhost", 9160); TProtocol framedProtocol = new TBinaryProtocol(framedTransport); TProtocol socketProtocol = new TBinaryProtocol(socketTransport); var client = new Cassandra.Client(framedProtocol, framedProtocol); // all framed //var client = new Cassandra.Client(socketProtocol, socketProtocol); // all socket //var client = new Cassandra.Client(framedProtocol, socketProtocol); // in: framed out: socket //var client = new Cassandra.Client(socketProtocol, framedProtocol); // in: socket out: framed framedTransport.Open(); socketTransport.Open(); Console.WriteLine("Start"); client.set_keyspace("Keyspace1"); Console.WriteLine("Count Key"); var key = Encoding.ASCII.GetBytes("MyKey"); var columns = new List<byte[]>(new[] { Encoding.ASCII.GetBytes("MyColumn") }); var column_parent = new ColumnParent { Column_family = "Standard1" }; var predicate = new SlicePredicate { Column_names = columns }; client.get_count(key, column_parent, predicate, ConsistencyLevel.ALL); Console.WriteLine("Done"); Console.Read(); }
// private string columnFamily = "users"; // private string keyspace = "test"; // private string key = "testKey"; // // Insert statement //// private byte[] key = ByteEncoderHelper.LongEncoder.ToByteArray(i); // private byte[] key = ByteEncoderHelper.LongEncoder.ToByteArray(2); // private ColumnParent columnParent; // private Column column; // private ColumnPath columnPath; // private ICluster cluster; // // // //cluster.Execute(new ExecutionBlock(delegate(CassandraClient client) { // client.insert(key, columnParent, column, ConsistencyLevel.ONE); // return null; //}), keyspace); // // Get statement // byte[] key = ByteEncoderHelper.LongEncoder.ToByteArray(2); // // ICluster cluster = AquilesHelper.RetrieveCluster("Cassandra1"); // object rtnValue = cluster.Execute(new ExecutionBlock(delegate(CassandraClient client) // { // return client.get(key, columnPath, ConsistencyLevel.ONE); // }), keyspace); public void Run() { string columnFamily = "TestColumnFamily"; string keyspace = "TestKeyspace"; string key = "testKey"; string columnName = "testColumn"; string columnValue = "testValue"; // Insert statement byte[] key2 = ByteEncoderHelper.LongEncoder.ToByteArray(1); ColumnParent columnParent = new ColumnParent(); Column column = new Column() { Name = ByteEncoderHelper.UTF8Encoder.ToByteArray(columnName), Timestamp = UnixHelper.UnixTimestamp, Value = ByteEncoderHelper.UTF8Encoder.ToByteArray(columnValue), }; columnParent.Column_family = columnFamily; ICluster cluster = AquilesHelper.RetrieveCluster("Cassandra1"); cluster.Execute(new ExecutionBlock(delegate(CassandraClient client) { client.insert(key, columnParent, column, ConsistencyLevel.ONE); return null; }), keyspace); }
/// <summary> /// Biuld Thrift ColumnParent structure using ColumnFamily and SuperColumn information /// </summary> /// <param name="superColumn">name for the supercolumn (null in case there is not one)</param> /// <returns>Thrift ColumnParent</returns> protected ColumnParent BuildColumnParent(string superColumn) { ColumnParent columnParent = new ColumnParent(); bool isSuperColumnMissing = String.IsNullOrEmpty(superColumn); columnParent.Column_family = this.ColumnFamily; if (!isSuperColumnMissing) { columnParent.Super_column = ByteEncoderHelper.ToByteArray(superColumn); } return columnParent; }
public int get_count(byte[] key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) { send_get_count(key, column_parent, predicate, consistency_level); return recv_get_count(); }
public void Read(TProtocol iprot) { TField field; iprot.ReadStructBegin(); while (true) { field = iprot.ReadFieldBegin(); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.List) { { Keys = new List<byte[]>(); TList _list87 = iprot.ReadListBegin(); for( int _i88 = 0; _i88 < _list87.Count; ++_i88) { byte[] _elem89 = null; _elem89 = iprot.ReadBinary(); Keys.Add(_elem89); } iprot.ReadListEnd(); } } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 2: if (field.Type == TType.Struct) { Column_parent = new ColumnParent(); Column_parent.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 3: if (field.Type == TType.Struct) { Predicate = new SlicePredicate(); Predicate.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 4: if (field.Type == TType.I32) { Consistency_level = (ConsistencyLevel)iprot.ReadI32(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; default: TProtocolUtil.Skip(iprot, field.Type); break; } iprot.ReadFieldEnd(); } iprot.ReadStructEnd(); }
public void Read(TProtocol iprot) { TField field; iprot.ReadStructBegin(); while (true) { field = iprot.ReadFieldBegin(); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.Struct) { Column_parent = new ColumnParent(); Column_parent.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 2: if (field.Type == TType.Struct) { Index_clause = new IndexClause(); Index_clause.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 3: if (field.Type == TType.Struct) { Column_predicate = new SlicePredicate(); Column_predicate.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 4: if (field.Type == TType.I32) { Consistency_level = (ConsistencyLevel)iprot.ReadI32(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; default: TProtocolUtil.Skip(iprot, field.Type); break; } iprot.ReadFieldEnd(); } iprot.ReadStructEnd(); }
public void send_insert(byte[] key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level) { oprot_.WriteMessageBegin(new TMessage("insert", TMessageType.Call, seqid_)); insert_args args = new insert_args(); args.Key = key; args.Column_parent = column_parent; args.Column = column; args.Consistency_level = consistency_level; args.Write(oprot_); oprot_.WriteMessageEnd(); oprot_.Transport.Flush(); }
public void send_get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) { oprot_.WriteMessageBegin(new TMessage("get_indexed_slices", TMessageType.Call, seqid_)); get_indexed_slices_args args = new get_indexed_slices_args(); args.Column_parent = column_parent; args.Index_clause = index_clause; args.Column_predicate = column_predicate; args.Consistency_level = consistency_level; args.Write(oprot_); oprot_.WriteMessageEnd(); oprot_.Transport.Flush(); }
public Dictionary<byte[], List<ColumnOrSuperColumn>> multiget_slice(List<byte[]> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) { send_multiget_slice(keys, column_parent, predicate, consistency_level); return recv_multiget_slice(); }
public void insert(byte[] key, ColumnParent column_parent, Column column, ConsistencyLevel consistency_level) { send_insert(key, column_parent, column, consistency_level); recv_insert(); }
/// <summary> /// Returns a dictionary of parents with a list of items Dictionary[FolderID, List[Items]] /// </summary> /// <param name="itemIds"></param> /// <returns></returns> public Dictionary<UUID, List<UUID>> FindItemParentFolderIds(IEnumerable<UUID> itemIds) { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = ITEMPARENTS_CF; SlicePredicate pred = new SlicePredicate(); pred.Column_names = new List<byte[]>(); pred.Column_names.Add(ByteEncoderHelper.UTF8Encoder.ToByteArray("parent")); List<byte[]> allItemIdBytes = new List<byte[]>(); foreach (UUID id in itemIds) { allItemIdBytes.Add(ByteEncoderHelper.GuidEncoder.ToByteArray(id.Guid)); } ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object val = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.multiget_slice(allItemIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); Dictionary<byte[], List<ColumnOrSuperColumn>> itemParentCols = (Dictionary<byte[], List<ColumnOrSuperColumn>>)val; Dictionary<UUID, List<UUID>> retParents = new Dictionary<UUID, List<UUID>>(); foreach (KeyValuePair<byte[], List<ColumnOrSuperColumn>> kvp in itemParentCols) { if (kvp.Value.Count == 1) { Column col = kvp.Value[0].Column; UUID parentId = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Value)); if (!retParents.ContainsKey(parentId)) { retParents.Add(parentId, new List<UUID>()); } retParents[parentId].Add(new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(kvp.Key))); } } return retParents; }
public Guid FindItemParentFolderId(UUID itemId) { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = ITEMPARENTS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; byte[] itemIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(itemId.Guid); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object val = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get_slice(itemIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); List<ColumnOrSuperColumn> indexCols = (List<ColumnOrSuperColumn>)val; //no index means the item doesnt exist if (indexCols.Count == 0) { return Guid.Empty; } var indexedColsByName = this.IndexColumnsByUTF8Name(indexCols); return ByteEncoderHelper.GuidEncoder.FromByteArray(indexedColsByName["parent"].Value); }
/// <summary> /// Retrieves all the rows for the given list of keys in chunkSize chunks /// </summary> /// <param name="chunkSize"></param> /// <param name="allKeys"></param> /// <param name="colParent"></param> /// <param name="pred"></param> /// <param name="consistencyLevel"></param> /// <returns></returns> private Dictionary<byte[], List<ColumnOrSuperColumn>> RetrieveRowsInChunks(int chunkSize, List<byte[]> allKeys, ColumnParent colParent, SlicePredicate pred, ConsistencyLevel consistencyLevel) { ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); if (allKeys.Count <= chunkSize) { object val = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.multiget_slice(allKeys, colParent, pred, consistencyLevel); }), KEYSPACE); return (Dictionary<byte[], List<ColumnOrSuperColumn>>)val; } else { Dictionary<byte[], List<ColumnOrSuperColumn>> ret = new Dictionary<byte[], List<ColumnOrSuperColumn>>(); for (int i = 0; i < allKeys.Count; i += chunkSize) { int remaining = allKeys.Count - i; List<byte[]> keys = allKeys.GetRange(i, remaining >= chunkSize ? chunkSize : remaining); object val = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.multiget_slice(keys, colParent, pred, consistencyLevel); }), KEYSPACE); Dictionary<byte[], List<ColumnOrSuperColumn>> chunk = (Dictionary<byte[], List<ColumnOrSuperColumn>>)val; foreach(KeyValuePair<byte[], List<ColumnOrSuperColumn>> kvp in chunk) { ret.Add(kvp.Key, kvp.Value); } } return ret; } }
public InventoryItemBase GetItem(UUID itemId, UUID parentFolderHint) { //Retrieving an item requires a lookup of the parent folder followed by //a retrieval of the item. This was a consious decision made since the //inventory item data currently takes up the most space and a //duplication of this data to prevent the index lookup //would be expensive in terms of space required try { Guid parentId; if (parentFolderHint != UUID.Zero) { parentId = parentFolderHint.Guid; } else { parentId = FindItemParentFolderId(itemId); } if (parentId == Guid.Empty) { throw new InventoryObjectMissingException(String.Format("Item with ID {0} could not be found", itemId), "Item was not found in the index"); } //try to retrieve the item. note that even though we have an index there is a chance we will //not have the item data due to a race condition between index mutation and item mutation byte[] itemIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(itemId.Guid); byte[] folderIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(parentId); ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = FOLDERS_CF; columnParent.Super_column = itemIdBytes; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object itemDataObj = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get_slice(folderIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); List<ColumnOrSuperColumn> itemCols = (List<ColumnOrSuperColumn>)itemDataObj; if (itemCols.Count == 0) { throw new InventoryObjectMissingException(String.Format("Item with ID {0} could not be found", itemId), "Item was not found in its folder"); } InventoryItemBase item = this.DecodeInventoryItem(itemCols, itemId.Guid, parentId); return item; } catch (InventoryStorageException) { throw; } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve item {0}: {1}", itemId, e); throw new InventoryStorageException(e.Message, e); } }
public void Read(TProtocol iprot) { iprot.IncrementRecursionDepth(); try { TField field; iprot.ReadStructBegin(); while (true) { field = iprot.ReadFieldBegin(); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.String) { Key = iprot.ReadBinary(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 2: if (field.Type == TType.Struct) { Column_parent = new ColumnParent(); Column_parent.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 3: if (field.Type == TType.List) { { Column_slices = new List <ColumnSlice>(); TList _list108 = iprot.ReadListBegin(); for (int _i109 = 0; _i109 < _list108.Count; ++_i109) { ColumnSlice _elem110; _elem110 = new ColumnSlice(); _elem110.Read(iprot); Column_slices.Add(_elem110); } iprot.ReadListEnd(); } } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 4: if (field.Type == TType.Bool) { Reversed = iprot.ReadBool(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 5: if (field.Type == TType.I32) { Count = iprot.ReadI32(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 6: if (field.Type == TType.I32) { Consistency_level = (ConsistencyLevel)iprot.ReadI32(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; default: TProtocolUtil.Skip(iprot, field.Type); break; } iprot.ReadFieldEnd(); } iprot.ReadStructEnd(); } finally { iprot.DecrementRecursionDepth(); } }
public List<ColumnOrSuperColumn> get_slice(byte[] key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) { send_get_slice(key, column_parent, predicate, consistency_level); return recv_get_slice(); }
private List<UUID> GetActiveGestureItemIds(UUID userId) { try { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = USERACTIVEGESTURES_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object retobj = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { byte[] userIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(userId.Guid); return client.get_slice(userIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); List<ColumnOrSuperColumn> cols = (List<ColumnOrSuperColumn>)retobj; List<UUID> ret = new List<UUID>(); foreach (ColumnOrSuperColumn col in cols) { ret.Add(new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Column.Name))); } return ret; } catch (Exception e) { throw new InventoryStorageException(e.Message, e); } }
public Dictionary<byte[], int> multiget_count(List<byte[]> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) { send_multiget_count(keys, column_parent, predicate, consistency_level); return recv_multiget_count(); }
private List<ColumnOrSuperColumn> RetrieveAllColumnsInChunks(int chunkSize, byte[] key, ColumnParent columnParent, SlicePredicate pred, ConsistencyLevel consistencyLevel) { pred.Slice_range.Count = chunkSize; List<ColumnOrSuperColumn> retColumns = new List<ColumnOrSuperColumn>(); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); while (true) { object val = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get_slice(key, columnParent, pred, consistencyLevel); }), KEYSPACE); List<ColumnOrSuperColumn> cols = (List<ColumnOrSuperColumn>)val; if (pred.Slice_range.Start.Length == 0) { //no start specified, beginning of range retColumns.AddRange(cols); } else { //omit the first returned item since it will be the last item //in the previous get for (int i = 1; i < cols.Count; i++) { retColumns.Add(cols[i]); } } //if we didnt retrieve chunkSize rows, we finished if (cols.Count < chunkSize) { break; } //else, we need to set the new start and continue if (cols[cols.Count - 1].Column != null) { pred.Slice_range.Start = cols[cols.Count - 1].Column.Name; } else { pred.Slice_range.Start = cols[cols.Count - 1].Super_column.Name; } } return retColumns; }
public void send_get_count(byte[] key, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) { oprot_.WriteMessageBegin(new TMessage("get_count", TMessageType.Call, seqid_)); get_count_args args = new get_count_args(); args.Key = key; args.Column_parent = column_parent; args.Predicate = predicate; args.Consistency_level = consistency_level; args.Write(oprot_); oprot_.WriteMessageEnd(); oprot_.Transport.Flush(); }
/// <summary> /// Retrieves the index of all folders owned by this user and attempts to /// find and repair any inconsistencies /// </summary> /// <param name="ownerId"></param> /// <returns></returns> public void Maint_RepairFolderIndex(UUID ownerId) { try { byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid); ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = USERFOLDERS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ, ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); List<Guid> badIndexFolders = new List<Guid>(); foreach (ColumnOrSuperColumn col in cols) { Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns); try { InventoryFolderBase folder = new InventoryFolderBase { ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)), Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0], Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value), Owner = ownerId, ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)), Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value), }; } catch (KeyNotFoundException) { //there is a corruption, this folder can not be read badIndexFolders.Add(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)); } } List<Guid> destroyedFolders = new List<Guid>(); List<InventoryFolderBase> recoverableFolders = new List<InventoryFolderBase>(); //for each folder that has a bad index, try to read the folder. //if we can read the folder, restore it from the data we have //otherwise delete the index, the data is gone foreach (Guid id in badIndexFolders) { try { InventoryFolderBase folder = this.GetFolderAttributes(new UUID(id)); //also verify the parent exists and is readable InventoryFolderBase parentFolder = this.GetFolderAttributes(folder.ParentID); recoverableFolders.Add(folder); } catch (KeyNotFoundException) { destroyedFolders.Add(id); } catch (InventoryObjectMissingException) { destroyedFolders.Add(id); } } long timeStamp = Util.UnixTimeSinceEpochInMicroseconds(); foreach (InventoryFolderBase folder in recoverableFolders) { //recover anything recoverable this.CreateFolderInternal(folder, timeStamp); } foreach (Guid id in destroyedFolders) { this.RemoveFromIndex(ownerId.Guid, id, timeStamp); } } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to recover folder index: {0}", e); throw new InventoryStorageException(e.Message, e); } }
public void send_get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) { oprot_.WriteMessageBegin(new TMessage("get_range_slices", TMessageType.Call, seqid_)); get_range_slices_args args = new get_range_slices_args(); args.Column_parent = column_parent; args.Predicate = predicate; args.Range = range; args.Consistency_level = consistency_level; args.Write(oprot_); oprot_.WriteMessageEnd(); oprot_.Transport.Flush(); }
/// <summary> /// Pulls down all folders and collects the list of subfolder /// UUIDs for each. Then attempts to read each of the sub folders /// listed in the subfolder index, and removes any indexed subfolders /// that are no longer readable due to a partial deletion /// </summary> /// <param name="ownerId"></param> /// <returns></returns> public void Maint_RepairSubfolderIndexes(UUID ownerId) { try { byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid); ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = USERFOLDERS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ, ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); List<InventoryFolderBase> goodIndexFolders = new List<InventoryFolderBase>(); foreach (ColumnOrSuperColumn col in cols) { Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns); try { InventoryFolderBase folder = new InventoryFolderBase { ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)), Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0], Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value), Owner = ownerId, ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)), Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value), }; goodIndexFolders.Add(folder); } catch (KeyNotFoundException) { //there is a corruption, this folder can not be read. Ignore since there is //another maint that can fix this that should be run first. } } List<KeyValuePair<Guid, Guid>> invalidParentChild = new List<KeyValuePair<Guid, Guid>>(); //for each folder in the index, retrieve it and check for unreadable subfolders foreach (InventoryFolderBase indexFolder in goodIndexFolders) { try { InventoryFolderBase folder = this.GetFolder(indexFolder.ID); foreach (var subfolder in folder.SubFolders) { try { InventoryFolderBase subFolderDetails = this.GetFolderAttributes(subfolder.ID); } catch (InventoryObjectMissingException) { invalidParentChild.Add(new KeyValuePair<Guid, Guid>(folder.ID.Guid, subfolder.ID.Guid)); } catch (InventoryStorageException e) { if (e.InnerException != null && e.InnerException is KeyNotFoundException) { invalidParentChild.Add(new KeyValuePair<Guid, Guid>(folder.ID.Guid, subfolder.ID.Guid)); } } } } catch (Exception e) { //we can't even get the folder, so no subfolders to fix _log.ErrorFormat("[InWorldz.Data.Inventory.Cassandra] Indexed folder {0} could not be retrieved to look for children: {1}", indexFolder.ID, e); } } _log.InfoFormat("[InWorldz.Data.Inventory.Cassandra][MAINT] Found {0} subfolder indexes to repair", invalidParentChild.Count); long timeStamp = Util.UnixTimeSinceEpochInMicroseconds(); //we couldn't read the folder, pull it from the subfolder list of its parent foreach (KeyValuePair<Guid, Guid> parentChildKvp in invalidParentChild) { byte[] childIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(parentChildKvp.Value); Dictionary<byte[], Dictionary<string, List<Mutation>>> muts = new Dictionary<byte[], Dictionary<string, List<Mutation>>>(); GenerateSubfolderIndexDeletion(parentChildKvp.Key, timeStamp, childIdBytes, muts); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { client.batch_mutate(muts, DEFAULT_CONSISTENCY_LEVEL); return null; }), KEYSPACE); } } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to repair subfolder index: {0}", e); throw new InventoryStorageException(e.Message, e); } }
public void send_multiget_slice(List<byte[]> keys, ColumnParent column_parent, SlicePredicate predicate, ConsistencyLevel consistency_level) { oprot_.WriteMessageBegin(new TMessage("multiget_slice", TMessageType.Call, seqid_)); multiget_slice_args args = new multiget_slice_args(); args.Keys = keys; args.Column_parent = column_parent; args.Predicate = predicate; args.Consistency_level = consistency_level; args.Write(oprot_); oprot_.WriteMessageEnd(); oprot_.Transport.Flush(); }
/// <summary> /// Retrieves the index of all folders owned by this user /// </summary> /// <param name="ownerId"></param> /// <returns></returns> private Dictionary<Guid, InventoryFolderBase> GetFolderIndex(UUID ownerId) { try { byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid); ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = USERFOLDERS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ, ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); Dictionary<Guid, InventoryFolderBase> retIndex = new Dictionary<Guid, InventoryFolderBase>(); foreach (ColumnOrSuperColumn col in cols) { Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns); try { InventoryFolderBase folder = new InventoryFolderBase { ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)), Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0], Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value), Owner = ownerId, ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)), Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value), }; retIndex.Add(folder.ID.Guid, folder); } catch (KeyNotFoundException) { //there is a corruption, this folder can not be read _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra]: Unable to read all columns from folder index item: {0}", new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)).ToString()); } } return retIndex; } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder index: {0}", e); throw new InventoryStorageException(e.Message, e); } }
public void Read(TProtocol iprot) { TField field; iprot.ReadStructBegin(); while (true) { field = iprot.ReadFieldBegin(); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.String) { Key = iprot.ReadBinary(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 2: if (field.Type == TType.Struct) { Column_parent = new ColumnParent(); Column_parent.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 3: if (field.Type == TType.Struct) { Column = new Column(); Column.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 4: if (field.Type == TType.I32) { Consistency_level = (ConsistencyLevel)iprot.ReadI32(); } else { TProtocolUtil.Skip(iprot, field.Type); } break; default: TProtocolUtil.Skip(iprot, field.Type); break; } iprot.ReadFieldEnd(); } iprot.ReadStructEnd(); }
public InventoryFolderBase GetFolder(UUID folderId) { if (folderId == UUID.Zero) throw new InventorySecurityException("Not returning folder with ID UUID.Zero"); try { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = FOLDERS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; byte[] folderIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(folderId.Guid); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_CONTENTS_CHUNK_SZ, folderIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); if (cols.Count == 0) { throw new InventoryObjectMissingException(String.Format("Folder with ID {0} could not be found", folderId)); } ParsedFolderData parsedData = new ParsedFolderData(cols); InventoryFolderBase folder = DecodeFolderBase(folderId.Guid, parsedData.Properties.Super_column.Columns); if (parsedData.SubFolders != null) { foreach (Column col in parsedData.SubFolders.Super_column.Columns) { SubFolderData data = SubFolderData.FromByteArray(col.Value); InventorySubFolderBase folderStub = new InventorySubFolderBase { ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Name)), Name = data.Name, Owner = folder.Owner, Type = data.Type }; folder.SubFolders.Add(folderStub); } } foreach (ColumnOrSuperColumn itemCol in parsedData.Items) { Guid itemId = ByteEncoderHelper.GuidEncoder.FromByteArray(itemCol.Super_column.Name); InventoryItemBase item = this.DecodeInventoryItem(itemCol.Super_column.Columns, itemId, folder.ID.Guid); folder.Items.Add(item); } //grab the folder version try { ColumnPath path = new ColumnPath { Column = ByteEncoderHelper.UTF8Encoder.ToByteArray("count"), Column_family = FOLDERVERSIONS_CF }; object verVal = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get(folderIdArray, path, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); ColumnOrSuperColumn verColumn = (ColumnOrSuperColumn)verVal; folder.Version = (ushort)(verColumn.Counter_column.Value % (long)ushort.MaxValue); } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Could not retrieve the version for folder {0} substituting 1: {1}", folderId, e); //version column missing. this is either a partially deleted folder //or the version mutation never happened. Return 1 folder.Version = 1; } return folder; } catch (InventoryObjectMissingException e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e); throw; } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e); throw new InventoryStorageException(e.Message, e); } }
public void add(byte[] key, ColumnParent column_parent, CounterColumn column, ConsistencyLevel consistency_level) { send_add(key, column_parent, column, consistency_level); recv_add(); }
public InventoryFolderBase GetFolderAttributes(UUID folderId) { if (folderId == UUID.Zero) throw new InventorySecurityException("Not returning folder with ID UUID.Zero"); try { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = FOLDERS_CF; columnParent.Super_column = ByteEncoderHelper.UTF8Encoder.ToByteArray("properties"); SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; byte[] folderIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(folderId.Guid); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object val = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get_slice(folderIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); List<ColumnOrSuperColumn> cols = (List<ColumnOrSuperColumn>)val; if (cols.Count == 0) { throw new InventoryObjectMissingException(String.Format("Folder with ID {0} could not be found", folderId)); } InventoryFolderBase folder = DecodeFolderBase(folderId.Guid, cols); //grab the folder version ColumnPath path = new ColumnPath { Column = ByteEncoderHelper.UTF8Encoder.ToByteArray("count"), Column_family = FOLDERVERSIONS_CF }; object verVal = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get(folderIdArray, path, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); ColumnOrSuperColumn verColumn = (ColumnOrSuperColumn)verVal; folder.Version = (ushort)(verColumn.Counter_column.Value % (long)ushort.MaxValue); return folder; } catch (InventoryObjectMissingException e) { // _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e); throw; // produces a duplicate error farther up with more context } catch (Exception e) { // _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e); throw new InventoryStorageException(e.Message, e); // produces another error farther up with more context } }
public List<KeySlice> get_indexed_slices(ColumnParent column_parent, IndexClause index_clause, SlicePredicate column_predicate, ConsistencyLevel consistency_level) { send_get_indexed_slices(column_parent, index_clause, column_predicate, consistency_level); return recv_get_indexed_slices(); }
public List<KeySlice> get_range_slices(ColumnParent column_parent, SlicePredicate predicate, KeyRange range, ConsistencyLevel consistency_level) { send_get_range_slices(column_parent, predicate, range, consistency_level); return recv_get_range_slices(); }