public SlicePredicate(SliceRange sliceRange) { if (sliceRange != null && sliceRange.Count > 0) { Slice_range = sliceRange; } }
public SlicePredicate(List <byte[]> column_names, SliceRange sliceRange) : this(sliceRange) { if (column_names != null && column_names.Count > 0) { Column_names = column_names; } }
public void Read(TProtocol iprot) { TField field; iprot.ReadStructBegin(); while (true) { field = iprot.ReadFieldBegin(); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.List) { { this.column_names = new List <byte[]>(); TList _list4 = iprot.ReadListBegin(); for (int _i5 = 0; _i5 < _list4.Count; ++_i5) { byte[] _elem6 = null; _elem6 = iprot.ReadBinary(); this.column_names.Add(_elem6); } iprot.ReadListEnd(); } this.__isset.column_names = true; } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 2: if (field.Type == TType.Struct) { this.slice_range = new SliceRange(); this.slice_range.Read(iprot); this.__isset.slice_range = true; } else { TProtocolUtil.Skip(iprot, field.Type); } break; default: TProtocolUtil.Skip(iprot, field.Type); break; } iprot.ReadFieldEnd(); } iprot.ReadStructEnd(); }
public void Read(TProtocol iprot) { TField field; iprot.ReadStructBegin(); while (true) { field = iprot.ReadFieldBegin(); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.List) { { Column_names = new List <byte[]>(); TList _list8 = iprot.ReadListBegin(); for (int _i9 = 0; _i9 < _list8.Count; ++_i9) { byte[] _elem10 = null; _elem10 = iprot.ReadBinary(); Column_names.Add(_elem10); } iprot.ReadListEnd(); } } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 2: if (field.Type == TType.Struct) { Slice_range = new SliceRange(); Slice_range.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; default: TProtocolUtil.Skip(iprot, field.Type); break; } iprot.ReadFieldEnd(); } iprot.ReadStructEnd(); }
public InventoryFolderBase GetFolderAttributes(UUID folderId) { if (folderId == UUID.Zero) throw new InventorySecurityException("Not returning folder with ID UUID.Zero"); try { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = FOLDERS_CF; columnParent.Super_column = ByteEncoderHelper.UTF8Encoder.ToByteArray("properties"); SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; byte[] folderIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(folderId.Guid); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object val = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get_slice(folderIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); List<ColumnOrSuperColumn> cols = (List<ColumnOrSuperColumn>)val; if (cols.Count == 0) { throw new InventoryObjectMissingException(String.Format("Folder with ID {0} could not be found", folderId)); } InventoryFolderBase folder = DecodeFolderBase(folderId.Guid, cols); //grab the folder version ColumnPath path = new ColumnPath { Column = ByteEncoderHelper.UTF8Encoder.ToByteArray("count"), Column_family = FOLDERVERSIONS_CF }; object verVal = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get(folderIdArray, path, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); ColumnOrSuperColumn verColumn = (ColumnOrSuperColumn)verVal; folder.Version = (ushort)(verColumn.Counter_column.Value % (long)ushort.MaxValue); return folder; } catch (InventoryObjectMissingException e) { // _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e); throw; // produces a duplicate error farther up with more context } catch (Exception e) { // _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e); throw new InventoryStorageException(e.Message, e); // produces another error farther up with more context } }
public InventoryFolderBase GetFolder(UUID folderId) { if (folderId == UUID.Zero) throw new InventorySecurityException("Not returning folder with ID UUID.Zero"); try { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = FOLDERS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; byte[] folderIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(folderId.Guid); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_CONTENTS_CHUNK_SZ, folderIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); if (cols.Count == 0) { throw new InventoryObjectMissingException(String.Format("Folder with ID {0} could not be found", folderId)); } ParsedFolderData parsedData = new ParsedFolderData(cols); InventoryFolderBase folder = DecodeFolderBase(folderId.Guid, parsedData.Properties.Super_column.Columns); if (parsedData.SubFolders != null) { foreach (Column col in parsedData.SubFolders.Super_column.Columns) { SubFolderData data = SubFolderData.FromByteArray(col.Value); InventorySubFolderBase folderStub = new InventorySubFolderBase { ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Name)), Name = data.Name, Owner = folder.Owner, Type = data.Type }; folder.SubFolders.Add(folderStub); } } foreach (ColumnOrSuperColumn itemCol in parsedData.Items) { Guid itemId = ByteEncoderHelper.GuidEncoder.FromByteArray(itemCol.Super_column.Name); InventoryItemBase item = this.DecodeInventoryItem(itemCol.Super_column.Columns, itemId, folder.ID.Guid); folder.Items.Add(item); } //grab the folder version try { ColumnPath path = new ColumnPath { Column = ByteEncoderHelper.UTF8Encoder.ToByteArray("count"), Column_family = FOLDERVERSIONS_CF }; object verVal = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get(folderIdArray, path, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); ColumnOrSuperColumn verColumn = (ColumnOrSuperColumn)verVal; folder.Version = (ushort)(verColumn.Counter_column.Value % (long)ushort.MaxValue); } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Could not retrieve the version for folder {0} substituting 1: {1}", folderId, e); //version column missing. this is either a partially deleted folder //or the version mutation never happened. Return 1 folder.Version = 1; } return folder; } catch (InventoryObjectMissingException e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e); throw; } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder attributes: {0}", e); throw new InventoryStorageException(e.Message, e); } }
/// <summary> /// Retrieves the index of all folders owned by this user /// </summary> /// <param name="ownerId"></param> /// <returns></returns> private Dictionary<Guid, InventoryFolderBase> GetFolderIndex(UUID ownerId) { try { byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid); ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = USERFOLDERS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ, ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); Dictionary<Guid, InventoryFolderBase> retIndex = new Dictionary<Guid, InventoryFolderBase>(); foreach (ColumnOrSuperColumn col in cols) { Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns); try { InventoryFolderBase folder = new InventoryFolderBase { ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)), Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0], Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value), Owner = ownerId, ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)), Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value), }; retIndex.Add(folder.ID.Guid, folder); } catch (KeyNotFoundException) { //there is a corruption, this folder can not be read _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra]: Unable to read all columns from folder index item: {0}", new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)).ToString()); } } return retIndex; } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve folder index: {0}", e); throw new InventoryStorageException(e.Message, e); } }
/// <summary> /// Pulls down all folders and collects the list of subfolder /// UUIDs for each. Then attempts to read each of the sub folders /// listed in the subfolder index, and removes any indexed subfolders /// that are no longer readable due to a partial deletion /// </summary> /// <param name="ownerId"></param> /// <returns></returns> public void Maint_RepairSubfolderIndexes(UUID ownerId) { try { byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid); ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = USERFOLDERS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ, ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); List<InventoryFolderBase> goodIndexFolders = new List<InventoryFolderBase>(); foreach (ColumnOrSuperColumn col in cols) { Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns); try { InventoryFolderBase folder = new InventoryFolderBase { ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)), Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0], Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value), Owner = ownerId, ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)), Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value), }; goodIndexFolders.Add(folder); } catch (KeyNotFoundException) { //there is a corruption, this folder can not be read. Ignore since there is //another maint that can fix this that should be run first. } } List<KeyValuePair<Guid, Guid>> invalidParentChild = new List<KeyValuePair<Guid, Guid>>(); //for each folder in the index, retrieve it and check for unreadable subfolders foreach (InventoryFolderBase indexFolder in goodIndexFolders) { try { InventoryFolderBase folder = this.GetFolder(indexFolder.ID); foreach (var subfolder in folder.SubFolders) { try { InventoryFolderBase subFolderDetails = this.GetFolderAttributes(subfolder.ID); } catch (InventoryObjectMissingException) { invalidParentChild.Add(new KeyValuePair<Guid, Guid>(folder.ID.Guid, subfolder.ID.Guid)); } catch (InventoryStorageException e) { if (e.InnerException != null && e.InnerException is KeyNotFoundException) { invalidParentChild.Add(new KeyValuePair<Guid, Guid>(folder.ID.Guid, subfolder.ID.Guid)); } } } } catch (Exception e) { //we can't even get the folder, so no subfolders to fix _log.ErrorFormat("[InWorldz.Data.Inventory.Cassandra] Indexed folder {0} could not be retrieved to look for children: {1}", indexFolder.ID, e); } } _log.InfoFormat("[InWorldz.Data.Inventory.Cassandra][MAINT] Found {0} subfolder indexes to repair", invalidParentChild.Count); long timeStamp = Util.UnixTimeSinceEpochInMicroseconds(); //we couldn't read the folder, pull it from the subfolder list of its parent foreach (KeyValuePair<Guid, Guid> parentChildKvp in invalidParentChild) { byte[] childIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(parentChildKvp.Value); Dictionary<byte[], Dictionary<string, List<Mutation>>> muts = new Dictionary<byte[], Dictionary<string, List<Mutation>>>(); GenerateSubfolderIndexDeletion(parentChildKvp.Key, timeStamp, childIdBytes, muts); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { client.batch_mutate(muts, DEFAULT_CONSISTENCY_LEVEL); return null; }), KEYSPACE); } } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to repair subfolder index: {0}", e); throw new InventoryStorageException(e.Message, e); } }
/// <summary> /// Retrieves the index of all folders owned by this user and attempts to /// find and repair any inconsistencies /// </summary> /// <param name="ownerId"></param> /// <returns></returns> public void Maint_RepairFolderIndex(UUID ownerId) { try { byte[] ownerIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(ownerId.Guid); ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = USERFOLDERS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; List<ColumnOrSuperColumn> cols = this.RetrieveAllColumnsInChunks(FOLDER_INDEX_CHUNK_SZ, ownerIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); List<Guid> badIndexFolders = new List<Guid>(); foreach (ColumnOrSuperColumn col in cols) { Dictionary<string, Column> columns = this.IndexColumnsByUTF8Name(col.Super_column.Columns); try { InventoryFolderBase folder = new InventoryFolderBase { ID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)), Level = (InventoryFolderBase.FolderLevel)columns["level"].Value[0], Name = ByteEncoderHelper.UTF8Encoder.FromByteArray(columns["name"].Value), Owner = ownerId, ParentID = new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(columns["parent_folder"].Value)), Type = (short)ByteEncoderHelper.Int32Encoder.FromByteArray(columns["type"].Value), }; } catch (KeyNotFoundException) { //there is a corruption, this folder can not be read badIndexFolders.Add(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Super_column.Name)); } } List<Guid> destroyedFolders = new List<Guid>(); List<InventoryFolderBase> recoverableFolders = new List<InventoryFolderBase>(); //for each folder that has a bad index, try to read the folder. //if we can read the folder, restore it from the data we have //otherwise delete the index, the data is gone foreach (Guid id in badIndexFolders) { try { InventoryFolderBase folder = this.GetFolderAttributes(new UUID(id)); //also verify the parent exists and is readable InventoryFolderBase parentFolder = this.GetFolderAttributes(folder.ParentID); recoverableFolders.Add(folder); } catch (KeyNotFoundException) { destroyedFolders.Add(id); } catch (InventoryObjectMissingException) { destroyedFolders.Add(id); } } long timeStamp = Util.UnixTimeSinceEpochInMicroseconds(); foreach (InventoryFolderBase folder in recoverableFolders) { //recover anything recoverable this.CreateFolderInternal(folder, timeStamp); } foreach (Guid id in destroyedFolders) { this.RemoveFromIndex(ownerId.Guid, id, timeStamp); } } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to recover folder index: {0}", e); throw new InventoryStorageException(e.Message, e); } }
private List<UUID> GetActiveGestureItemIds(UUID userId) { try { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = USERACTIVEGESTURES_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object retobj = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { byte[] userIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(userId.Guid); return client.get_slice(userIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); List<ColumnOrSuperColumn> cols = (List<ColumnOrSuperColumn>)retobj; List<UUID> ret = new List<UUID>(); foreach (ColumnOrSuperColumn col in cols) { ret.Add(new UUID(ByteEncoderHelper.GuidEncoder.FromByteArray(col.Column.Name))); } return ret; } catch (Exception e) { throw new InventoryStorageException(e.Message, e); } }
public Guid FindItemParentFolderId(UUID itemId) { ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = ITEMPARENTS_CF; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; byte[] itemIdArray = ByteEncoderHelper.GuidEncoder.ToByteArray(itemId.Guid); ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object val = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get_slice(itemIdArray, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); List<ColumnOrSuperColumn> indexCols = (List<ColumnOrSuperColumn>)val; //no index means the item doesnt exist if (indexCols.Count == 0) { return Guid.Empty; } var indexedColsByName = this.IndexColumnsByUTF8Name(indexCols); return ByteEncoderHelper.GuidEncoder.FromByteArray(indexedColsByName["parent"].Value); }
public InventoryItemBase GetItem(UUID itemId, UUID parentFolderHint) { //Retrieving an item requires a lookup of the parent folder followed by //a retrieval of the item. This was a consious decision made since the //inventory item data currently takes up the most space and a //duplication of this data to prevent the index lookup //would be expensive in terms of space required try { Guid parentId; if (parentFolderHint != UUID.Zero) { parentId = parentFolderHint.Guid; } else { parentId = FindItemParentFolderId(itemId); } if (parentId == Guid.Empty) { throw new InventoryObjectMissingException(String.Format("Item with ID {0} could not be found", itemId), "Item was not found in the index"); } //try to retrieve the item. note that even though we have an index there is a chance we will //not have the item data due to a race condition between index mutation and item mutation byte[] itemIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(itemId.Guid); byte[] folderIdBytes = ByteEncoderHelper.GuidEncoder.ToByteArray(parentId); ColumnParent columnParent = new ColumnParent(); columnParent.Column_family = FOLDERS_CF; columnParent.Super_column = itemIdBytes; SlicePredicate pred = new SlicePredicate(); SliceRange range = new SliceRange(); range.Start = new byte[0]; range.Finish = new byte[0]; range.Reversed = false; range.Count = int.MaxValue; pred.Slice_range = range; ICluster cluster = AquilesHelper.RetrieveCluster(_clusterName); object itemDataObj = cluster.Execute(new ExecutionBlock(delegate(Apache.Cassandra.Cassandra.Client client) { return client.get_slice(folderIdBytes, columnParent, pred, DEFAULT_CONSISTENCY_LEVEL); }), KEYSPACE); List<ColumnOrSuperColumn> itemCols = (List<ColumnOrSuperColumn>)itemDataObj; if (itemCols.Count == 0) { throw new InventoryObjectMissingException(String.Format("Item with ID {0} could not be found", itemId), "Item was not found in its folder"); } InventoryItemBase item = this.DecodeInventoryItem(itemCols, itemId.Guid, parentId); return item; } catch (InventoryStorageException) { throw; } catch (Exception e) { _log.ErrorFormat("[Inworldz.Data.Inventory.Cassandra] Unable to retrieve item {0}: {1}", itemId, e); throw new InventoryStorageException(e.Message, e); } }
public void Read(TProtocol iprot) { TField field; iprot.ReadStructBegin(); while (true) { field = iprot.ReadFieldBegin(); if (field.Type == TType.Stop) { break; } switch (field.ID) { case 1: if (field.Type == TType.List) { { Column_names = new List<byte[]>(); TList _list4 = iprot.ReadListBegin(); for( int _i5 = 0; _i5 < _list4.Count; ++_i5) { byte[] _elem6 = null; _elem6 = iprot.ReadBinary(); Column_names.Add(_elem6); } iprot.ReadListEnd(); } } else { TProtocolUtil.Skip(iprot, field.Type); } break; case 2: if (field.Type == TType.Struct) { Slice_range = new SliceRange(); Slice_range.Read(iprot); } else { TProtocolUtil.Skip(iprot, field.Type); } break; default: TProtocolUtil.Skip(iprot, field.Type); break; } iprot.ReadFieldEnd(); } iprot.ReadStructEnd(); }
public SlicePredicate(List<byte[]> column_names, SliceRange sliceRange) : this(sliceRange) { if (column_names != null && column_names.Count > 0) Column_names = column_names; }
public SlicePredicate(SliceRange sliceRange) { if (sliceRange != null && sliceRange.Count > 0) Slice_range = sliceRange; }