internal override void Visit(ReadVersionRequest req) { VersionEntry versionEntry = req.RemoteVerEntry; if (versionEntry == null) { ConcurrentDictionary <long, VersionEntry> versionList = null; if (!this.dict.TryGetValue(req.RecordKey, out versionList)) { req.Result = null; req.Finished = true; return; } if (!versionList.TryGetValue(req.VersionKey, out versionEntry)) { req.Result = null; req.Finished = true; return; } } Debug.Assert(Interlocked.Read(ref versionEntry.VersionKey) == req.VersionKey); // Debug Assertion // if (!versionEntry.RecordKey.Equals(req.RecordKey)) // { // throw new Exception("Inconsistent record key"); // } //int ticket = versionEntry.EnterQueuedLatch(); versionEntry.ReadLock(); VersionEntry.CopyFromRemote(versionEntry, req.LocalVerEntry); //versionEntry.ExitQueuedLatch(ticket); versionEntry.UnReadLock(); req.Result = req.LocalVerEntry; req.Finished = true; }
public void Set( string tableId, object recordKey, long versionKey, long beginTimestamp, long endTimestamp, long txId, long readTxId, long expectedEndTimestamp, VersionEntry localVersionEntry = null, VersionEntry remoteVersionEntry = null) { this.TableId = tableId; this.RecordKey = recordKey; this.VersionKey = versionKey; this.BeginTs = beginTimestamp; this.EndTs = endTimestamp; this.TxId = txId; this.SenderId = readTxId; this.ExpectedEndTs = expectedEndTimestamp; this.LocalVerEntry = localVersionEntry; this.RemoteVerEntry = remoteVersionEntry; }
/// <summary> /// Only one thread will enter this method to remove the head. /// </summary> /// <returns></returns> internal VersionEntry RemoveFromHead() { long localMetaIndex = Interlocked.Read(ref this.metaIndex); int size = (int)(localMetaIndex >> 32); int tailIndex = (int)(localMetaIndex & 0xFFFFFFFFL); if (size == 0) { return(null); } int headIndex = (tailIndex - size + 1 + this.internalArray.Length) % this.internalArray.Length; VersionEntry element = this.internalArray[headIndex]; long newMetaIndex = ((long)size - 1) << 32 | (long)tailIndex; // As soon as the meta-index is updated, other threads will see the new head. Interlocked.Exchange(ref this.metaIndex, newMetaIndex); return(element); }
private int ExtractVersionEntry( byte[][] response, TxList <VersionEntry> dest) { Debug.Assert(response.Length <= 4 || response.Length % 2 == 0); int entryCount = response.Length / 2; long[] debugKeys = new long[entryCount]; for (int i = 0; i < entryCount; ++i) { int versionKeyIndex = i * 2; long versionKey = BitConverter.ToInt64( response[versionKeyIndex], 0); byte[] entryBytes = response[versionKeyIndex + 1]; VersionEntry.Deserialize( versionKey, entryBytes, dest[i]); debugKeys[i] = versionKey; } if (debugKeys.Length == 2) { Debug.Assert(debugKeys[0] + 1 == debugKeys[1]); } return(entryCount); }
/// <summary> /// Peeks last two elements in the list. /// /// Note that since concurrent threads may remove elements from the head or the tail, /// the elements returned by this method may have been removed. /// What's more, since removed elements are recycled to accommondate new data, /// it's possible the returned elements do not contain the original data anymore. /// It's the caller's responsibility to check this discrepancy. /// </summary> /// <param name="last">The last element</param> /// <param name="secondToLast">The second to the last element</param> public void PeekTail(out VersionEntry last, out VersionEntry secondToLast) { long localMetaIndex = Interlocked.Read(ref this.metaIndex); int size = (int)(localMetaIndex >> 32); int tailIndex = (int)(localMetaIndex & 0xFFFFFFFFL); if (size < 1) { last = null; secondToLast = null; return; } last = Volatile.Read(ref this.internalArray[tailIndex]); if (size < 2) { secondToLast = null; return; } secondToLast = Volatile.Read(ref this.internalArray[tailIndex - 1]); }
public void Set( string tableId, object recordKey, long versionKey, long beginTimestamp, long endTimestamp, long txId, object record, long tailKey, VersionEntry remoteVerEntry = null, IDictionary <long, VersionEntry> remoteVerList = null) { this.TableId = tableId; this.RecordKey = recordKey; this.VersionKey = versionKey; this.BeginTimestamp = beginTimestamp; this.EndTimestamp = endTimestamp; this.TxId = txId; this.Record = record; this.TailKey = tailKey; this.RemoteVerEntry = remoteVerEntry; this.RemoteVerList = remoteVerList; }
/// <summary> /// Only one thread will enter this method to append a new element to the tail. /// </summary> /// <param name="element">The element to append to the list</param> /// <returns></returns> internal bool AddToTail(VersionEntry element) { long localMetaIndex = Interlocked.Read(ref this.metaIndex); int size = (int)(localMetaIndex >> 32); int tailIndex = (int)(localMetaIndex & 0xFFFFFFFFL); if (size + 1 > this.internalArray.Length) { return(false); } int newTailIndex = (tailIndex + 1) % this.internalArray.Length; // By the time Interlocked.Exchange() finishes, the new element is added to the array. // But the meta-index has not been updated. As a result, other reading threads // continue to see an old version of the list, until the meta-index is updated. Interlocked.Exchange(ref this.internalArray[newTailIndex], element); long newMetaIndex = ((long)size + 1) << 32 | (long)newTailIndex; Interlocked.Exchange(ref this.metaIndex, newMetaIndex); return(true); }
internal override void Visit(ReadVersionRequest req) { VersionEntry versionEntry = req.RemoteVerEntry; if (versionEntry == null) { throw new TransactionException("The version entry should be referenced for main-memory k-v."); } if (versionEntry.VersionKey != req.VersionKey) { throw new TransactionException("The referenced version entry has been recycled for new data."); } while (Interlocked.CompareExchange(ref versionEntry.latch, 1, 0) != 0) { ; } VersionEntry.CopyValue(versionEntry, req.LocalVerEntry); Interlocked.Exchange(ref versionEntry.latch, 0); req.Result = req.LocalVerEntry; req.Finished = true; }
internal VersionEntry GetVersionEntryByKey(string tableId, object recordKey, long versionKey, VersionEntry ve) { var rs = this.CQLExecute(string.Format(PartitionedCassandraVersionTable.CQL_GET_VERSION_ENTRY, tableId, recordKey.ToString(), versionKey)); var rse = rs.GetEnumerator(); rse.MoveNext(); Row row = rse.Current; if (row == null) { return(null); } if (ve == null) { return(new VersionEntry(versionKey, row.GetValue <long>("begintimestamp"), row.GetValue <long>("endtimestamp"), BytesSerializer.Deserialize(row.GetValue <byte[]>("record")), row.GetValue <long>("txid"), row.GetValue <long>("maxcommitts"))); } else { ve.Set( versionKey, row.GetValue <long>("begintimestamp"), row.GetValue <long>("endtimestamp"), BytesSerializer.Deserialize(row.GetValue <byte[]>("record")), row.GetValue <long>("txid"), row.GetValue <long>("maxcommitts")); return(ve); } }
internal override void Visit(DeleteVersionRequest req) { ConcurrentDictionary <long, VersionEntry> versionList = req.RemoteVerList as ConcurrentDictionary <long, VersionEntry>; // Only get the version list location when version list is null if (versionList == null) { if (!this.dict.TryGetValue(req.RecordKey, out versionList)) { req.Result = true; req.Finished = true; return; } } VersionEntry tailEntry = null; versionList.TryGetValue(SingletonDictionaryVersionTable.TAIL_KEY, out tailEntry); long headKey = Interlocked.Read(ref tailEntry.EndTimestamp); long tailKey = Interlocked.Read(ref tailEntry.BeginTimestamp); // Debug Assertion // if (tailKey != req.VersionKey) // { // throw new Exception("Abort isn't deleting it's dirty write?"); // } // As only if a tx uploads a version entry, it will change the value tailEntry // Here the dirty version entry hasn't been deleted, so there will no other txs update the tailEntry // We don't need to take Interlocked to update its value // when the deleted entry is the only one in version list if (headKey == tailKey) { ResetTailEntry(tailEntry); req.Result = true; req.Finished = true; return; } Interlocked.Exchange(ref tailEntry.BeginTimestamp, tailKey - 1); VersionEntry versionEntry = null; if (headKey > 0) { versionList.TryAdd(headKey - 1, versionEntry); Interlocked.Exchange(ref tailEntry.EndTimestamp, headKey - 1); } if (versionList.TryRemove(req.VersionKey, out versionEntry)) { // Debug Assertion // if (Interlocked.Read(ref versionEntry.TxId) != req.SenderId) // { // throw new Exception("I'm not deleting my own dirty write?"); // } this.recordPool.TryCache(versionEntry.Record); versionEntry.Record = null; req.Result = true; } else { req.Result = false; } req.Finished = true; }
public UploadVersionRequest( string tableId, object recordKey, long versionKey, VersionEntry versionEntry) : base(tableId, recordKey, versionKey) { this.VersionEntry = versionEntry; }
internal override void Visit(GetVersionListRequest req) { ConcurrentDictionary <long, VersionEntry> versionList = null; if (!this.dict.TryGetValue(req.RecordKey, out versionList)) { req.RemoteVerList = null; req.Result = 0; req.Finished = true; return; } int entryCount = 0; // The value at -1 in the version list is a special entry, // whose beginTimestamp points to the newest version. VersionEntry tailEntry = null; versionList.TryGetValue(SingletonDictionaryVersionTable.TAIL_KEY, out tailEntry); long lastVersionKey = Interlocked.Read(ref tailEntry.BeginTimestamp); TxList <VersionEntry> localList = req.LocalContainer; TxList <VersionEntry> remoteList = req.RemoteContainer; // Only returns top 2 newest versions. This is enough for serializability. // For other isolation levels, more versions may need to be returned. // When old versions may be truncated, it is desirable to maintain a head pointer as well, // so as to increase the lower bound of version keys and reduce the number of iterations. while (lastVersionKey >= 0 && entryCount < 2) { VersionEntry verEntry = null; if (versionList.TryGetValue(lastVersionKey, out verEntry)) { //int ticket = verEntry.EnterQueuedLatch(); verEntry.ReadLock(); // Debug Assertion // if (!verEntry.RecordKey.Equals(req.RecordKey)) // { // throw new Exception("Inconsistent record key"); // } VersionEntry.CopyFromRemote(verEntry, localList[entryCount]); //verEntry.ExitQueuedLatch(ticket); verEntry.UnReadLock(); // Here only add a reference to the list, no need to take the latch remoteList.Add(verEntry); entryCount++; if (Interlocked.Read(ref verEntry.TxId) == VersionEntry.EMPTY_TXID) { break; } } lastVersionKey--; } req.RemoteVerList = versionList; req.Result = entryCount; req.Finished = true; }
internal override void Visit(UploadVersionRequest req) { ConcurrentDictionary <long, VersionEntry> versionList = req.RemoteVerList as ConcurrentDictionary <long, VersionEntry>; if (versionList == null) { if (!this.dict.TryGetValue(req.RecordKey, out versionList)) { throw new TransactionException("The specified record does not exist."); } } if (versionList.TryAdd(req.VersionKey, req.VersionEntry)) { req.VersionEntry.ResetLatchQueue(); // The new version has been inserted successfully. Re-directs the tail pointer to the new version. VersionEntry tailEntry = null; versionList.TryGetValue(SingletonDictionaryVersionTable.TAIL_KEY, out tailEntry); long headKey = Interlocked.Read(ref tailEntry.EndTimestamp); long tailKey = Interlocked.Read(ref tailEntry.BeginTimestamp); if (req.VersionKey < headKey) { versionList.Remove(req.VersionKey); UploadFail(req); return; } // Here we use Interlocked to atomically update the tail entry, instead of ConcurrentDict.TryUpdate(). // This is because once created, the whole tail entry always stays and is never replaced. // All concurrent tx's only access the tail pointer, i.e., the beginTimestamp field. long oldVersion = Interlocked.Exchange(ref tailEntry.BeginTimestamp, req.VersionKey); // Debug Assertion // if (oldVersion != req.VersionKey - 1) // { // throw new Exception("inconsistent version key"); // } VersionEntry oldVerEntry = null; // EndTimestamp (headKey) being never set means we just insert // the first valid version. So the headKey should be redirected. if (headKey == VersionEntry.DEFAULT_END_TIMESTAMP) { Interlocked.Exchange(ref tailEntry.EndTimestamp, tailKey); } else if (versionList.Count > VersionTable.VERSION_CAPACITY) { Interlocked.Exchange(ref tailEntry.EndTimestamp, headKey + 1); versionList.TryRemove(headKey, out oldVerEntry); if (oldVerEntry != null) { this.recordPool.TryCache(oldVerEntry.Record); oldVerEntry.Record = null; } } // Debug Assertion // long debugTailkey = Interlocked.Read(ref tailEntry.BeginTimestamp); // if (debugTailkey != req.VersionKey) // { // throw new Exception("Someone kicks in :("); // } req.RemoteVerEntry = oldVerEntry == null ? new VersionEntry() : oldVerEntry; req.Result = true; req.Finished = true; return; } else { // The same version key has been added before or by a concurrent tx. // The new version cannot be inserted. UploadFail(req); } }
static void ResetTailEntry(VersionEntry tailEntry) { tailEntry.BeginTimestamp = VersionEntry.DEFAULT_BEGIN_TIMESTAMP; tailEntry.EndTimestamp = VersionEntry.DEFAULT_END_TIMESTAMP; }
internal override void MockLoadData(int recordCount) { int pk = 0; RedisConnectionPool connPool = null; if (this.redisVersionDbMode == RedisVersionDbMode.Cluster) { connPool = this.singletonConnPool; } int loaded = 0; while (pk < this.PartitionCount) { Console.WriteLine("Loading Partition {0}", pk); if (connPool == null) { connPool = this.RedisManager.GetClientPool(this.redisDbIndex, RedisVersionDb.GetRedisInstanceIndex(pk)); } using (RedisClient redisClient = connPool.GetRedisClient()) { int batchSize = 100; int partitions = this.PartitionCount; for (int i = pk; i < recordCount; i += partitions * batchSize) { int upperBound = Math.Min(recordCount, i + partitions * batchSize); using (IRedisPipeline pipe = redisClient.CreatePipeline()) { for (int j = i; j < upperBound; j += partitions) { object recordKey = j; string hashId = recordKey.ToString(); if (this.redisVersionDbMode == RedisVersionDbMode.Cluster) { hashId = RedisVersionDb.PACK_KEY(RedisVersionDb.VER_KEY_PREFIX, hashId); } VersionEntry versionEntry = new VersionEntry(); VersionEntry.InitFirstVersionEntry(new String('a', 100), versionEntry); byte[] key = BitConverter.GetBytes(VersionEntry.VERSION_KEY_START_INDEX + 1); byte[] value = VersionEntry.Serialize(versionEntry); pipe.QueueCommand(r => ((RedisNativeClient)r).HSet(hashId, key, value)); pipe.QueueCommand(r => ((RedisNativeClient)r).HSet(hashId, RedisVersionDb.LATEST_VERSION_PTR_FIELD, key)); loaded++; } pipe.Flush(); } } } pk++; if (this.redisVersionDbMode != RedisVersionDbMode.Cluster) { connPool = null; } } Console.WriteLine("Loaded {0} records Successfully", loaded); }
internal override bool UploadNewVersionEntry(object recordKey, long versionKey, VersionEntry versionEntry) { ConcurrentDictionary <long, VersionEntry> versionList = null; if (!this.dict.TryGetValue(recordKey, out versionList)) { throw new TransactionException("The specified record does not exist."); } if (versionList.TryAdd(versionKey, versionEntry)) { // The new version has been inserted successfully. Re-directs the tail pointer to the new version. VersionEntry tailEntry = null; versionList.TryGetValue(SingletonDictionaryVersionTable.TAIL_KEY, out tailEntry); if (tailEntry == null) { throw new TransactionException("Tail pointer is missing from the version list."); } long tailKey = tailEntry.VersionKey; while (tailKey < versionKey) { // Here we use Interlocked to atomically update the tail entry, instead of ConcurrentDict.TryUpdate(). // This is because once created, the whole tail entry always stays and is never replaced. // All concurrent tx's only access the tail pointer, i.e., the beginTimestamp field. Interlocked.CompareExchange(ref tailEntry.VersionKey, versionKey, tailKey); tailKey = tailEntry.VersionKey; } return(true); } else { // The same version key has been added before or by a concurrent tx. // The new version cannot be inserted. return(false); } }
/// <summary> /// Upload a new version entry when insert or update a version /// </summary> /// <returns>True of False</returns> internal virtual bool UploadNewVersionEntry(object recordKey, long versionKey, VersionEntry versionEntry) { throw new NotImplementedException(); }
/// <summary> /// Replace the whole version entry, which will be called in the commit postprocessing phase. /// For write operations, like delete and update, the old version entry must be holden by the current /// transaction, which can be replaced directly rather than call lua script /// </summary> /// <param name="recordKey">The specify record key</param> /// <param name="versionKey">The specify version key</param> /// <param name="versionEntry">The version entry will be put</param> /// <returns></returns> internal virtual bool ReplaceWholeVersionEntry(object recordKey, long versionKey, VersionEntry versionEntry) { throw new NotImplementedException(); }