/// <summary>
/// Does bulk Brunet DHT gets by calling the bulk DHT operation API of its
/// XML-RPC interface.
/// </summary>
/// <returns>Null if not all the pieces are successfully retrieved and parsed.</returns>
/// <exception cref="Exception">Parsing errors.</exception>
private MemBlock GetFragsInBulk(byte[] base_key, int piece_num,
out int largest_age, out int smallest_ttl)
{
largest_age = 0;
smallest_ttl = Int32.MaxValue;
MemBlock fragments = new MemBlock();
byte[][] keys = new byte[piece_num][];
for (int i = 0; i < piece_num; i++)
{
byte[] piece_key = BuildFragmentKey(base_key, i);
keys[i] = piece_key;
}
DhtGetResult[] dgrs = _dht.BulkGet(keys);
for (int i = 0; i < dgrs.Length; i++)
{
DhtGetResult dgr = dgrs[i];
if (dgr == null)
{
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Piece #{0} is null. Retrying...", i));
dgr = GetWithRetries(keys[i], 2);
if (dgr == null)
{
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Piece #{0} is null after retries. Skipping " +
"further parsing and returning...", i));
return(null);
}
}
FingerprintedData fpd;
try {
fpd =
(FingerprintedData)DictionaryData.CreateDictionaryData(dgr.value);
} catch (Exception ex) {
Logger.WriteLineIf(LogLevel.Error, _log_props,
ex);
throw ex;
}
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Piece #{0} retrieved and successfully parsed", i));
RegularData rd = fpd.InnerData as RegularData;
fragments = MemBlock.Concat(fragments, MemBlock.Reference(rd.PayLoad));
if (smallest_ttl > dgr.ttl)
{
smallest_ttl = dgr.ttl;
}
if (largest_age < dgr.age)
{
largest_age = dgr.age;
}
//Now it's safe to say, this attempt succeeded.
}
return(fragments);
}
protected MemBlock DecodeAndClear(uint crc32, int id, ushort total)
{
ICopyable[] blocks = new ICopyable[total];
bool missing_block = false;
ushort last_idx = (ushort)(total - 1);
for (ushort i = 0; i < last_idx; i++)
{
var key = new Triple <uint, int, ushort>(crc32, id, i);
var this_block = (MemBlock)_fragments.Remove(key);
if (null != this_block)
{
blocks[i] = this_block;
}
else
{
//We are missing one:
missing_block = true;
break;
}
}
//Get the last one:
ushort last_id = (ushort)(last_idx ^ FragmentingSender.LAST_BLOCK_FLAG);
var lastkey = new Triple <uint, int, ushort>(crc32, id, last_id);
var last_block = (MemBlock)_fragments.Remove(lastkey);
blocks[last_idx] = last_block;
missing_block = missing_block || (null == last_block);
MemBlock result = null;
if (false == missing_block)
{
result = MemBlock.Concat(blocks);
uint crc32_ck = result.Read <uint>(Crc32.ComputeChecksum);
if (crc32_ck != crc32)
{
//Something went wrong:
result = null;
}
}
_frag_count.Remove(new Pair <uint, int>(crc32, id));
return(result);
}
public void TestCAS()
{
Random r = new Random();
System.Collections.Hashtable ht = new System.Collections.Hashtable();
for (int i = 0; i < 128; i++)
{
byte[] key_buf = new byte[r.Next(1024)];
r.NextBytes(key_buf);
MemBlock key = MemBlock.Reference(key_buf);
byte[] val_buf = new byte[r.Next(1024)];
r.NextBytes(val_buf);
MemBlock val = MemBlock.Reference(val_buf);
MemBlock old_v = CompareSwap(key, val, null);
ht[key] = val;
Assert.IsNull(old_v, "old value is null");
//Try it again, make sure it doesn't work:
Assert.IsNotNull(CompareSwap(key, val, null), "old value is not null");
//Try it again with a different value:
MemBlock other_v = MemBlock.Concat(key, val);
Assert.AreEqual(val, CompareSwap(key, other_v, other_v), "update failed");
MemBlock current = Read(key);
Assert.AreEqual(val, current, "still not updated");
Assert.AreNotEqual(other_v, current, "make sure update didn't work");
//Now do a real update:
Assert.AreEqual(val, CompareSwap(key, other_v, val), "first update");
Assert.AreEqual(other_v, Read(key), "update worked");
ht[key] = other_v;
}
foreach (System.Collections.DictionaryEntry de in ht)
{
MemBlock recall_v = Read((MemBlock)de.Key);
Assert.AreEqual(recall_v, de.Value, "check recall");
}
}
/// <returns>The memblock assembled from pieces. Null if operation failed.</returns>
private MemBlock GetFragsSequentially(byte[] base_key, int piece_num,
out int largest_age, out int smallest_ttl)
{
largest_age = 0;
smallest_ttl = Int32.MaxValue;
MemBlock fragments = new MemBlock();
for (int i = 0; i < piece_num; i++)
{
byte[] piece_key = BuildFragmentKey(base_key, i);
bool succ = false; //set to false iff bad things happen
int retries = 3; //After that we fail the operation
for (; !succ && retries > 0; retries--)
{
if (retries == 3)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Getting: {0}", piece_key));
}
else
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Retrying..."));
}
try {
DhtGetResult[] dgrs = _dht.Get(piece_key);
//It should have only one entry. If not, just let the the exception caught
//and retry.
DhtGetResult dgr = dgrs[0];
FingerprintedData fpd =
(FingerprintedData)DictionaryData.CreateDictionaryData(dgr.value);
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Piece {0} retrieved and successfully parsed", piece_key));
RegularData rd = fpd.InnerData as RegularData;
fragments = MemBlock.Concat(fragments, MemBlock.Reference(rd.PayLoad));
if (smallest_ttl > dgr.ttl)
{
smallest_ttl = dgr.ttl;
}
if (largest_age < dgr.age)
{
largest_age = dgr.age;
}
//Now it's safe to say, this attempt succeeded.
succ = true;
} catch (Exception ex) {
Logger.WriteLineIf(LogLevel.Error, _log_props,
ex);
succ = false;
}
} //if succ then we are good
if (retries <= 0)
{
//Quit because retries exhausted.
throw new Exception(string.Format("Retries exhausted when retrieving "
+ "and deserializing piece : {0}", piece_key));
}
else
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format(
"Done with piece {0}",
Encoding.UTF8.GetString(piece_key)));
}
}
return(fragments);
}
/// <summary>
/// Concurrenly gets the fragments from DHT.
/// </summary>
/// <returns>The fragments</returns>
private MemBlock GetFragsConcurrently(byte[] baseKey, int pieceNum,
out int largestAge, out int smallestTtl)
{
// First set the int values to be invalid.
smallestTtl = Int32.MaxValue;
largestAge = 0;
int max_enqueues = 1;
BlockingQueue bq_result = new BlockingQueue(max_enqueues);
AsyncGetFragsGlobalState global_state =
new AsyncGetFragsGlobalState(pieceNum, baseKey, bq_result);
int concurrency_degree = Concurrency;
global_state.Concurrency = concurrency_degree;
for (int i = 0; i < pieceNum; i++)
{
int index = i;
byte[] piece_key = BuildFragmentKey(baseKey, i);
string piece_key_string = Encoding.UTF8.GetString(piece_key);
// piece state
AsyncOpState aps = new AsyncOpState(piece_key);
AsyncFragsOpState afos = new AsyncFragsOpState(global_state, aps);
// async get, one instance of IDht per get because thread safty not guaranteed
IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port);
BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations(
proxy);
lock (global_state.SyncRoot) {
brdht_ops.BeginGetWithCallback(piece_key,
new AsyncCallback(this.OnDhtGetReturns), afos);
}
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Get piece {0} from DHT began (asynchronously)",
piece_key_string));
if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) ||
index == pieceNum - 1)
{
// Stop to wait for batch finish or all finish
global_state.BatchFinishedEvent.WaitOne();
if (concurrency_degree > 1)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Batch {0} finished. Moving on",
(int)((index + 1) / concurrency_degree)));
}
}
}
//block here until result comes
GetFragsStoppedEventArgs stop_args =
(GetFragsStoppedEventArgs)bq_result.Dequeue();
//All results have returned.
MemBlock ret = new MemBlock();
if (stop_args.IsSuccessful)
{
// We successfully got everything but aren't sure whether they are
// correct. Now parse them.
for (int i = 0; i < global_state.Fragments.Length; i++)
{
try {
DhtGetResult dgr = global_state.Fragments[i];
FingerprintedData fpd =
(FingerprintedData)DictionaryData.CreateDictionaryData(dgr.value);
RegularData rd = fpd.InnerData as RegularData;
// This piece is OK.
ret = MemBlock.Concat(ret, MemBlock.Reference(rd.PayLoad));
if (smallestTtl > dgr.ttl)
{
smallestTtl = dgr.ttl;
}
if (largestAge < dgr.age)
{
largestAge = dgr.age;
}
//Now it's safe to say, this attempt succeeded.
} catch (Exception) {
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Parsing Piece failed at index {0}", i));
throw;
}
}
return(ret);
}
return(null);
}