/// <summary>
/// Puts fragments to Dht.
/// </summary>
/// <param name="bde">The data object.</param>
/// <param name="fragInfo">Object that contains the meta info.</param>
/// <param name="concurrently">Puts concurrently if set to true</param>
/// <returns>True if the operation succeeds.</returns>
public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo,
bool concurrently)
{
byte[] info_key = bde.Key;
int ttl = bde.Ttl; //This ttl is used by every piece and the frag info
MemBlock data = MemBlock.Reference(bde.Value);
int piece_length = 0;
int offset = 0;
IList <FingerprintedData> fragments = new List <FingerprintedData>();
while (offset < data.Length)
{
piece_length = (offset + fragInfo.PieceLength > data.Length) ?
data.Length - offset : fragInfo.PieceLength;
MemBlock piece = data.Slice(offset, piece_length);
offset += piece_length;
FingerprintedData fpd = new FingerprintedData(new RegularData(piece));
fragments.Add(fpd);
}
//Update the piece number in fragInfo.
Logger.WriteLineIf(LogLevel.Info, _log_props,
string.Format("Data fragmented into {0} pieces", fragments.Count));
fragInfo.PieceNum = fragments.Count;
#if FUSHARE_PF
DateTime put_started = DateTime.UtcNow;
#endif
bool succ;
if (!concurrently)
{
succ = PutFragsSequentially(fragInfo, info_key, ttl, fragments);
}
else
{
//succ = PutFragsConcurrently(fragInfo, info_key, ttl, fragments);
succ = PutFragsInBulk(fragInfo, info_key, ttl, fragments);
}
#if FUSHARE_PF
DateTime put_finished = DateTime.UtcNow;
TimeSpan put_time = put_finished - put_started;
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Total time used to put {0}: {1} seconds",
Encoding.UTF8.GetString(fragInfo.BaseKey), put_time));
#endif
return(succ);
}
/// <returns>The memblock assembled from pieces. Null if operation failed.</returns>
private MemBlock GetFragsSequentially(byte[] base_key, int piece_num,
out int largest_age, out int smallest_ttl)
{
largest_age = 0;
smallest_ttl = Int32.MaxValue;
MemBlock fragments = new MemBlock();
for (int i = 0; i < piece_num; i++)
{
byte[] piece_key = BuildFragmentKey(base_key, i);
bool succ = false; //set to false iff bad things happen
int retries = 3; //After that we fail the operation
for (; !succ && retries > 0; retries--)
{
if (retries == 3)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Getting: {0}", piece_key));
}
else
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Retrying..."));
}
try {
DhtGetResult[] dgrs = _dht.Get(piece_key);
//It should have only one entry. If not, just let the the exception caught
//and retry.
DhtGetResult dgr = dgrs[0];
FingerprintedData fpd =
(FingerprintedData)DictionaryData.CreateDictionaryData(dgr.value);
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Piece {0} retrieved and successfully parsed", piece_key));
RegularData rd = fpd.InnerData as RegularData;
fragments = MemBlock.Concat(fragments, MemBlock.Reference(rd.PayLoad));
if (smallest_ttl > dgr.ttl)
{
smallest_ttl = dgr.ttl;
}
if (largest_age < dgr.age)
{
largest_age = dgr.age;
}
//Now it's safe to say, this attempt succeeded.
succ = true;
} catch (Exception ex) {
Logger.WriteLineIf(LogLevel.Error, _log_props,
ex);
succ = false;
}
} //if succ then we are good
if (retries <= 0)
{
//Quit because retries exhausted.
throw new Exception(string.Format("Retries exhausted when retrieving "
+ "and deserializing piece : {0}", piece_key));
}
else
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format(
"Done with piece {0}",
Encoding.UTF8.GetString(piece_key)));
}
}
return(fragments);
}
/// <summary>
/// Concurrently puts pieces into DHT
/// </summary>
/// <returns>True if successful</returns>
private bool PutFragsConcurrently(FragmentationInfo fragInfo,
byte[] infoKey, int ttl, IList <FingerprintedData> fragments)
{
bool ret;
//prepare the global state
int max_enqueues = 1; //we only need a true/false answer
BlockingQueue bq_result = new BlockingQueue(max_enqueues);
AsyncPutFragsGlobalState global_state =
new AsyncPutFragsGlobalState(fragments.Count, fragInfo.BaseKey, bq_result);
//@TODO tweak the number of concurrency_degree here.
int concurrency_degree = Concurrency;
global_state.Concurrency = concurrency_degree;
//Put pieces
for (int index = 0; index < fragments.Count; index++)
{
FingerprintedData fpd = fragments[index];
byte[] serializedFpd = fpd.SerializeTo();
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Size after serialization {0}", serializedFpd.Length));
byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index);
string piece_key_string = Encoding.UTF8.GetString(piece_key);
//piece state
AsyncOpState aps = new AsyncOpState(piece_key, serializedFpd, ttl);
AsyncFragsOpState apfs = new AsyncFragsOpState(global_state, aps);
//async put, one instance of IDht per put because thread safty not guaranteed
IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port);
BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations(
proxy);
// Fire the async XML-RPC call.
lock (global_state.SyncRoot) {
brdht_ops.BeginPutWithCallback(piece_key, serializedFpd, ttl,
new AsyncCallback(this.OnDhtPutReturns), apfs);
}
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Put piece {0} to DHT began (asynchronously)",
piece_key_string));
if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) ||
index == fragments.Count - 1)
{
// Stop to wait for batch finish or all finish
global_state.BatchFinishedEvent.WaitOne();
if (concurrency_degree > 1)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Batch {0} finished. Moving on",
(int)((index + 1) / concurrency_degree)));
}
}
}
//block here until result comes
PutFragsStoppedEventArgs stop_args =
(PutFragsStoppedEventArgs)bq_result.Dequeue();
// Deal with info object.
if (stop_args.IsSuccessful)
{
ret = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl);
if (ret)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("FragmentationInfo {0} successfully put",
Encoding.UTF8.GetString(infoKey)));
}
}
else
{
ret = false;
}
return(ret);
}
/// <summary>
/// Concurrenly gets the fragments from DHT.
/// </summary>
/// <returns>The fragments</returns>
private MemBlock GetFragsConcurrently(byte[] baseKey, int pieceNum,
out int largestAge, out int smallestTtl)
{
// First set the int values to be invalid.
smallestTtl = Int32.MaxValue;
largestAge = 0;
int max_enqueues = 1;
BlockingQueue bq_result = new BlockingQueue(max_enqueues);
AsyncGetFragsGlobalState global_state =
new AsyncGetFragsGlobalState(pieceNum, baseKey, bq_result);
int concurrency_degree = Concurrency;
global_state.Concurrency = concurrency_degree;
for (int i = 0; i < pieceNum; i++)
{
int index = i;
byte[] piece_key = BuildFragmentKey(baseKey, i);
string piece_key_string = Encoding.UTF8.GetString(piece_key);
// piece state
AsyncOpState aps = new AsyncOpState(piece_key);
AsyncFragsOpState afos = new AsyncFragsOpState(global_state, aps);
// async get, one instance of IDht per get because thread safty not guaranteed
IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port);
BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations(
proxy);
lock (global_state.SyncRoot) {
brdht_ops.BeginGetWithCallback(piece_key,
new AsyncCallback(this.OnDhtGetReturns), afos);
}
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Get piece {0} from DHT began (asynchronously)",
piece_key_string));
if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) ||
index == pieceNum - 1)
{
// Stop to wait for batch finish or all finish
global_state.BatchFinishedEvent.WaitOne();
if (concurrency_degree > 1)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Batch {0} finished. Moving on",
(int)((index + 1) / concurrency_degree)));
}
}
}
//block here until result comes
GetFragsStoppedEventArgs stop_args =
(GetFragsStoppedEventArgs)bq_result.Dequeue();
//All results have returned.
MemBlock ret = new MemBlock();
if (stop_args.IsSuccessful)
{
// We successfully got everything but aren't sure whether they are
// correct. Now parse them.
for (int i = 0; i < global_state.Fragments.Length; i++)
{
try {
DhtGetResult dgr = global_state.Fragments[i];
FingerprintedData fpd =
(FingerprintedData)DictionaryData.CreateDictionaryData(dgr.value);
RegularData rd = fpd.InnerData as RegularData;
// This piece is OK.
ret = MemBlock.Concat(ret, MemBlock.Reference(rd.PayLoad));
if (smallestTtl > dgr.ttl)
{
smallestTtl = dgr.ttl;
}
if (largestAge < dgr.age)
{
largestAge = dgr.age;
}
//Now it's safe to say, this attempt succeeded.
} catch (Exception) {
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Parsing Piece failed at index {0}", i));
throw;
}
}
return(ret);
}
return(null);
}
/// <summary>
/// Puts fragments to Dht.
/// </summary>
/// <param name="bde">The data object.</param>
/// <param name="fragInfo">Object that contains the meta info.</param>
/// <param name="concurrently">Puts concurrently if set to true</param>
/// <returns>True if the operation succeeds.</returns>
public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo,
bool concurrently)
{
byte[] info_key = bde.Key;
int ttl = bde.Ttl; //This ttl is used by every piece and the frag info
MemBlock data = MemBlock.Reference(bde.Value);
int piece_length = 0;
int offset = 0;
IList<FingerprintedData> fragments = new List<FingerprintedData>();
while (offset < data.Length) {
piece_length = (offset + fragInfo.PieceLength > data.Length) ?
data.Length - offset : fragInfo.PieceLength;
MemBlock piece = data.Slice(offset, piece_length);
offset += piece_length;
FingerprintedData fpd = new FingerprintedData(new RegularData(piece));
fragments.Add(fpd);
}
//Update the piece number in fragInfo.
Logger.WriteLineIf(LogLevel.Info, _log_props,
string.Format("Data fragmented into {0} pieces", fragments.Count));
fragInfo.PieceNum = fragments.Count;
#if FUSHARE_PF
DateTime put_started = DateTime.UtcNow;
#endif
bool succ;
if (!concurrently) {
succ = PutFragsSequentially(fragInfo, info_key, ttl, fragments);
} else {
//succ = PutFragsConcurrently(fragInfo, info_key, ttl, fragments);
succ = PutFragsInBulk(fragInfo, info_key, ttl, fragments);
}
#if FUSHARE_PF
DateTime put_finished = DateTime.UtcNow;
TimeSpan put_time = put_finished - put_started;
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Total time used to put {0}: {1} seconds",
Encoding.UTF8.GetString(fragInfo.BaseKey), put_time));
#endif
return succ;
}
