/// <summary>
/// Put with a size limit. Data larger than the limit will be splitted into
/// pieces and put to different under different DHT keys.
/// </summary>
/// <param name="maxSize">The size limit</param>
/// <param name="split">Whether to split the oversized data</param>
/// <exception cref="ArgumentException">Data larger than the limit and
/// split specified as false</exception>
public bool Put(byte[] key, byte[] value, int ttl, int maxSize, bool split)
{
bool result;
if (value.Length > maxSize)
{
if (split)
{
BrunetDhtEntry bde = new BrunetDhtEntry(key, value, ttl);
FragmentationInfo frag_info = new FragmentationInfo(key);
frag_info.PieceLength = maxSize;
result = PutFragments(bde, frag_info);
}
else
{
throw new ArgumentException("Data too large but split opted out.");
}
}
else
{
result = _dht.Put(key, value, ttl);
}
return(result);
}
/// <summary>
/// Gets the first data item of the given name and possibly gets the
/// indicated pieces from DHT.
/// </summary>
/// <param name="getPieces">Whether to get pieces if the data at the
/// specified DHT name is a FragmentationInfo</param>
public DhtGetResult GetOneDatum(byte[] key, bool getPieces,
OneDatumMode mode)
{
DhtGetResult[] results = _dht.Get(key);
DhtGetResult ret;
if (results.Length == 0)
{
ret = null;
}
else
{
DhtGetResult dgr;
if (mode == OneDatumMode.FirstOne)
{
dgr = results[0];
}
else if (mode == OneDatumMode.LastOne)
{
dgr = results[results.Length - 1];
}
else
{
throw new NotImplementedException(
"This OneDatumMode not implemented.");
}
DictionaryData dd = null;
try {
dd = DictionaryData.CreateDictionaryData(dgr.value);
} catch (Exception ex) {
// Not an error in this case. Log with verbose level.
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
ex);
}
if (dd != null && dd is FragmentationInfo)
{
FragmentationInfo frag_info = dd as FragmentationInfo;
BrunetDhtEntry bde = null;
try {
bde = GetFragments(frag_info) as BrunetDhtEntry;
RegularData rd = (RegularData)DictionaryData.CreateDictionaryData(
bde.Value);
//Only 1 entry (if any) in this array
ret = new DhtGetResult(rd.PayLoad, bde.Age, bde.Ttl);
} catch (Exception ex) {
Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Can't get fragments."), ex);
ret = null;
}
}
else
{
ret = dgr;
}
}
return(ret);
}
/// <exception cref="Exception">Operation Failed</exception>
/// <returns>A BrunetDhtEntry that has the whole chunk of data</returns>
public DictionaryData GetFragments(FragmentationInfo info, bool concurrently)
{
BrunetDhtEntry ret = null;
byte[] base_key = info.BaseKey;
int piece_num = info.PieceNum;
MemBlock fragments;
int largest_age;
int smallest_ttl;
/*
* @TODO: It doesn't have to be sequential but let's first get it work with
* this approach
*/
#if FUSHARE_PF
DateTime get_started = DateTime.UtcNow;
#endif
if (!concurrently)
{
fragments = GetFragsSequentially(base_key, piece_num, out largest_age, out smallest_ttl);
}
else
{
//fragments = GetFragsConcurrently(base_key, piece_num, out largest_age, out smallest_ttl);
fragments = GetFragsInBulk(base_key, piece_num, out largest_age, out smallest_ttl);
}
#if FUSHARE_PF
DateTime get_finished = DateTime.UtcNow;
TimeSpan get_time = get_finished - get_started;
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Total time used to get {0}: {1} seconds",
Encoding.UTF8.GetString(base_key), get_time));
#endif
if (fragments != null)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Fragments successfully got from DHT"));
//Got the fragments correctly
ret = new BrunetDhtEntry(base_key, (new RegularData(fragments)).SerializeTo(),
largest_age, smallest_ttl);
return(ret);
}
else
{
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("DHT Get of fragments failed"));
throw new Exception("DHT Get of fragments failed.");
}
}
/// <summary>
/// Puts fragments to Dht.
/// </summary>
/// <param name="bde">The data object.</param>
/// <param name="fragInfo">Object that contains the meta info.</param>
/// <param name="concurrently">Puts concurrently if set to true</param>
/// <returns>True if the operation succeeds.</returns>
public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo,
bool concurrently)
{
byte[] info_key = bde.Key;
int ttl = bde.Ttl; //This ttl is used by every piece and the frag info
MemBlock data = MemBlock.Reference(bde.Value);
int piece_length = 0;
int offset = 0;
IList <FingerprintedData> fragments = new List <FingerprintedData>();
while (offset < data.Length)
{
piece_length = (offset + fragInfo.PieceLength > data.Length) ?
data.Length - offset : fragInfo.PieceLength;
MemBlock piece = data.Slice(offset, piece_length);
offset += piece_length;
FingerprintedData fpd = new FingerprintedData(new RegularData(piece));
fragments.Add(fpd);
}
//Update the piece number in fragInfo.
Logger.WriteLineIf(LogLevel.Info, _log_props,
string.Format("Data fragmented into {0} pieces", fragments.Count));
fragInfo.PieceNum = fragments.Count;
#if FUSHARE_PF
DateTime put_started = DateTime.UtcNow;
#endif
bool succ;
if (!concurrently)
{
succ = PutFragsSequentially(fragInfo, info_key, ttl, fragments);
}
else
{
//succ = PutFragsConcurrently(fragInfo, info_key, ttl, fragments);
succ = PutFragsInBulk(fragInfo, info_key, ttl, fragments);
}
#if FUSHARE_PF
DateTime put_finished = DateTime.UtcNow;
TimeSpan put_time = put_finished - put_started;
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Total time used to put {0}: {1} seconds",
Encoding.UTF8.GetString(fragInfo.BaseKey), put_time));
#endif
return(succ);
}
public void TestPutAndGetFragments()
{
MockBrunetDht mock_dht = new MockBrunetDht();
BrunetDht dht = new BrunetDht(mock_dht);
FragmentationInfo frag_info = new FragmentationInfo();
byte[] b_key = MakeByteArray(20);
string key = Encoding.UTF8.GetString(b_key);
frag_info.BaseKey = Encoding.UTF8.GetString(b_key);
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("key : {0}", Base32.Encode(b_key)));
byte[] expected = MakeByteArray(1024 * 10);
int ttl = 1000;
BrunetDhtEntry bde = new BrunetDhtEntry(key, expected, ttl);
dht.PutFragments(bde, frag_info);
Hashtable ht = mock_dht.HTStorage;
Assert.AreEqual(11, ht.Count);
////
byte[] serialized_fragInfo = ((DhtGetResult[])dht.Get(key))[0].value;
FragmentationInfo frag_info_actual =
DictionaryData.CreateDictionaryData(serialized_fragInfo) as FragmentationInfo;
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("BaseKey: {0}", Base32.Encode(Encoding.UTF8.GetBytes(frag_info_actual.BaseKey))));
BrunetDhtEntry actual = dht.GetFragments(frag_info) as BrunetDhtEntry;
MemBlock mb = MemBlock.Reference(actual.Value);
MemBlock mb_exp = MemBlock.Reference(expected);
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
mb.ToBase32String().Substring(0, 50));
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
mb_exp.ToBase32String().Substring(0, 50));
Assert.AreEqual(mb_exp, mb);
}
public void TestPutAndGetFragments()
{
MockBrunetDht mock_dht = new MockBrunetDht();
BrunetDht dht = new BrunetDht(mock_dht);
FragmentationInfo frag_info = new FragmentationInfo();
byte[] b_key = MakeByteArray(20);
string key = Encoding.UTF8.GetString(b_key);
frag_info.BaseKey = Encoding.UTF8.GetString(b_key);
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("key : {0}", Base32.Encode(b_key)));
byte[] expected = MakeByteArray(1024 * 10);
int ttl = 1000;
BrunetDhtEntry bde = new BrunetDhtEntry(key, expected, ttl);
dht.PutFragments(bde, frag_info);
Hashtable ht = mock_dht.HTStorage;
Assert.AreEqual(11, ht.Count);
////
byte[] serialized_fragInfo = ((DhtGetResult[])dht.Get(key))[0].value;
FragmentationInfo frag_info_actual =
DictionaryData.CreateDictionaryData(serialized_fragInfo) as FragmentationInfo;
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("BaseKey: {0}", Base32.Encode(Encoding.UTF8.GetBytes(frag_info_actual.BaseKey))));
BrunetDhtEntry actual = dht.GetFragments(frag_info) as BrunetDhtEntry;
MemBlock mb = MemBlock.Reference(actual.Value);
MemBlock mb_exp = MemBlock.Reference(expected);
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
mb.ToBase32String().Substring(0, 50));
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
mb_exp.ToBase32String().Substring(0, 50));
Assert.AreEqual(mb_exp, mb);
}
public DictionaryData GetFragments(FragmentationInfo info)
{
return this.GetFragments(info, true);
}
/// <summary>
/// Puts fragments and chooses the concurrent option
/// </summary>
public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo)
{
return(this.PutFragments(bde, fragInfo, true));
}
/// <exception cref="Exception">Operation Failed</exception>
/// <returns>A BrunetDhtEntry that has the whole chunk of data</returns>
public DictionaryData GetFragments(FragmentationInfo info, bool concurrently)
{
BrunetDhtEntry ret = null;
byte[] base_key = info.BaseKey;
int piece_num = info.PieceNum;
MemBlock fragments;
int largest_age;
int smallest_ttl;
/*
* @TODO: It doesn't have to be sequential but let's first get it work with
* this approach
*/
#if FUSHARE_PF
DateTime get_started = DateTime.UtcNow;
#endif
if (!concurrently) {
fragments = GetFragsSequentially(base_key, piece_num, out largest_age, out smallest_ttl);
} else {
//fragments = GetFragsConcurrently(base_key, piece_num, out largest_age, out smallest_ttl);
fragments = GetFragsInBulk(base_key, piece_num, out largest_age, out smallest_ttl);
}
#if FUSHARE_PF
DateTime get_finished = DateTime.UtcNow;
TimeSpan get_time = get_finished - get_started;
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Total time used to get {0}: {1} seconds",
Encoding.UTF8.GetString(base_key), get_time));
#endif
if (fragments != null) {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Fragments successfully got from DHT"));
//Got the fragments correctly
ret = new BrunetDhtEntry(base_key, (new RegularData(fragments)).SerializeTo(),
largest_age, smallest_ttl);
return ret;
} else {
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("DHT Get of fragments failed"));
throw new Exception("DHT Get of fragments failed.");
}
}
public DictionaryData GetFragments(FragmentationInfo info)
{
return(this.GetFragments(info, true));
}
/// <summary>
/// Concurrently puts pieces into DHT
/// </summary>
/// <returns>True if successful</returns>
private bool PutFragsConcurrently(FragmentationInfo fragInfo,
byte[] infoKey, int ttl, IList <FingerprintedData> fragments)
{
bool ret;
//prepare the global state
int max_enqueues = 1; //we only need a true/false answer
BlockingQueue bq_result = new BlockingQueue(max_enqueues);
AsyncPutFragsGlobalState global_state =
new AsyncPutFragsGlobalState(fragments.Count, fragInfo.BaseKey, bq_result);
//@TODO tweak the number of concurrency_degree here.
int concurrency_degree = Concurrency;
global_state.Concurrency = concurrency_degree;
//Put pieces
for (int index = 0; index < fragments.Count; index++)
{
FingerprintedData fpd = fragments[index];
byte[] serializedFpd = fpd.SerializeTo();
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Size after serialization {0}", serializedFpd.Length));
byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index);
string piece_key_string = Encoding.UTF8.GetString(piece_key);
//piece state
AsyncOpState aps = new AsyncOpState(piece_key, serializedFpd, ttl);
AsyncFragsOpState apfs = new AsyncFragsOpState(global_state, aps);
//async put, one instance of IDht per put because thread safty not guaranteed
IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port);
BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations(
proxy);
// Fire the async XML-RPC call.
lock (global_state.SyncRoot) {
brdht_ops.BeginPutWithCallback(piece_key, serializedFpd, ttl,
new AsyncCallback(this.OnDhtPutReturns), apfs);
}
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Put piece {0} to DHT began (asynchronously)",
piece_key_string));
if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) ||
index == fragments.Count - 1)
{
// Stop to wait for batch finish or all finish
global_state.BatchFinishedEvent.WaitOne();
if (concurrency_degree > 1)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Batch {0} finished. Moving on",
(int)((index + 1) / concurrency_degree)));
}
}
}
//block here until result comes
PutFragsStoppedEventArgs stop_args =
(PutFragsStoppedEventArgs)bq_result.Dequeue();
// Deal with info object.
if (stop_args.IsSuccessful)
{
ret = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl);
if (ret)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("FragmentationInfo {0} successfully put",
Encoding.UTF8.GetString(infoKey)));
}
}
else
{
ret = false;
}
return(ret);
}
/// <summary>
/// Put with a size limit. Data larger than the limit will be splitted into
/// pieces and put to different under different DHT keys.
/// </summary>
/// <param name="maxSize">The size limit</param>
/// <param name="split">Whether to split the oversized data</param>
/// <exception cref="ArgumentException">Data larger than the limit and
/// split specified as false</exception>
public bool Put(byte[] key, byte[] value, int ttl, int maxSize, bool split)
{
bool result;
if (value.Length > maxSize) {
if (split) {
BrunetDhtEntry bde = new BrunetDhtEntry(key, value, ttl);
FragmentationInfo frag_info = new FragmentationInfo(key);
frag_info.PieceLength = maxSize;
result = PutFragments(bde, frag_info);
} else {
throw new ArgumentException("Data too large but split opted out.");
}
} else {
result = _dht.Put(key, value, ttl);
}
return result;
}
private bool PutFragsInBulk(FragmentationInfo fragInfo,
byte[] infoKey, int ttl, IList<FingerprintedData> fragments)
{
string info_key_string = Encoding.UTF8.GetString(infoKey);
XmlRpcStruct[] parameters = new XmlRpcStruct[fragments.Count];
// Prepare parameters for bulk put.
int index = 0;
foreach (FingerprintedData fpd in fragments) {
byte[] serializedFpd = fpd.SerializeTo();
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Size of piece {0} after serialization: {1}", index,
serializedFpd.Length));
byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index);
XmlRpcStruct param_dict = new XmlRpcStruct();
param_dict.Add("name", piece_key);
param_dict.Add("value", serializedFpd);
param_dict.Add("ttl", ttl);
parameters[index] = param_dict;
index++;
}
bool[] results;
try {
results = _dht.BulkPut(parameters);
} catch (Exception ex) {
// Network related operation, log the exception and let it be handled
// upper stream caller.
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Exception caught in BulkPut"));
Logger.WriteLineIf(LogLevel.Error, _log_props,
ex);
throw;
}
IList<int> failed_pieces = new List<int>();
for (int i = 0; i < results.Length; i++) {
bool result = results[i];
if (!result) {
failed_pieces.Add(i);
}
}
// Log failed pieces
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Failed {0} pieces are:", failed_pieces.Count));
StringBuilder sb = new StringBuilder();
foreach (int pieceIndx in failed_pieces) {
sb.Append(pieceIndx);
sb.Append(" ");
}
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
sb.ToString());
// Decide whether to retry.
if (failed_pieces.Count > 20) {
Logger.WriteLineIf(LogLevel.Info, _log_props,
string.Format("Too many pieces failed. Returning..."));
return false;
}
//Retry failed pieces.
foreach (int pieceIndx in failed_pieces) {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Retrying piece {0}", pieceIndx));
bool succ_retries = PutWithRetries((byte[])parameters[pieceIndx]["name"], (byte[])
parameters[pieceIndx]["value"], (int)parameters[pieceIndx]["ttl"], 2);
if (!succ_retries) {
// Cannot succeed on this piece even after retry...
return false;
}
}
//Put info
bool succ_info = PutWithRetries(infoKey, fragInfo.SerializeTo(), ttl, 2);
return succ_info;
}
private bool PutFragsSequentially(FragmentationInfo fragInfo,
byte[] infoKey, int ttl, IList <FingerprintedData> fragments)
{
string info_key_string = Encoding.UTF8.GetString(infoKey);
//Put pieces
int index = 0;
foreach (FingerprintedData fpd in fragments)
{
byte[] serializedFpd = fpd.SerializeTo();
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Size after serialization {0}", serializedFpd.Length));
int i = index++;
bool succ = false;
int retries = 3;
byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, i);
string piece_key_string = Encoding.UTF8.GetString(piece_key);
for (; !succ && retries > 0; retries--)
{
if (retries < 3)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Retrying..."));
}
succ = _dht.Put(piece_key, serializedFpd, ttl);
if (!succ)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Put failed."));
}
}
if (retries <= 0)
{
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Retries exhausted when putting piece : {0}",
piece_key_string));
return(false);
}
else
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Piece {0} successfully put to DHT", piece_key_string));
}
}
//Put info
bool succ_info = false;
int retries_info = 3;
for (; !succ_info && retries_info > 0; retries_info--)
{
if (retries_info < 3)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Retrying..."));
}
succ_info = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl);
}
if (retries_info <= 0)
{
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Retries exhausted when putting FragmentationInfo : {0}",
info_key_string));
return(false);
}
else
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("FragmentationInfo successfully put to DHT with name {0}",
info_key_string));
}
return(true);
}
private bool PutFragsSequentially(FragmentationInfo fragInfo,
byte[] infoKey, int ttl, IList<FingerprintedData> fragments)
{
string info_key_string = Encoding.UTF8.GetString(infoKey);
//Put pieces
int index = 0;
foreach (FingerprintedData fpd in fragments) {
byte[] serializedFpd = fpd.SerializeTo();
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Size after serialization {0}", serializedFpd.Length));
int i = index++;
bool succ = false;
int retries = 3;
byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, i);
string piece_key_string = Encoding.UTF8.GetString(piece_key);
for (; !succ && retries > 0; retries--) {
if (retries < 3) {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Retrying..."));
}
succ = _dht.Put(piece_key, serializedFpd, ttl);
if (!succ) {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Put failed."));
}
}
if (retries <= 0) {
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Retries exhausted when putting piece : {0}",
piece_key_string));
return false;
} else {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Piece {0} successfully put to DHT", piece_key_string));
}
}
//Put info
bool succ_info = false;
int retries_info = 3;
for (; !succ_info && retries_info > 0; retries_info--) {
if (retries_info < 3) {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Retrying..."));
}
succ_info = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl);
}
if (retries_info <= 0) {
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Retries exhausted when putting FragmentationInfo : {0}",
info_key_string));
return false;
} else {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("FragmentationInfo successfully put to DHT with name {0}",
info_key_string));
}
return true;
}
/// <summary>
/// Puts fragments to Dht.
/// </summary>
/// <param name="bde">The data object.</param>
/// <param name="fragInfo">Object that contains the meta info.</param>
/// <param name="concurrently">Puts concurrently if set to true</param>
/// <returns>True if the operation succeeds.</returns>
public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo,
bool concurrently)
{
byte[] info_key = bde.Key;
int ttl = bde.Ttl; //This ttl is used by every piece and the frag info
MemBlock data = MemBlock.Reference(bde.Value);
int piece_length = 0;
int offset = 0;
IList<FingerprintedData> fragments = new List<FingerprintedData>();
while (offset < data.Length) {
piece_length = (offset + fragInfo.PieceLength > data.Length) ?
data.Length - offset : fragInfo.PieceLength;
MemBlock piece = data.Slice(offset, piece_length);
offset += piece_length;
FingerprintedData fpd = new FingerprintedData(new RegularData(piece));
fragments.Add(fpd);
}
//Update the piece number in fragInfo.
Logger.WriteLineIf(LogLevel.Info, _log_props,
string.Format("Data fragmented into {0} pieces", fragments.Count));
fragInfo.PieceNum = fragments.Count;
#if FUSHARE_PF
DateTime put_started = DateTime.UtcNow;
#endif
bool succ;
if (!concurrently) {
succ = PutFragsSequentially(fragInfo, info_key, ttl, fragments);
} else {
//succ = PutFragsConcurrently(fragInfo, info_key, ttl, fragments);
succ = PutFragsInBulk(fragInfo, info_key, ttl, fragments);
}
#if FUSHARE_PF
DateTime put_finished = DateTime.UtcNow;
TimeSpan put_time = put_finished - put_started;
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Total time used to put {0}: {1} seconds",
Encoding.UTF8.GetString(fragInfo.BaseKey), put_time));
#endif
return succ;
}
/// <summary>
/// Puts fragments and chooses the concurrent option
/// </summary>
public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo)
{
return this.PutFragments(bde, fragInfo, true);
}
private bool PutFragsInBulk(FragmentationInfo fragInfo,
byte[] infoKey, int ttl, IList <FingerprintedData> fragments)
{
string info_key_string = Encoding.UTF8.GetString(infoKey);
XmlRpcStruct[] parameters = new XmlRpcStruct[fragments.Count];
// Prepare parameters for bulk put.
int index = 0;
foreach (FingerprintedData fpd in fragments)
{
byte[] serializedFpd = fpd.SerializeTo();
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Size of piece {0} after serialization: {1}", index,
serializedFpd.Length));
byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index);
XmlRpcStruct param_dict = new XmlRpcStruct();
param_dict.Add("name", piece_key);
param_dict.Add("value", serializedFpd);
param_dict.Add("ttl", ttl);
parameters[index] = param_dict;
index++;
}
bool[] results;
try {
results = _dht.BulkPut(parameters);
} catch (Exception ex) {
// Network related operation, log the exception and let it be handled
// upper stream caller.
Logger.WriteLineIf(LogLevel.Error, _log_props,
string.Format("Exception caught in BulkPut"));
Logger.WriteLineIf(LogLevel.Error, _log_props,
ex);
throw;
}
IList <int> failed_pieces = new List <int>();
for (int i = 0; i < results.Length; i++)
{
bool result = results[i];
if (!result)
{
failed_pieces.Add(i);
}
}
// Log failed pieces
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Failed {0} pieces are:", failed_pieces.Count));
StringBuilder sb = new StringBuilder();
foreach (int pieceIndx in failed_pieces)
{
sb.Append(pieceIndx);
sb.Append(" ");
}
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
sb.ToString());
// Decide whether to retry.
if (failed_pieces.Count > 20)
{
Logger.WriteLineIf(LogLevel.Info, _log_props,
string.Format("Too many pieces failed. Returning..."));
return(false);
}
//Retry failed pieces.
foreach (int pieceIndx in failed_pieces)
{
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Retrying piece {0}", pieceIndx));
bool succ_retries = PutWithRetries((byte[])parameters[pieceIndx]["name"], (byte[])
parameters[pieceIndx]["value"], (int)parameters[pieceIndx]["ttl"], 2);
if (!succ_retries)
{
// Cannot succeed on this piece even after retry...
return(false);
}
}
//Put info
bool succ_info = PutWithRetries(infoKey, fragInfo.SerializeTo(), ttl, 2);
return(succ_info);
}
/// <summary>
/// Concurrently puts pieces into DHT
/// </summary>
/// <returns>True if successful</returns>
private bool PutFragsConcurrently(FragmentationInfo fragInfo,
byte[] infoKey, int ttl, IList<FingerprintedData> fragments)
{
bool ret;
//prepare the global state
int max_enqueues = 1; //we only need a true/false answer
BlockingQueue bq_result = new BlockingQueue(max_enqueues);
AsyncPutFragsGlobalState global_state =
new AsyncPutFragsGlobalState(fragments.Count, fragInfo.BaseKey, bq_result);
//@TODO tweak the number of concurrency_degree here.
int concurrency_degree = Concurrency;
global_state.Concurrency = concurrency_degree;
//Put pieces
for (int index = 0; index < fragments.Count; index++) {
FingerprintedData fpd = fragments[index];
byte[] serializedFpd = fpd.SerializeTo();
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Size after serialization {0}", serializedFpd.Length));
byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index);
string piece_key_string = Encoding.UTF8.GetString(piece_key);
//piece state
AsyncOpState aps = new AsyncOpState(piece_key, serializedFpd, ttl);
AsyncFragsOpState apfs = new AsyncFragsOpState(global_state, aps);
//async put, one instance of IDht per put because thread safty not guaranteed
IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port);
BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations(
proxy);
// Fire the async XML-RPC call.
lock (global_state.SyncRoot) {
brdht_ops.BeginPutWithCallback(piece_key, serializedFpd, ttl,
new AsyncCallback(this.OnDhtPutReturns), apfs);
}
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Put piece {0} to DHT began (asynchronously)",
piece_key_string));
if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) ||
index == fragments.Count - 1) {
// Stop to wait for batch finish or all finish
global_state.BatchFinishedEvent.WaitOne();
if (concurrency_degree > 1) {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("Batch {0} finished. Moving on",
(int)((index + 1) / concurrency_degree)));
}
}
}
//block here until result comes
PutFragsStoppedEventArgs stop_args =
(PutFragsStoppedEventArgs)bq_result.Dequeue();
// Deal with info object.
if (stop_args.IsSuccessful) {
ret = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl);
if (ret) {
Logger.WriteLineIf(LogLevel.Verbose, _log_props,
string.Format("FragmentationInfo {0} successfully put",
Encoding.UTF8.GetString(infoKey)));
}
} else {
ret = false;
}
return ret;
}
