private bool PutFragsSequentially(FragmentationInfo fragInfo, byte[] infoKey, int ttl, IList <FingerprintedData> fragments) { string info_key_string = Encoding.UTF8.GetString(infoKey); //Put pieces int index = 0; foreach (FingerprintedData fpd in fragments) { byte[] serializedFpd = fpd.SerializeTo(); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Size after serialization {0}", serializedFpd.Length)); int i = index++; bool succ = false; int retries = 3; byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, i); string piece_key_string = Encoding.UTF8.GetString(piece_key); for (; !succ && retries > 0; retries--) { if (retries < 3) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Retrying...")); } succ = _dht.Put(piece_key, serializedFpd, ttl); if (!succ) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Put failed.")); } } if (retries <= 0) { Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Retries exhausted when putting piece : {0}", piece_key_string)); return(false); } else { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Piece {0} successfully put to DHT", piece_key_string)); } } //Put info bool succ_info = false; int retries_info = 3; for (; !succ_info && retries_info > 0; retries_info--) { if (retries_info < 3) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Retrying...")); } succ_info = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl); } if (retries_info <= 0) { Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Retries exhausted when putting FragmentationInfo : {0}", info_key_string)); return(false); } else { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("FragmentationInfo successfully put to DHT with name {0}", info_key_string)); } return(true); }
/// <summary> /// Concurrently puts pieces into DHT /// </summary> /// <returns>True if successful</returns> private bool PutFragsConcurrently(FragmentationInfo fragInfo, byte[] infoKey, int ttl, IList <FingerprintedData> fragments) { bool ret; //prepare the global state int max_enqueues = 1; //we only need a true/false answer BlockingQueue bq_result = new BlockingQueue(max_enqueues); AsyncPutFragsGlobalState global_state = new AsyncPutFragsGlobalState(fragments.Count, fragInfo.BaseKey, bq_result); //@TODO tweak the number of concurrency_degree here. int concurrency_degree = Concurrency; global_state.Concurrency = concurrency_degree; //Put pieces for (int index = 0; index < fragments.Count; index++) { FingerprintedData fpd = fragments[index]; byte[] serializedFpd = fpd.SerializeTo(); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Size after serialization {0}", serializedFpd.Length)); byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index); string piece_key_string = Encoding.UTF8.GetString(piece_key); //piece state AsyncOpState aps = new AsyncOpState(piece_key, serializedFpd, ttl); AsyncFragsOpState apfs = new AsyncFragsOpState(global_state, aps); //async put, one instance of IDht per put because thread safty not guaranteed IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port); BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations( proxy); // Fire the async XML-RPC call. lock (global_state.SyncRoot) { brdht_ops.BeginPutWithCallback(piece_key, serializedFpd, ttl, new AsyncCallback(this.OnDhtPutReturns), apfs); } Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Put piece {0} to DHT began (asynchronously)", piece_key_string)); if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) || index == fragments.Count - 1) { // Stop to wait for batch finish or all finish global_state.BatchFinishedEvent.WaitOne(); if (concurrency_degree > 1) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Batch {0} finished. Moving on", (int)((index + 1) / concurrency_degree))); } } } //block here until result comes PutFragsStoppedEventArgs stop_args = (PutFragsStoppedEventArgs)bq_result.Dequeue(); // Deal with info object. if (stop_args.IsSuccessful) { ret = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl); if (ret) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("FragmentationInfo {0} successfully put", Encoding.UTF8.GetString(infoKey))); } } else { ret = false; } return(ret); }
private bool PutFragsInBulk(FragmentationInfo fragInfo, byte[] infoKey, int ttl, IList <FingerprintedData> fragments) { string info_key_string = Encoding.UTF8.GetString(infoKey); XmlRpcStruct[] parameters = new XmlRpcStruct[fragments.Count]; // Prepare parameters for bulk put. int index = 0; foreach (FingerprintedData fpd in fragments) { byte[] serializedFpd = fpd.SerializeTo(); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Size of piece {0} after serialization: {1}", index, serializedFpd.Length)); byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index); XmlRpcStruct param_dict = new XmlRpcStruct(); param_dict.Add("name", piece_key); param_dict.Add("value", serializedFpd); param_dict.Add("ttl", ttl); parameters[index] = param_dict; index++; } bool[] results; try { results = _dht.BulkPut(parameters); } catch (Exception ex) { // Network related operation, log the exception and let it be handled // upper stream caller. Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Exception caught in BulkPut")); Logger.WriteLineIf(LogLevel.Error, _log_props, ex); throw; } IList <int> failed_pieces = new List <int>(); for (int i = 0; i < results.Length; i++) { bool result = results[i]; if (!result) { failed_pieces.Add(i); } } // Log failed pieces Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Failed {0} pieces are:", failed_pieces.Count)); StringBuilder sb = new StringBuilder(); foreach (int pieceIndx in failed_pieces) { sb.Append(pieceIndx); sb.Append(" "); } Logger.WriteLineIf(LogLevel.Verbose, _log_props, sb.ToString()); // Decide whether to retry. if (failed_pieces.Count > 20) { Logger.WriteLineIf(LogLevel.Info, _log_props, string.Format("Too many pieces failed. Returning...")); return(false); } //Retry failed pieces. foreach (int pieceIndx in failed_pieces) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Retrying piece {0}", pieceIndx)); bool succ_retries = PutWithRetries((byte[])parameters[pieceIndx]["name"], (byte[]) parameters[pieceIndx]["value"], (int)parameters[pieceIndx]["ttl"], 2); if (!succ_retries) { // Cannot succeed on this piece even after retry... return(false); } } //Put info bool succ_info = PutWithRetries(infoKey, fragInfo.SerializeTo(), ttl, 2); return(succ_info); }
private bool PutFragsSequentially(FragmentationInfo fragInfo, byte[] infoKey, int ttl, IList<FingerprintedData> fragments) { string info_key_string = Encoding.UTF8.GetString(infoKey); //Put pieces int index = 0; foreach (FingerprintedData fpd in fragments) { byte[] serializedFpd = fpd.SerializeTo(); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Size after serialization {0}", serializedFpd.Length)); int i = index++; bool succ = false; int retries = 3; byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, i); string piece_key_string = Encoding.UTF8.GetString(piece_key); for (; !succ && retries > 0; retries--) { if (retries < 3) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Retrying...")); } succ = _dht.Put(piece_key, serializedFpd, ttl); if (!succ) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Put failed.")); } } if (retries <= 0) { Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Retries exhausted when putting piece : {0}", piece_key_string)); return false; } else { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Piece {0} successfully put to DHT", piece_key_string)); } } //Put info bool succ_info = false; int retries_info = 3; for (; !succ_info && retries_info > 0; retries_info--) { if (retries_info < 3) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Retrying...")); } succ_info = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl); } if (retries_info <= 0) { Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Retries exhausted when putting FragmentationInfo : {0}", info_key_string)); return false; } else { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("FragmentationInfo successfully put to DHT with name {0}", info_key_string)); } return true; }
private bool PutFragsInBulk(FragmentationInfo fragInfo, byte[] infoKey, int ttl, IList<FingerprintedData> fragments) { string info_key_string = Encoding.UTF8.GetString(infoKey); XmlRpcStruct[] parameters = new XmlRpcStruct[fragments.Count]; // Prepare parameters for bulk put. int index = 0; foreach (FingerprintedData fpd in fragments) { byte[] serializedFpd = fpd.SerializeTo(); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Size of piece {0} after serialization: {1}", index, serializedFpd.Length)); byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index); XmlRpcStruct param_dict = new XmlRpcStruct(); param_dict.Add("name", piece_key); param_dict.Add("value", serializedFpd); param_dict.Add("ttl", ttl); parameters[index] = param_dict; index++; } bool[] results; try { results = _dht.BulkPut(parameters); } catch (Exception ex) { // Network related operation, log the exception and let it be handled // upper stream caller. Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Exception caught in BulkPut")); Logger.WriteLineIf(LogLevel.Error, _log_props, ex); throw; } IList<int> failed_pieces = new List<int>(); for (int i = 0; i < results.Length; i++) { bool result = results[i]; if (!result) { failed_pieces.Add(i); } } // Log failed pieces Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Failed {0} pieces are:", failed_pieces.Count)); StringBuilder sb = new StringBuilder(); foreach (int pieceIndx in failed_pieces) { sb.Append(pieceIndx); sb.Append(" "); } Logger.WriteLineIf(LogLevel.Verbose, _log_props, sb.ToString()); // Decide whether to retry. if (failed_pieces.Count > 20) { Logger.WriteLineIf(LogLevel.Info, _log_props, string.Format("Too many pieces failed. Returning...")); return false; } //Retry failed pieces. foreach (int pieceIndx in failed_pieces) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Retrying piece {0}", pieceIndx)); bool succ_retries = PutWithRetries((byte[])parameters[pieceIndx]["name"], (byte[]) parameters[pieceIndx]["value"], (int)parameters[pieceIndx]["ttl"], 2); if (!succ_retries) { // Cannot succeed on this piece even after retry... return false; } } //Put info bool succ_info = PutWithRetries(infoKey, fragInfo.SerializeTo(), ttl, 2); return succ_info; }
/// <summary> /// Concurrently puts pieces into DHT /// </summary> /// <returns>True if successful</returns> private bool PutFragsConcurrently(FragmentationInfo fragInfo, byte[] infoKey, int ttl, IList<FingerprintedData> fragments) { bool ret; //prepare the global state int max_enqueues = 1; //we only need a true/false answer BlockingQueue bq_result = new BlockingQueue(max_enqueues); AsyncPutFragsGlobalState global_state = new AsyncPutFragsGlobalState(fragments.Count, fragInfo.BaseKey, bq_result); //@TODO tweak the number of concurrency_degree here. int concurrency_degree = Concurrency; global_state.Concurrency = concurrency_degree; //Put pieces for (int index = 0; index < fragments.Count; index++) { FingerprintedData fpd = fragments[index]; byte[] serializedFpd = fpd.SerializeTo(); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Size after serialization {0}", serializedFpd.Length)); byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index); string piece_key_string = Encoding.UTF8.GetString(piece_key); //piece state AsyncOpState aps = new AsyncOpState(piece_key, serializedFpd, ttl); AsyncFragsOpState apfs = new AsyncFragsOpState(global_state, aps); //async put, one instance of IDht per put because thread safty not guaranteed IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port); BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations( proxy); // Fire the async XML-RPC call. lock (global_state.SyncRoot) { brdht_ops.BeginPutWithCallback(piece_key, serializedFpd, ttl, new AsyncCallback(this.OnDhtPutReturns), apfs); } Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Put piece {0} to DHT began (asynchronously)", piece_key_string)); if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) || index == fragments.Count - 1) { // Stop to wait for batch finish or all finish global_state.BatchFinishedEvent.WaitOne(); if (concurrency_degree > 1) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Batch {0} finished. Moving on", (int)((index + 1) / concurrency_degree))); } } } //block here until result comes PutFragsStoppedEventArgs stop_args = (PutFragsStoppedEventArgs)bq_result.Dequeue(); // Deal with info object. if (stop_args.IsSuccessful) { ret = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl); if (ret) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("FragmentationInfo {0} successfully put", Encoding.UTF8.GetString(infoKey))); } } else { ret = false; } return ret; }