/// <summary> /// Puts fragments to Dht. /// </summary> /// <param name="bde">The data object.</param> /// <param name="fragInfo">Object that contains the meta info.</param> /// <param name="concurrently">Puts concurrently if set to true</param> /// <returns>True if the operation succeeds.</returns> public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo, bool concurrently) { byte[] info_key = bde.Key; int ttl = bde.Ttl; //This ttl is used by every piece and the frag info MemBlock data = MemBlock.Reference(bde.Value); int piece_length = 0; int offset = 0; IList <FingerprintedData> fragments = new List <FingerprintedData>(); while (offset < data.Length) { piece_length = (offset + fragInfo.PieceLength > data.Length) ? data.Length - offset : fragInfo.PieceLength; MemBlock piece = data.Slice(offset, piece_length); offset += piece_length; FingerprintedData fpd = new FingerprintedData(new RegularData(piece)); fragments.Add(fpd); } //Update the piece number in fragInfo. Logger.WriteLineIf(LogLevel.Info, _log_props, string.Format("Data fragmented into {0} pieces", fragments.Count)); fragInfo.PieceNum = fragments.Count; #if FUSHARE_PF DateTime put_started = DateTime.UtcNow; #endif bool succ; if (!concurrently) { succ = PutFragsSequentially(fragInfo, info_key, ttl, fragments); } else { //succ = PutFragsConcurrently(fragInfo, info_key, ttl, fragments); succ = PutFragsInBulk(fragInfo, info_key, ttl, fragments); } #if FUSHARE_PF DateTime put_finished = DateTime.UtcNow; TimeSpan put_time = put_finished - put_started; Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Total time used to put {0}: {1} seconds", Encoding.UTF8.GetString(fragInfo.BaseKey), put_time)); #endif return(succ); }
/// <returns>The memblock assembled from pieces. Null if operation failed.</returns> private MemBlock GetFragsSequentially(byte[] base_key, int piece_num, out int largest_age, out int smallest_ttl) { largest_age = 0; smallest_ttl = Int32.MaxValue; MemBlock fragments = new MemBlock(); for (int i = 0; i < piece_num; i++) { byte[] piece_key = BuildFragmentKey(base_key, i); bool succ = false; //set to false iff bad things happen int retries = 3; //After that we fail the operation for (; !succ && retries > 0; retries--) { if (retries == 3) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Getting: {0}", piece_key)); } else { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Retrying...")); } try { DhtGetResult[] dgrs = _dht.Get(piece_key); //It should have only one entry. If not, just let the the exception caught //and retry. DhtGetResult dgr = dgrs[0]; FingerprintedData fpd = (FingerprintedData)DictionaryData.CreateDictionaryData(dgr.value); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Piece {0} retrieved and successfully parsed", piece_key)); RegularData rd = fpd.InnerData as RegularData; fragments = MemBlock.Concat(fragments, MemBlock.Reference(rd.PayLoad)); if (smallest_ttl > dgr.ttl) { smallest_ttl = dgr.ttl; } if (largest_age < dgr.age) { largest_age = dgr.age; } //Now it's safe to say, this attempt succeeded. succ = true; } catch (Exception ex) { Logger.WriteLineIf(LogLevel.Error, _log_props, ex); succ = false; } } //if succ then we are good if (retries <= 0) { //Quit because retries exhausted. throw new Exception(string.Format("Retries exhausted when retrieving " + "and deserializing piece : {0}", piece_key)); } else { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format( "Done with piece {0}", Encoding.UTF8.GetString(piece_key))); } } return(fragments); }
/// <summary> /// Concurrently puts pieces into DHT /// </summary> /// <returns>True if successful</returns> private bool PutFragsConcurrently(FragmentationInfo fragInfo, byte[] infoKey, int ttl, IList <FingerprintedData> fragments) { bool ret; //prepare the global state int max_enqueues = 1; //we only need a true/false answer BlockingQueue bq_result = new BlockingQueue(max_enqueues); AsyncPutFragsGlobalState global_state = new AsyncPutFragsGlobalState(fragments.Count, fragInfo.BaseKey, bq_result); //@TODO tweak the number of concurrency_degree here. int concurrency_degree = Concurrency; global_state.Concurrency = concurrency_degree; //Put pieces for (int index = 0; index < fragments.Count; index++) { FingerprintedData fpd = fragments[index]; byte[] serializedFpd = fpd.SerializeTo(); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Size after serialization {0}", serializedFpd.Length)); byte[] piece_key = BuildFragmentKey(fragInfo.BaseKey, index); string piece_key_string = Encoding.UTF8.GetString(piece_key); //piece state AsyncOpState aps = new AsyncOpState(piece_key, serializedFpd, ttl); AsyncFragsOpState apfs = new AsyncFragsOpState(global_state, aps); //async put, one instance of IDht per put because thread safty not guaranteed IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port); BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations( proxy); // Fire the async XML-RPC call. lock (global_state.SyncRoot) { brdht_ops.BeginPutWithCallback(piece_key, serializedFpd, ttl, new AsyncCallback(this.OnDhtPutReturns), apfs); } Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Put piece {0} to DHT began (asynchronously)", piece_key_string)); if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) || index == fragments.Count - 1) { // Stop to wait for batch finish or all finish global_state.BatchFinishedEvent.WaitOne(); if (concurrency_degree > 1) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Batch {0} finished. Moving on", (int)((index + 1) / concurrency_degree))); } } } //block here until result comes PutFragsStoppedEventArgs stop_args = (PutFragsStoppedEventArgs)bq_result.Dequeue(); // Deal with info object. if (stop_args.IsSuccessful) { ret = _dht.Put(infoKey, fragInfo.SerializeTo(), ttl); if (ret) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("FragmentationInfo {0} successfully put", Encoding.UTF8.GetString(infoKey))); } } else { ret = false; } return(ret); }
/// <summary> /// Concurrenly gets the fragments from DHT. /// </summary> /// <returns>The fragments</returns> private MemBlock GetFragsConcurrently(byte[] baseKey, int pieceNum, out int largestAge, out int smallestTtl) { // First set the int values to be invalid. smallestTtl = Int32.MaxValue; largestAge = 0; int max_enqueues = 1; BlockingQueue bq_result = new BlockingQueue(max_enqueues); AsyncGetFragsGlobalState global_state = new AsyncGetFragsGlobalState(pieceNum, baseKey, bq_result); int concurrency_degree = Concurrency; global_state.Concurrency = concurrency_degree; for (int i = 0; i < pieceNum; i++) { int index = i; byte[] piece_key = BuildFragmentKey(baseKey, i); string piece_key_string = Encoding.UTF8.GetString(piece_key); // piece state AsyncOpState aps = new AsyncOpState(piece_key); AsyncFragsOpState afos = new AsyncFragsOpState(global_state, aps); // async get, one instance of IDht per get because thread safty not guaranteed IDht proxy = DhtServiceClient.GetXmlRpcDhtClient(_svc_uri.Port); BrunetDhtClientOperations brdht_ops = new BrunetDhtClientOperations( proxy); lock (global_state.SyncRoot) { brdht_ops.BeginGetWithCallback(piece_key, new AsyncCallback(this.OnDhtGetReturns), afos); } Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Get piece {0} from DHT began (asynchronously)", piece_key_string)); if ((concurrency_degree > 1 && (index + 1) % concurrency_degree == 0) || index == pieceNum - 1) { // Stop to wait for batch finish or all finish global_state.BatchFinishedEvent.WaitOne(); if (concurrency_degree > 1) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Batch {0} finished. Moving on", (int)((index + 1) / concurrency_degree))); } } } //block here until result comes GetFragsStoppedEventArgs stop_args = (GetFragsStoppedEventArgs)bq_result.Dequeue(); //All results have returned. MemBlock ret = new MemBlock(); if (stop_args.IsSuccessful) { // We successfully got everything but aren't sure whether they are // correct. Now parse them. for (int i = 0; i < global_state.Fragments.Length; i++) { try { DhtGetResult dgr = global_state.Fragments[i]; FingerprintedData fpd = (FingerprintedData)DictionaryData.CreateDictionaryData(dgr.value); RegularData rd = fpd.InnerData as RegularData; // This piece is OK. ret = MemBlock.Concat(ret, MemBlock.Reference(rd.PayLoad)); if (smallestTtl > dgr.ttl) { smallestTtl = dgr.ttl; } if (largestAge < dgr.age) { largestAge = dgr.age; } //Now it's safe to say, this attempt succeeded. } catch (Exception) { Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Parsing Piece failed at index {0}", i)); throw; } } return(ret); } return(null); }
/// <summary> /// Puts fragments to Dht. /// </summary> /// <param name="bde">The data object.</param> /// <param name="fragInfo">Object that contains the meta info.</param> /// <param name="concurrently">Puts concurrently if set to true</param> /// <returns>True if the operation succeeds.</returns> public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo, bool concurrently) { byte[] info_key = bde.Key; int ttl = bde.Ttl; //This ttl is used by every piece and the frag info MemBlock data = MemBlock.Reference(bde.Value); int piece_length = 0; int offset = 0; IList<FingerprintedData> fragments = new List<FingerprintedData>(); while (offset < data.Length) { piece_length = (offset + fragInfo.PieceLength > data.Length) ? data.Length - offset : fragInfo.PieceLength; MemBlock piece = data.Slice(offset, piece_length); offset += piece_length; FingerprintedData fpd = new FingerprintedData(new RegularData(piece)); fragments.Add(fpd); } //Update the piece number in fragInfo. Logger.WriteLineIf(LogLevel.Info, _log_props, string.Format("Data fragmented into {0} pieces", fragments.Count)); fragInfo.PieceNum = fragments.Count; #if FUSHARE_PF DateTime put_started = DateTime.UtcNow; #endif bool succ; if (!concurrently) { succ = PutFragsSequentially(fragInfo, info_key, ttl, fragments); } else { //succ = PutFragsConcurrently(fragInfo, info_key, ttl, fragments); succ = PutFragsInBulk(fragInfo, info_key, ttl, fragments); } #if FUSHARE_PF DateTime put_finished = DateTime.UtcNow; TimeSpan put_time = put_finished - put_started; Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Total time used to put {0}: {1} seconds", Encoding.UTF8.GetString(fragInfo.BaseKey), put_time)); #endif return succ; }