/// <summary> /// Put with a size limit. Data larger than the limit will be splitted into /// pieces and put to different under different DHT keys. /// </summary> /// <param name="maxSize">The size limit</param> /// <param name="split">Whether to split the oversized data</param> /// <exception cref="ArgumentException">Data larger than the limit and /// split specified as false</exception> public bool Put(byte[] key, byte[] value, int ttl, int maxSize, bool split) { bool result; if (value.Length > maxSize) { if (split) { BrunetDhtEntry bde = new BrunetDhtEntry(key, value, ttl); FragmentationInfo frag_info = new FragmentationInfo(key); frag_info.PieceLength = maxSize; result = PutFragments(bde, frag_info); } else { throw new ArgumentException("Data too large but split opted out."); } } else { result = _dht.Put(key, value, ttl); } return(result); }
/// <summary> /// Gets the first data item of the given name and possibly gets the /// indicated pieces from DHT. /// </summary> /// <param name="getPieces">Whether to get pieces if the data at the /// specified DHT name is a FragmentationInfo</param> public DhtGetResult GetOneDatum(byte[] key, bool getPieces, OneDatumMode mode) { DhtGetResult[] results = _dht.Get(key); DhtGetResult ret; if (results.Length == 0) { ret = null; } else { DhtGetResult dgr; if (mode == OneDatumMode.FirstOne) { dgr = results[0]; } else if (mode == OneDatumMode.LastOne) { dgr = results[results.Length - 1]; } else { throw new NotImplementedException( "This OneDatumMode not implemented."); } DictionaryData dd = null; try { dd = DictionaryData.CreateDictionaryData(dgr.value); } catch (Exception ex) { // Not an error in this case. Log with verbose level. Logger.WriteLineIf(LogLevel.Verbose, _log_props, ex); } if (dd != null && dd is FragmentationInfo) { FragmentationInfo frag_info = dd as FragmentationInfo; BrunetDhtEntry bde = null; try { bde = GetFragments(frag_info) as BrunetDhtEntry; RegularData rd = (RegularData)DictionaryData.CreateDictionaryData( bde.Value); //Only 1 entry (if any) in this array ret = new DhtGetResult(rd.PayLoad, bde.Age, bde.Ttl); } catch (Exception ex) { Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("Can't get fragments."), ex); ret = null; } } else { ret = dgr; } } return(ret); }
/// <exception cref="Exception">Operation Failed</exception> /// <returns>A BrunetDhtEntry that has the whole chunk of data</returns> public DictionaryData GetFragments(FragmentationInfo info, bool concurrently) { BrunetDhtEntry ret = null; byte[] base_key = info.BaseKey; int piece_num = info.PieceNum; MemBlock fragments; int largest_age; int smallest_ttl; /* * @TODO: It doesn't have to be sequential but let's first get it work with * this approach */ #if FUSHARE_PF DateTime get_started = DateTime.UtcNow; #endif if (!concurrently) { fragments = GetFragsSequentially(base_key, piece_num, out largest_age, out smallest_ttl); } else { //fragments = GetFragsConcurrently(base_key, piece_num, out largest_age, out smallest_ttl); fragments = GetFragsInBulk(base_key, piece_num, out largest_age, out smallest_ttl); } #if FUSHARE_PF DateTime get_finished = DateTime.UtcNow; TimeSpan get_time = get_finished - get_started; Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Total time used to get {0}: {1} seconds", Encoding.UTF8.GetString(base_key), get_time)); #endif if (fragments != null) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Fragments successfully got from DHT")); //Got the fragments correctly ret = new BrunetDhtEntry(base_key, (new RegularData(fragments)).SerializeTo(), largest_age, smallest_ttl); return(ret); } else { Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("DHT Get of fragments failed")); throw new Exception("DHT Get of fragments failed."); } }
/// <summary> /// Puts fragments to Dht. /// </summary> /// <param name="bde">The data object.</param> /// <param name="fragInfo">Object that contains the meta info.</param> /// <param name="concurrently">Puts concurrently if set to true</param> /// <returns>True if the operation succeeds.</returns> public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo, bool concurrently) { byte[] info_key = bde.Key; int ttl = bde.Ttl; //This ttl is used by every piece and the frag info MemBlock data = MemBlock.Reference(bde.Value); int piece_length = 0; int offset = 0; IList <FingerprintedData> fragments = new List <FingerprintedData>(); while (offset < data.Length) { piece_length = (offset + fragInfo.PieceLength > data.Length) ? data.Length - offset : fragInfo.PieceLength; MemBlock piece = data.Slice(offset, piece_length); offset += piece_length; FingerprintedData fpd = new FingerprintedData(new RegularData(piece)); fragments.Add(fpd); } //Update the piece number in fragInfo. Logger.WriteLineIf(LogLevel.Info, _log_props, string.Format("Data fragmented into {0} pieces", fragments.Count)); fragInfo.PieceNum = fragments.Count; #if FUSHARE_PF DateTime put_started = DateTime.UtcNow; #endif bool succ; if (!concurrently) { succ = PutFragsSequentially(fragInfo, info_key, ttl, fragments); } else { //succ = PutFragsConcurrently(fragInfo, info_key, ttl, fragments); succ = PutFragsInBulk(fragInfo, info_key, ttl, fragments); } #if FUSHARE_PF DateTime put_finished = DateTime.UtcNow; TimeSpan put_time = put_finished - put_started; Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Total time used to put {0}: {1} seconds", Encoding.UTF8.GetString(fragInfo.BaseKey), put_time)); #endif return(succ); }
public void TestPutAndGetFragments() { MockBrunetDht mock_dht = new MockBrunetDht(); BrunetDht dht = new BrunetDht(mock_dht); FragmentationInfo frag_info = new FragmentationInfo(); byte[] b_key = MakeByteArray(20); string key = Encoding.UTF8.GetString(b_key); frag_info.BaseKey = Encoding.UTF8.GetString(b_key); Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("key : {0}", Base32.Encode(b_key))); byte[] expected = MakeByteArray(1024 * 10); int ttl = 1000; BrunetDhtEntry bde = new BrunetDhtEntry(key, expected, ttl); dht.PutFragments(bde, frag_info); Hashtable ht = mock_dht.HTStorage; Assert.AreEqual(11, ht.Count); //// byte[] serialized_fragInfo = ((DhtGetResult[])dht.Get(key))[0].value; FragmentationInfo frag_info_actual = DictionaryData.CreateDictionaryData(serialized_fragInfo) as FragmentationInfo; Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("BaseKey: {0}", Base32.Encode(Encoding.UTF8.GetBytes(frag_info_actual.BaseKey)))); BrunetDhtEntry actual = dht.GetFragments(frag_info) as BrunetDhtEntry; MemBlock mb = MemBlock.Reference(actual.Value); MemBlock mb_exp = MemBlock.Reference(expected); Logger.WriteLineIf(LogLevel.Verbose, _log_props, mb.ToBase32String().Substring(0, 50)); Logger.WriteLineIf(LogLevel.Verbose, _log_props, mb_exp.ToBase32String().Substring(0, 50)); Assert.AreEqual(mb_exp, mb); }
/// <summary> /// Puts fragments and chooses the concurrent option /// </summary> public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo) { return(this.PutFragments(bde, fragInfo, true)); }
/// <summary> /// Puts fragments to Dht. /// </summary> /// <param name="bde">The data object.</param> /// <param name="fragInfo">Object that contains the meta info.</param> /// <param name="concurrently">Puts concurrently if set to true</param> /// <returns>True if the operation succeeds.</returns> public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo, bool concurrently) { byte[] info_key = bde.Key; int ttl = bde.Ttl; //This ttl is used by every piece and the frag info MemBlock data = MemBlock.Reference(bde.Value); int piece_length = 0; int offset = 0; IList<FingerprintedData> fragments = new List<FingerprintedData>(); while (offset < data.Length) { piece_length = (offset + fragInfo.PieceLength > data.Length) ? data.Length - offset : fragInfo.PieceLength; MemBlock piece = data.Slice(offset, piece_length); offset += piece_length; FingerprintedData fpd = new FingerprintedData(new RegularData(piece)); fragments.Add(fpd); } //Update the piece number in fragInfo. Logger.WriteLineIf(LogLevel.Info, _log_props, string.Format("Data fragmented into {0} pieces", fragments.Count)); fragInfo.PieceNum = fragments.Count; #if FUSHARE_PF DateTime put_started = DateTime.UtcNow; #endif bool succ; if (!concurrently) { succ = PutFragsSequentially(fragInfo, info_key, ttl, fragments); } else { //succ = PutFragsConcurrently(fragInfo, info_key, ttl, fragments); succ = PutFragsInBulk(fragInfo, info_key, ttl, fragments); } #if FUSHARE_PF DateTime put_finished = DateTime.UtcNow; TimeSpan put_time = put_finished - put_started; Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Total time used to put {0}: {1} seconds", Encoding.UTF8.GetString(fragInfo.BaseKey), put_time)); #endif return succ; }
/// <summary> /// Puts fragments and chooses the concurrent option /// </summary> public bool PutFragments(BrunetDhtEntry bde, FragmentationInfo fragInfo) { return this.PutFragments(bde, fragInfo, true); }
/// <summary> /// Put with a size limit. Data larger than the limit will be splitted into /// pieces and put to different under different DHT keys. /// </summary> /// <param name="maxSize">The size limit</param> /// <param name="split">Whether to split the oversized data</param> /// <exception cref="ArgumentException">Data larger than the limit and /// split specified as false</exception> public bool Put(byte[] key, byte[] value, int ttl, int maxSize, bool split) { bool result; if (value.Length > maxSize) { if (split) { BrunetDhtEntry bde = new BrunetDhtEntry(key, value, ttl); FragmentationInfo frag_info = new FragmentationInfo(key); frag_info.PieceLength = maxSize; result = PutFragments(bde, frag_info); } else { throw new ArgumentException("Data too large but split opted out."); } } else { result = _dht.Put(key, value, ttl); } return result; }
/// <exception cref="Exception">Operation Failed</exception> /// <returns>A BrunetDhtEntry that has the whole chunk of data</returns> public DictionaryData GetFragments(FragmentationInfo info, bool concurrently) { BrunetDhtEntry ret = null; byte[] base_key = info.BaseKey; int piece_num = info.PieceNum; MemBlock fragments; int largest_age; int smallest_ttl; /* * @TODO: It doesn't have to be sequential but let's first get it work with * this approach */ #if FUSHARE_PF DateTime get_started = DateTime.UtcNow; #endif if (!concurrently) { fragments = GetFragsSequentially(base_key, piece_num, out largest_age, out smallest_ttl); } else { //fragments = GetFragsConcurrently(base_key, piece_num, out largest_age, out smallest_ttl); fragments = GetFragsInBulk(base_key, piece_num, out largest_age, out smallest_ttl); } #if FUSHARE_PF DateTime get_finished = DateTime.UtcNow; TimeSpan get_time = get_finished - get_started; Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Total time used to get {0}: {1} seconds", Encoding.UTF8.GetString(base_key), get_time)); #endif if (fragments != null) { Logger.WriteLineIf(LogLevel.Verbose, _log_props, string.Format("Fragments successfully got from DHT")); //Got the fragments correctly ret = new BrunetDhtEntry(base_key, (new RegularData(fragments)).SerializeTo(), largest_age, smallest_ttl); return ret; } else { Logger.WriteLineIf(LogLevel.Error, _log_props, string.Format("DHT Get of fragments failed")); throw new Exception("DHT Get of fragments failed."); } }