public void OnNext(LookupMessage msg) { if (Util.GetLocalFileChunks().Any(entry => entry.FileId == msg.FileId)) { Core.Instance.MCChannel.Send(new GotMessage(msg.FileId)); } }
private static void CheckChunks() { var chunks = Util.GetLocalFileChunks(); var chunkList = new List <KeyValuePair <FileChunk, Tuple <int, int> > >(); foreach (var c in chunks) { int wantedDegree, actualDegree; if (!Core.Instance.ChunkPeers.TryGetDegrees(c, out wantedDegree, out actualDegree)) { Core.Instance.Log.ErrorFormat("SpaceReclaimingProtocol: Could not get degrees for {0}", c); continue; } if (actualDegree < wantedDegree) { chunkList.Add(new KeyValuePair <FileChunk, Tuple <int, int> >(c, Tuple.Create(wantedDegree, actualDegree))); } } chunkList.Sort((pair1, pair2) => { var delta1 = Math.Abs(pair1.Value.Item1 - pair1.Value.Item2); var delta2 = Math.Abs(pair2.Value.Item1 - pair2.Value.Item2); if (delta1 > delta2) { return(-1); } if (delta1 < delta2) { return(1); } return(0); }); foreach (var chunk in chunkList) { try { if (!chunk.Key.Exists()) { Core.Instance.Log.ErrorFormat( "EnhancedSpaceReclaimingWatcher: Could not start BackupChunkProtocol" + " for {0} because it no longer exists here.", chunk.Key); Core.Instance.MCChannel.Send(new RemovedMessage(chunk.Key)); // we were supposed to have the file chunk continue; } new BackupChunkSubprotocol(chunk.Key, chunk.Value.Item1, chunk.Key.GetData()).Run().Wait(); } catch (Exception ex) { Core.Instance.Log.Error("CheckChunks", ex); } } }
/// <summary> /// Space reclaiming "algorithm": find all chunks whose actual replication degree is /// higher than the required application degree and delete them /// </summary> private void SimpleReclaimer() { var chunks = Util.GetLocalFileChunks(); foreach (var fc in chunks) { int wantedDegree, actualDegree; if (!Core.Instance.ChunkPeers.TryGetDegrees(fc, out wantedDegree, out actualDegree)) { Core.Instance.Log.ErrorFormat("SpaceReclaimingProtocol: Could not get degrees for {0}", fc); continue; } if (actualDegree > wantedDegree) { Core.Instance.Log.InfoFormat(@"SpaceReclaimingProtocol: Deleting chunk {0} because degree {1} is higher than the desired {2}", fc, actualDegree, wantedDegree); DeleteChunk(fc); } } }
public void OnNext(PutChunkMessage msg) { var fileChunk = new FileChunk(msg.FileId, msg.ChunkNo); Core.Instance.ChunkPeers.SetWantedReplicationDegree(fileChunk, msg.ReplicationDeg); var dirSize = Util.GetDirectorySize(Core.Instance.Config.BackupDirectory); if (dirSize + msg.Body.Length > Core.Instance.Config.MaxBackupSize) { Core.Instance.Log.InfoFormat( "BackupChunkService:OnNext: Got no space to store {0}, trying to evict some other chunks", fileChunk); new SpaceReclaimingProtocol(false).Run().Wait(); dirSize = Util.GetDirectorySize(Core.Instance.Config.BackupDirectory); if (dirSize + msg.Body.Length > Core.Instance.Config.MaxBackupSize) { Core.Instance.Log.InfoFormat( "BackupChunkService:OnNext: Really have no space to store any file. Giving up on storing {0}", fileChunk); return; } } var stored = fileChunk.SetData(msg.Body); if (!stored.HasValue) { Core.Instance.Log.ErrorFormat("BackupChunkService: Could not store file {0}", fileChunk); return; } // otherwise file is already created Task.Delay(Core.Instance.RandomDelay).Wait(); Core.Instance.MCChannel.Send(new StoredMessage(fileChunk)); }
private void ForcedReclaimer() { var directorySize = Utilities.Utilities.GetDirectorySize(Core.Instance.Config.BackupDirectory); var sizeLimit = Core.Instance.Config.MaxBackupSize; if (directorySize < sizeLimit) { Core.Instance.Log.Info("ForcedReclaimer: directory size is less than limit, no cleaning up to do."); return; } var chunks = Util.GetLocalFileChunks() .Select(fc => { long size; try { size = new FileInfo(fc.FullFileName).Length; } catch (Exception ex) { size = 0; Core.Instance.Log.Error("ForcedReclaimer", ex); } int wantedDegree, actualDegree; return(!Core.Instance.ChunkPeers.TryGetDegrees(fc, out wantedDegree, out actualDegree) ? null : Tuple.Create(fc, Tuple.Create(wantedDegree, actualDegree, size))); }).ToList(); if (chunks.Count == 0) { Core.Instance.Log.Info("SpaceReclaimingProtocol:ForcedReclaimer: empty dir, finished"); return; } chunks.Sort((pair1, pair2) => { if (pair1 == null || pair2 == null) { return(0); } var delta1 = Math.Abs(pair1.Item2.Item1 - pair1.Item2.Item2); // 1 - wanted, 2 - actual var delta2 = Math.Abs(pair2.Item2.Item1 - pair2.Item2.Item2); if (delta1 > delta2) { return(-1); } if (delta1 < delta2) { return(1); } return(0); }); var queue = new Queue <Tuple <FileChunk, Tuple <int, int, long> > >(chunks); while (directorySize > sizeLimit && queue.Count > 0) { var pair = queue.Dequeue(); if (pair == null) { continue; } if (!DeleteChunk(pair.Item1)) { continue; } directorySize -= pair.Item2.Item3; Core.Instance.Log.InfoFormat("ForcedReclaimer: decreased dir size by {0} to {1}. limit is {2}", pair.Item2.Item3, directorySize, sizeLimit); } }