// splits a big file ( size > maxchunkfile) into multiple chunks private List <BChunk> GetBigFileChunks(IFSEntry bigFile, long filePosInChunk) { long pos = 0; long remaining = bigFile.FileSize; List <BChunk> chunks = new List <BChunk>(); while (remaining > 0) { chunkOrder++; IFSEntry f = bigFile.Clone(); BChunk chunk = new BChunk(this.TaskId); chunk.Order = chunkOrder; chunk.RootDriveId = this.backupRootDrive.ID; f.ChunkStartPos = 0; chunk.Add(f); chunks.Add(chunk); f.FileStartPos = pos; if (remaining > maxChunkSize) { pos += maxChunkSize; } else { pos += remaining; } remaining = bigFile.FileSize - pos; //if(remaining <0) Logger.Append(Severity.TRIVIA, "GetNextChunk() : splitted file " + f.SnapFullPath + " (size " + f.FileSize + ") , remaining=" + remaining + " to chunk - " + chunk.Name + " starting @ offset " + f.FileStartPos); } return(chunks); }
private BChunk BuildIndexChunk() { backup.AddHubNotificationEvent(704, "", ""); BChunk iChunk = new BChunk(/*backup.TaskId, */ backup.Index.FullName, backup.Index.FullName, backup.TaskId); //string name, int bsid, string bPath, string snapPath) try{ //iChunk.Add(FileProvider.GetFile(index.FullName)); iChunk.Add(new MinimalFsItem(backup.Index.FullName)); if (backup.DataFlags.HasFlag(DataProcessingFlags.CDedup)) // backup the deduplication database { iChunk.Add(ItemProvider.GetProvider().GetItemByPath(DedupIndex.Instance().IndexDBName)); } /*string sumHash; * using(FileStream cksumFS = new FileStream(backup.Index.FullName, FileMode.Open, FileAccess.Read)){ * sumHash = BitConverter.ToString(SHA1.Create().ComputeHash(cksumFS)); * iChunk.Sum = sumHash; * }*/ iChunk.Sum = IndexManager.CheckSumIndex(backup.TaskId, (backup.Level != BackupLevel.Full)); // register for session received, to process index transfer User.StorageSessionReceivedEvent += new User.StorageSessionReceivedHandler(this.SendIndex); User.AskIndexDest(backup.TaskId, backup.Index.Name, iChunk.Sum); Logger.Append(Severity.DEBUG, "Asked index destination to hub"); return(iChunk); } catch (Exception e) { Logger.Append(Severity.ERROR, "Couldn't checksum index and/or ask destination to hub: " + e.Message + "---" + e.StackTrace); backup.AddHubNotificationEvent(808, e.Message, ""); } return(null); }
private void DoIndex() { while (!indexerChunksFeed.IsCompleted || !cancellationTokenSource.IsCancellationRequested) { try{ BChunk toBeIndexed = indexerChunksFeed.Take(cancellationTokenSource.Token); backup.Index.AddChunk(toBeIndexed); Logger.Append(Severity.DEBUG2, "Added chunk " + toBeIndexed.Name + " to index"); } catch (Exception e) { if (e is OperationCanceledException) { Logger.Append(Severity.DEBUG2, "Indexer has been manually cancelled on purpose, stopping..."); } else if (e is InvalidOperationException) { Logger.Append(Severity.DEBUG2, "Indexer : no more chunks to index"); } else { Console.WriteLine("////// unexpected DoIndex exception : " + e.ToString()); throw; } return; } } }
private BChunk BuildIndexChunk() { backup.AddHubNotificationEvent(704, "", ""); BChunk iChunk = null; string synthIndexFullPath = null; if (backup.Level != P2PBackup.Common.BackupLevel.Full && backup.Level != P2PBackup.Common.BackupLevel.SnapshotOnly) { IndexManager idxManager = new IndexManager(); Logger.Append(Severity.INFO, "Building synthetic full index..."); synthIndexFullPath = idxManager.CreateSyntheticFullIndex(backup.ParentTrackingId, backup.Id, backup.RootDrives); backup.AddHubNotificationEvent(707, "", ""); backup.SyntheticIndexSum = IndexManager.CheckSumIndex(backup.Id, false); // for synthetic backups } if (backup.Level == BackupLevel.Refresh) // backup the synth index { iChunk = new BChunk(backup.Id); iChunk.Add(new MinimalFsItem(synthIndexFullPath)); // minimalitem because we only care the data, not the rest } else // only backup the partial index { iChunk = new BChunk(backup.Id); //string name, int bsid, string bPath, string snapPath) iChunk.Add(new MinimalFsItem(backup.Index.FullName)); // minimalitem because we only care the data, not the rest } try{ //iChunk.Add(FileProvider.GetFile(index.FullName)); if (backup.Level != BackupLevel.SnapshotOnly && backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CDedup)) // backup the deduplication database { iChunk.Add(ItemProvider.GetProvider().GetItemByPath(dedupIndex.IndexDBName)); backup.DdbSum = dedupIndex.ChecksumDdb(); } iChunk.Sum = IndexManager.CheckSumIndex(backup.Id, (backup.Level == BackupLevel.Refresh)); if (backup.Level == BackupLevel.Full) { backup.SyntheticIndexSum = iChunk.Sum; // for Fulls } // stop waiting for sessions used for regular data transfer... //User.SessionReady -= this.SessionReceived; // ...but re-register for session that will transfer the index //User.SessionReady += this.SendIndex; StorageNeeded(new PeerSession { TaskId = backup.Id }, 1, true, false); Logger.Append(Severity.DEBUG, "Asked index destination to hub"); return(iChunk); } catch (Exception e) { Logger.Append(Severity.ERROR, "Couldn't checksum index and/or ask destination to hub: " + e.Message + "---" + e.StackTrace); backup.AddHubNotificationEvent(808, e.Message, ""); } return(null); }
/// <summary> /// One 'Produce' task generates chunks for one BackupRootDrive (ie 1 mountpoint/filesystem). /// </summary> /// <param name='bdr'> /// the BackupRootDrive to scan for items /// </param> private void Produce(Queue <BackupRootDrive> queue /*, BlockingCollection<BChunk> myFeed*/) { BlockingCollection <BChunk> myFeed = new BlockingCollection <BChunk>(new ConcurrentQueue <BChunk>(), 1); Console.WriteLine(" ------- Producer() has " + queue.Count + " drive items in its queue"); chunksFeeds.Add(myFeed); //IEnumerator<BChunk> chunkEnumerator = bdh.GetNextChunk().GetEnumerator();//backup.GetNextChunk().GetEnumerator(); while (queue.Count > 0) { BackupRootDrive bdr = queue.Dequeue(); Logger.Append(Severity.INFO, "Collecting items to backup for drive " + bdr.SystemDrive.MountPoint); BackupRootDriveHandler bdh = new BackupRootDriveHandler(bdr, this.backup.Id, backup.BackupSet.MaxChunkSize, backup.BackupSet.MaxChunkSize, backup.BackupSet.MaxChunkFiles, backup.Level, backup.RefStartDate, backup.RefEndDate, backup.ParentTrackingId); bdh.LogEvent += LogReceived; bdh.SubCompletionEvent += new BackupRootDriveHandler.SubCompletionHandler(IncrementSubCompletion); foreach (P2PBackup.Common.BasePath baseP in bdr.Paths) { bdh.SetCurrentPath(baseP); IEnumerator <BChunk> chunkEnumerator = bdh.GetNextChunk().GetEnumerator(); while (chunkEnumerator.MoveNext() && !cancellationTokenSource.IsCancellationRequested) { BChunk chunk = chunkEnumerator.Current; try{ myFeed.Add(chunk, cancellationTokenSource.Token); } catch (OperationCanceledException) { Logger.Append(Severity.TRIVIA, "Producer has been manually cancelled on purpose, stopping..."); return; } catch (Exception e) { Logger.Append(Severity.ERROR, "###################### Produce() : add refused : " + e.Message + " ---- " + e.StackTrace); return; } // stats foreach (IFSEntry item in chunk.Items) { backup.ItemsByType[(int)item.Kind]++; } Logger.Append(Severity.DEBUG, "Basepath " + baseP.Path + " : Added chunk " + chunk.Name + " containing " + chunk.Items.Count + " items "); } } bdh.SubCompletionEvent -= new BackupRootDriveHandler.SubCompletionHandler(IncrementSubCompletion); bdh.LogEvent -= LogReceived; if (!cancellationTokenSource.IsCancellationRequested) { ContinueProducing(); } else { bdh.Dispose(); } } Console.WriteLine("------------------------- PRODUCE(): done collecting ALL, complete feed adding, cancellationTokenSource.IsCancellationRequested=" + cancellationTokenSource.IsCancellationRequested); myFeed.CompleteAdding(); }
/// <summary> /// If file is recovery-file is received SendRecoveryToHub is called /// </summary> /// <param name="received">true if file has been received</param> /// <param name="bFReceived">the file that has been received</param> private void FileReceived(bool received, BChunk chunk) { /*if (received == true){ * foreach (BChunk bc in backup.Chunks){ * if(bc.Name == chunk.Name){ * bc.Fetched = true; * UpdateGUIEvent("prbRecovery", bc.Size.ToString()); * SendRecoveryToHub(); * } * } * }*/ }
private void RemoveChunk(BChunk chunkToRemove) { lock (processingChunks){ for (int i = processingChunks.Count - 1; i >= 0; i--) { if (processingChunks[i].Name == chunkToRemove.Name) { processingChunks.RemoveAt(i); } } } }
/// <summary> /// One 'Produce' task generates chunks for one BackupRootDrive (ie 1 mountpoint). /// </summary> /// <param name='bdr'> /// the BackupRootDrive to scan for items /// </param> private void Produce(BackupRootDrive bdr) { Logger.Append(Severity.INFO, "Collecting items to backup for drive " + bdr.SystemDrive.MountPoint); BackupRootDriveHandler bdh = new BackupRootDriveHandler(bdr, this.backup.TaskId, backup.MaxChunkSize, backup.MaxChunkSize, backup.MaxChunkFiles, backup.Level, backup.RefStartDate, backup.RefEndDate, backup.RefTaskId); bdh.LogEvent += LogReceived; bdh.SubCompletionEvent += new BackupRootDriveHandler.SubCompletionHandler(IncrementSubCompletion); //IEnumerator<BChunk> chunkEnumerator = bdh.GetNextChunk().GetEnumerator();//backup.GetNextChunk().GetEnumerator(); foreach (P2PBackup.Common.BasePath baseP in bdr.Paths) { bdh.SetCurrentPath(baseP); IEnumerator <BChunk> chunkEnumerator = bdh.GetNextChunk().GetEnumerator(); while (chunkEnumerator.MoveNext() && !cancellationTokenSource.IsCancellationRequested) { BChunk chunk = chunkEnumerator.Current; try{ chunkBuilderFeed.Add(chunk, cancellationTokenSource.Token); } catch (OperationCanceledException) { Logger.Append(Severity.DEBUG2, "Producer has been manually cancelled on purpose, stopping..."); return; } catch (Exception e) { Logger.Append(Severity.ERROR, "###################### Produce() : add refused : " + e.Message + " ---- " + e.StackTrace); return; } // stats foreach (IFSEntry item in chunk.Files) { backup.ItemsByType[(int)item.Kind]++; } Logger.Append(Severity.DEBUG, "Added chunk " + chunk.Name + " containing " + chunk.Files.Count + " items "); } } bdh.SubCompletionEvent -= new BackupRootDriveHandler.SubCompletionHandler(IncrementSubCompletion); bdh.LogEvent -= LogReceived; Logger.Append(Severity.INFO, "Producer has done collecting items to backup for drive " + bdr.SystemDrive.MountPoint); if (!cancellationTokenSource.IsCancellationRequested) { ContinueProducing(); } else { bdh.Dispose(); } }
private void ProcessIndex() { //if(cancellationTokenSource.IsCancellationRequested) return; //indexProcessing = true; if (backup.DataFlags.HasFlag(DataProcessingFlags.CDedup) /* && !cancellationTokenSource.IsCancellationRequested*/) { // save dedup and process index even if task is cancelled (for cleaning purposes) try{ DedupIndex.Instance().Persist(); } catch (Exception _e) { Logger.Append(Severity.ERROR, "Could not save deduplication indexes DB, backup data is therefore invalid. TODO: Report!!! : " + _e.Message + " ---- " + _e.StackTrace); backup.AddHubNotificationEvent(809, DedupIndex.Instance().IndexDBName, _e.Message); } } // now we have to send backup index and dedup index backup.Index.Terminate(); //backup.Index = null; indexChunk = BuildIndexChunk(); }
private void ProcessIndex() { if (backup.Level != BackupLevel.SnapshotOnly && backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CDedup) /* && !cancellationTokenSource.IsCancellationRequested*/) { // save dedup and process index even if task is cancelled (for cleaning purposes) try{ dedupIndex.Commit(); dedupIndex.Dispose(); } catch (Exception _e) { Logger.Append(Severity.ERROR, "Could not save deduplication indexes DB, backup data is therefore invalid. TODO: Report!!! : " + _e.Message + " ---- " + _e.StackTrace); backup.AddHubNotificationEvent(809, dedupIndex.IndexDBName, _e.Message); } } // now we have to send backup index and dedup index backup.Index.Terminate(); indexChunk = BuildIndexChunk(); backup.IndexName = indexChunk.Name; backup.IndexSum = indexChunk.Sum; backup.IndexSize = indexChunk.OriginalSize; }
private void Consume(Session s /*, int budget*/) { // Filter client-side processing flags DataProcessingFlags clientFlags = DataProcessingFlags.None; foreach (DataProcessingFlags flag in Enum.GetValues(typeof(DataProcessingFlags))) { if ((int)flag < 512 && backup.DataFlags.HasFlag(flag)) { clientFlags |= flag; } } DataPipeline pipeline = new DataPipeline(PipelineMode.Write, clientFlags, this.backup.Bs.Id); if (backup.DataFlags.HasFlag(DataProcessingFlags.CDedup)) { pipeline.StorageNode = s.ClientId; } if (backup.DataFlags.HasFlag(DataProcessingFlags.CEncrypt)) { //X509Certificate2 cert = new X509Certificate2(ConfigManager.GetValue("Security.CertificateFile"), ""); //pipeline.CryptoKey = (RSACryptoServiceProvider)cert.PublicKey.Key; pipeline.CryptoKey = s.CryptoKey; byte[] iv = new byte[16]; Array.Copy(System.BitConverter.GetBytes(backup.TaskId), iv, 8); Array.Copy(System.BitConverter.GetBytes(backup.TaskId), 0, iv, 8, 8); pipeline.IV = iv; //new byte[]{Convert.ToByte(backup.TaskId)}; } //pipeline.Init(); ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup); s.TransfertDoneEvent += new Session.TransfertDoneHandler(ManageChunkSent); // We transfer chunks until reaching budget or there is no more chunks to send (backup done, or severe error) while ((!chunkBuilderFeed.IsCompleted) && (s.Budget > 0)) { if (cancellationTokenSource.IsCancellationRequested) { s.LoggerInstance.Log(Severity.INFO, "Received cancellation request for task #" + backup.TaskId + ", stop processing..."); s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent); s.SendDisconnect(); s.Disconnect(); return; } BChunk chunk = null; try{ lock (processingChunks){ chunk = chunkBuilderFeed.Take(cancellationTokenSource.Token); s.LoggerInstance.Log(Severity.DEBUG2, "Processing chunk " + chunk.Name); processingChunks.Add(chunk); } cp.Process(chunk, backup.MaxChunkSize); /*backup.OriginalSize += chunk.OriginalSize; * backup.FinalSize += chunk.Size; * backup.TotalChunks ++; * backup.TotalItems += chunk.Files.Count;*/ //if(chunk.Size > pipeline.HeaderLength)// an empty chunk doesn't count // budget--; /// TODO replace waitone with a cancellationtoken-aware impl : http://msdn.microsoft.com/en-us/library/ee191552.aspx //if (chunk.SentEvent.WaitOne()){ // (60000, false)){ chunk.SentEvent.Wait(cancellationTokenSource.Token); s.LoggerInstance.Log(Severity.DEBUG2, "Processed chunk " + chunk.Name + ", remaining budget=" + s.Budget); chunk.SentEvent.Dispose(); /*} * else{ // timeout waiting for storage node confirmation * Logger.Append(Severity.WARNING, "Timeout waiting for storage node #"+s.ClientId+" ("+s.ClientIp+") confirmation, chunk "+chunk.Name); * // TODO : but if have an error with one chunk, it's likely we will have one with next chunks too. * // close session now instead of continuing??? * try{ * chunkBuilderFeed.Add(chunk, cancellationTokenSource.Token); * } * catch(InvalidOperationException){ * Logger.Append(Severity.ERROR, "Timeout waiting for storage node #"+s.ClientId+" : A session error occured, unable to use a new session to process chunk (queue is closed)"); * backup.AddHubNotificationEvent(811, chunk.Name, "Timeout waiting for storage node #"+s.ClientId+" : A session error occured, unable to use a new session to process chunk (queue is closed)"); * } * }*/ } catch (System.Net.Sockets.SocketException e) { // error sending to storage node. Re-add chunk to list and ask another storage session to hub. Console.WriteLine("############## Produce() : TAKE refused for chunk " + chunk.Name + ": " + e.Message + " ---- " + e.StackTrace); backup.AddHubNotificationEvent(811, chunk.Name, e.Message); if (chunk == null) { return; } RemoveChunk(chunk); //s.Disconnect(); try{ User.AskAlternateDestination(backup.TaskId, s.ClientId); chunkBuilderFeed.Add(chunk, cancellationTokenSource.Token); } catch (InvalidOperationException ioe) { Logger.Append(Severity.ERROR, "A session error occured, unable to use a new session to process chunk (queue is closed) : " + ioe.Message); backup.AddHubNotificationEvent(811, chunk.Name, "A session error occured, unable to use a new session to process chunk (queue is closed)"); } //throw new Exception("Something went wrong with this consumer"); } catch (OperationCanceledException) { Logger.Append(Severity.DEBUG2, "Consumer task has been manually cancelled on purpose, stopping..."); s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent); return; } /*Logger.Append(Severity.INFO, "DataProcessorStreams statistics : checksum="+BenchmarkStats.Instance().ChecksumTime +"ms, dedup="+BenchmarkStats.Instance().DedupTime +"ms, compress="+BenchmarkStats.Instance().CompressTime +"ms, send="+BenchmarkStats.Instance().SendTime+"ms.");*/ } s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent); Logger.Append(Severity.DEBUG, "Session with node #" + s.ClientId + ": processed and transferred all data chunks, unused budget=" + s.Budget); }
internal IEnumerable <BChunk> GetNextChunk() { chunkOrder++; BChunk chunk = new BChunk(this.TaskId); chunk.Order = chunkOrder; chunk.RootDriveId = this.backupRootDrive.ID; uint filePosInChunk = 0; long currentSize = 0; while (itemIterator.MoveNext()) { //foreach(IFSEntry ent in GetFilesToBackup()){ IFSEntry ent = itemIterator.Current; if (ent == null) { continue; } // 1st case : we can add more files to the chunk if (ent.FileSize < maxChunkSize) { //try{ filePosInChunk += (uint)ent.FileSize; //IFSEntry f = ent; ent.ChunkStartPos = filePosInChunk; ent.FileStartPos = 0; chunk.Add(ent); currentSize += ent.FileSize; //Console.WriteLine("GetNextChunk() : added new file to chunk - "+itemIterator.Current.FileName); //} //catch(Exception e){ // Logger.Append(Severity.ERROR, "Could not add file "+itemIterator.Current.SnapFullPath+" : "+e.Message); //} } //2nd case : a file is too big to fit into one chunk, split it else { if (chunk.Items.Count > 0) { yield return(chunk); } /*chunk = new BChunk(currentPath.Path, snapshottedPath, this.TaskId); * chunk.Order = chunkOrder; * chunk.RootDriveId = this.backupRootDrive.ID; * filePosInChunk = 0; * currentSize = 0;*/ foreach (BChunk bigFileChunk in GetBigFileChunks(itemIterator.Current, filePosInChunk)) { yield return(bigFileChunk); } chunkOrder++; chunk = new BChunk(this.TaskId); chunk.Order = chunkOrder; chunk.RootDriveId = this.backupRootDrive.ID; filePosInChunk = 0; currentSize = 0; } // 3rd case : if a chunk reaches its max packSize, we create another one if (currentSize > maxChunkSize || chunk.Items.Count > 0 && currentSize > maxPackSize /*|| currentSize == 0 && chunk.Files.Count ==0 */ || chunk.Items.Count > maxChunkFiles) { //Console.WriteLine("GetNextChunk() : chunk reached max files or max size:currentsize="+currentSize+"/"+maxChunkSize // +",chunkfilescount="+chunk.Files.Count+"/"+maxChunkFiles); yield return(chunk); chunkOrder++; chunk = new BChunk(this.TaskId); chunk.Order = chunkOrder; chunk.RootDriveId = this.backupRootDrive.ID; filePosInChunk = 0; currentSize = 0; } } //4th case : // done processing file list but chunk not complete Logger.Append(Severity.TRIVIA, "GetNextChunk() : Done gathering files inside '" + snapshottedPath + "' without reaching chunk max size. " + chunk.Items.Count + " files, " + currentSize / 1024 + "k"); //if(currentSize > 0){ yield return(chunk); //itemIterator = GetFilesToBackup().GetEnumerator(); //yield break; //} }
// TODO : split this into a dedicated consumer class, also merge ChunkProcessor in this future class // using a class will allow to track what the consumer is doing, and cancel it if it blocks on an empty chunksFeeds.Take (when everything has yet been processed) private void Consume(Session s /*, int budget*/) { if (/*chunksFeeds.IsCompleted*/ megaQueue.Count == 0 && doneBdhProducers == 0 && chunksFeeds.Count == 0) { Console.WriteLine("------------------------- CONSUME() : procducers queues marked complete and already processed, exiting"); return; } // Filter client-side processing flags DataProcessingFlags clientFlags = DataProcessingFlags.None; foreach (DataProcessingFlags flag in Enum.GetValues(typeof(DataProcessingFlags))) { if ((int)flag < 512 && backup.BackupSet.DataFlags.HasFlag(flag)) { clientFlags |= flag; } } DataPipeline pipeline = new DataPipeline(PipelineMode.Write, clientFlags, this.backup.BackupSet.Id, dedupIndex); if (backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CDedup)) { pipeline.StorageNode = s.ToNode.Id; } if (backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CEncrypt)) { //X509Certificate2 cert = new X509Certificate2(ConfigManager.GetValue("Security.CertificateFile"), ""); //pipeline.CryptoKey = (RSACryptoServiceProvider)cert.PublicKey.Key; pipeline.CryptoKey = s.CryptoKey; byte[] iv = new byte[16]; Array.Copy(System.BitConverter.GetBytes(backup.Id), iv, 8); Array.Copy(System.BitConverter.GetBytes(backup.Id), 0, iv, 8, 8); pipeline.IV = iv; //new byte[]{Convert.ToByte(backup.TaskId)}; } if (chunksFeeds.Count == 0) { return; } ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup, this.cancellationTokenSource.Token); s.TransfertDoneEvent += new Session.TransfertDoneHandler(ManageChunkSent); Console.WriteLine("------------------------- Consume(" + s.Id + ") : taking chunks feed from queue which has " + chunksFeeds.Count + " elements yet to processe"); BlockingCollection <BChunk> myFeed = chunksFeeds.Take(this.cancellationTokenSource.Token); Console.WriteLine("------------------------- Consume(" + s.Id + ") : got a new queue to process, yummy!"); // We transfer chunks until reaching budget or there is no more chunks to send (backup done, or severe error) while ((!myFeed.IsCompleted) && (s.Budget > 0)) { if (cancellationTokenSource.IsCancellationRequested) { s.LoggerInstance.Log(Severity.INFO, "Received cancellation request for task #" + backup.Id + ", stop processing..."); s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent); s.SendDisconnect(); s.Disconnect(); return; } BChunk chunk = null; try{ chunk = myFeed.Take(cancellationTokenSource.Token); s.LoggerInstance.Log(Severity.TRIVIA, "Processing chunk " + chunk.Name); lock (processingChunks){ processingChunks.Add(chunk); } cp.Process(chunk, backup.BackupSet.MaxChunkSize); if (chunk.SentEvent.Wait(180000, cancellationTokenSource.Token)) { s.LoggerInstance.Log(Severity.TRIVIA, "Processed chunk " + chunk.Name + ", remaining budget=" + s.Budget); } else // timeout waiting for storage node confirmation { Logger.Append(Severity.WARNING, "Timeout (3 minutes) waiting for storage node #" + s.ToNode.Id + " (" + s.ToNode.IP + ") confirmation, chunk " + chunk.Name); backup.AddHubNotificationEvent(811, chunk.Name, "Timeout waiting for storage node #" + s.ToNode.Id + " transfer confirmation"); // TODO : but if have an error with one chunk, it's likely we will have one with next chunks too. // close session now instead of continuing??? try{ myFeed.Add(chunk, cancellationTokenSource.Token); } catch (InvalidOperationException) { Logger.Append(Severity.ERROR, "Timeout waiting for storage node #" + s.ToNode.Id + " : A session error occured, unable to use a new session to process chunk (queue is closed)"); backup.AddHubNotificationEvent(811, chunk.Name, "Timeout waiting for storage node #" + s.ToNode.Id + " : A session error occured, unable to use a new session to process chunk (queue is closed)"); } } if (chunk.SentEvent != null) { chunk.SentEvent.Dispose(); } } catch (System.Net.Sockets.SocketException e) { // error sending to storage node. Re-add chunk to list and ask another storage session to hub. Console.WriteLine("############## Produce() : TAKE refused for chunk " + chunk.Name + ": " + e.Message + " ---- " + e.StackTrace); backup.AddHubNotificationEvent(811, chunk.Name, e.Message); if (chunk == null) { return; } RemoveChunk(chunk); try{ this.StorageNeeded(s, 1, false, true); myFeed.Add(chunk, cancellationTokenSource.Token); } catch (InvalidOperationException ioe) { Logger.Append(Severity.ERROR, "A session error occured, unable to use a new session to process chunk (queue is closed) : " + ioe.Message); backup.AddHubNotificationEvent(811, chunk.Name, "A session error occured, unable to use a new session to process chunk (queue is closed)"); } } catch (OperationCanceledException) { Logger.Append(Severity.TRIVIA, "Consumer task has been manually cancelled on purpose, stopping..."); s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent); return; } catch (InvalidOperationException) { Logger.Append(Severity.DEBUG, "Consumer task has done processing its chunks list"); } } if (!myFeed.IsCompleted) { // re-add this non-terminated chunk list on queue, to be processed by next session. Console.WriteLine(" ---------- Consumer : feed not completed, re-adding to chunksFeeds"); chunksFeeds.Add(myFeed, cancellationTokenSource.Token); Console.WriteLine(" ---------- Consumer : feed not completed, re-added, chunksFeeds count=" + chunksFeeds.Count); } s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent); Logger.Append(Severity.DEBUG, "Session #" + s.Id + " with node #" + s.ToNode.Id + ": processed and transferred all data chunks, unused budget=" + s.Budget); }