private BChunk BuildIndexChunk() { backup.AddHubNotificationEvent(704, "", ""); BChunk iChunk = new BChunk(/*backup.TaskId, */ backup.Index.FullName, backup.Index.FullName, backup.TaskId); //string name, int bsid, string bPath, string snapPath) try{ //iChunk.Add(FileProvider.GetFile(index.FullName)); iChunk.Add(new MinimalFsItem(backup.Index.FullName)); if (backup.DataFlags.HasFlag(DataProcessingFlags.CDedup)) // backup the deduplication database { iChunk.Add(ItemProvider.GetProvider().GetItemByPath(DedupIndex.Instance().IndexDBName)); } /*string sumHash; * using(FileStream cksumFS = new FileStream(backup.Index.FullName, FileMode.Open, FileAccess.Read)){ * sumHash = BitConverter.ToString(SHA1.Create().ComputeHash(cksumFS)); * iChunk.Sum = sumHash; * }*/ iChunk.Sum = IndexManager.CheckSumIndex(backup.TaskId, (backup.Level != BackupLevel.Full)); // register for session received, to process index transfer User.StorageSessionReceivedEvent += new User.StorageSessionReceivedHandler(this.SendIndex); User.AskIndexDest(backup.TaskId, backup.Index.Name, iChunk.Sum); Logger.Append(Severity.DEBUG, "Asked index destination to hub"); return(iChunk); } catch (Exception e) { Logger.Append(Severity.ERROR, "Couldn't checksum index and/or ask destination to hub: " + e.Message + "---" + e.StackTrace); backup.AddHubNotificationEvent(808, e.Message, ""); } return(null); }
private void SendIndex(long taskId, Session s /*, int budget*/) { System.Threading.Tasks.Task consumeIndexTask = System.Threading.Tasks.Task.Factory.StartNew(() => { DataPipeline pipeline = new DataPipeline(PipelineMode.Write, DataProcessingFlags.CCompress | DataProcessingFlags.CChecksum); //pipeline.Init(); ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup); cp.Process(indexChunk, backup.MaxChunkSize * 10); indexChunk.Size = pipeline.Stream.Length; indexChunk.AddDestination(s.ClientId); }, TaskCreationOptions.LongRunning); consumeIndexTask.ContinueWith(o => { Logger.Append(Severity.INFO, "Processed and sent backup index"); backup.AddHubNotificationEvent(705, Math.Round((double)indexChunk.Size / 1024 / 1024, 1).ToString(), ""); string synthIndexSum = indexChunk.Sum; // for Fulls if (backup.Bs.ScheduleTimes[0].Level != P2PBackup.Common.BackupLevel.Full) { IndexManager idxManager = new IndexManager(); Logger.Append(Severity.INFO, "Building synthetic full index..."); idxManager.CreateSyntheticFullIndex(backup.RefTaskId, taskId, backup.RootDrives); backup.AddHubNotificationEvent(707, "", ""); synthIndexSum = IndexManager.CheckSumIndex(taskId, false); // for synthetic backups } User.SendDoneBackup(taskId, backup.OriginalSize, backup.FinalSize, backup.TotalItems, indexChunk.Name, indexChunk.Sum, synthIndexSum, indexChunk.StorageDestinations, 100); Logger.Append(Severity.INFO, "Task " + taskId + " has finished. " + backup.TotalItems + " items, " + backup.TotalChunks + " chunks. Original data size=" + Math.Round((double)backup.OriginalSize / 1024 / 1024, 1) + "MB, final=" + Math.Round((double)backup.FinalSize / 1024 / 1024, 1) + "MB"); string statsByKind = "Task " + taskId + " processed: "; for (int i = 0; i < 10; i++) { statsByKind += backup.ItemsByType[i] + " " + ((FileType)i).ToString() + ", "; } Logger.Append(Severity.INFO, statsByKind); #if DEBUG Logger.Append(Severity.INFO, "DataProcessorStreams statistics : checksum=" + BenchmarkStats.Instance().ChecksumTime + "ms, dedup=" + BenchmarkStats.Instance().DedupTime + "ms, compress=" + BenchmarkStats.Instance().CompressTime + "ms, send=" + BenchmarkStats.Instance().SendTime + "ms."); Logger.Append(Severity.INFO, "Dedup statistics : lookups=" + BenchmarkStats.Instance().DedupLookups + ", hotfound=" + BenchmarkStats.Instance().DedupHotFound + ", coldfound=" + BenchmarkStats.Instance().DedupColdFound + ", add=" + BenchmarkStats.Instance().DedupAdd + "."); #endif User.StorageSessionReceivedEvent -= new User.StorageSessionReceivedHandler(this.SendIndex); //Console.WriteLine("IndexSessionReceived() : backup typre="+backup. .BackupTimes[0].Type); backup.AddHubNotificationEvent(706, "", ""); backup.Terminate(true); BackupDoneEvent(taskId); }, TaskContinuationOptions.OnlyOnRanToCompletion | TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.NotOnFaulted | TaskContinuationOptions.NotOnCanceled); //consumeTask.Dispose(); }
private BChunk BuildIndexChunk() { backup.AddHubNotificationEvent(704, "", ""); BChunk iChunk = null; string synthIndexFullPath = null; if (backup.Level != P2PBackup.Common.BackupLevel.Full && backup.Level != P2PBackup.Common.BackupLevel.SnapshotOnly) { IndexManager idxManager = new IndexManager(); Logger.Append(Severity.INFO, "Building synthetic full index..."); synthIndexFullPath = idxManager.CreateSyntheticFullIndex(backup.ParentTrackingId, backup.Id, backup.RootDrives); backup.AddHubNotificationEvent(707, "", ""); backup.SyntheticIndexSum = IndexManager.CheckSumIndex(backup.Id, false); // for synthetic backups } if (backup.Level == BackupLevel.Refresh) // backup the synth index { iChunk = new BChunk(backup.Id); iChunk.Add(new MinimalFsItem(synthIndexFullPath)); // minimalitem because we only care the data, not the rest } else // only backup the partial index { iChunk = new BChunk(backup.Id); //string name, int bsid, string bPath, string snapPath) iChunk.Add(new MinimalFsItem(backup.Index.FullName)); // minimalitem because we only care the data, not the rest } try{ //iChunk.Add(FileProvider.GetFile(index.FullName)); if (backup.Level != BackupLevel.SnapshotOnly && backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CDedup)) // backup the deduplication database { iChunk.Add(ItemProvider.GetProvider().GetItemByPath(dedupIndex.IndexDBName)); backup.DdbSum = dedupIndex.ChecksumDdb(); } iChunk.Sum = IndexManager.CheckSumIndex(backup.Id, (backup.Level == BackupLevel.Refresh)); if (backup.Level == BackupLevel.Full) { backup.SyntheticIndexSum = iChunk.Sum; // for Fulls } // stop waiting for sessions used for regular data transfer... //User.SessionReady -= this.SessionReceived; // ...but re-register for session that will transfer the index //User.SessionReady += this.SendIndex; StorageNeeded(new PeerSession { TaskId = backup.Id }, 1, true, false); Logger.Append(Severity.DEBUG, "Asked index destination to hub"); return(iChunk); } catch (Exception e) { Logger.Append(Severity.ERROR, "Couldn't checksum index and/or ask destination to hub: " + e.Message + "---" + e.StackTrace); backup.AddHubNotificationEvent(808, e.Message, ""); } return(null); }