private void SendIndex(long taskId, Session s /*, int budget*/)
{
System.Threading.Tasks.Task consumeIndexTask = System.Threading.Tasks.Task.Factory.StartNew(() => {
DataPipeline pipeline = new DataPipeline(PipelineMode.Write, DataProcessingFlags.CCompress | DataProcessingFlags.CChecksum);
//pipeline.Init();
ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup);
cp.Process(indexChunk, backup.MaxChunkSize * 10);
indexChunk.Size = pipeline.Stream.Length;
indexChunk.AddDestination(s.ClientId);
}, TaskCreationOptions.LongRunning);
consumeIndexTask.ContinueWith(o => {
Logger.Append(Severity.INFO, "Processed and sent backup index");
backup.AddHubNotificationEvent(705, Math.Round((double)indexChunk.Size / 1024 / 1024, 1).ToString(), "");
string synthIndexSum = indexChunk.Sum; // for Fulls
if (backup.Bs.ScheduleTimes[0].Level != P2PBackup.Common.BackupLevel.Full)
{
IndexManager idxManager = new IndexManager();
Logger.Append(Severity.INFO, "Building synthetic full index...");
idxManager.CreateSyntheticFullIndex(backup.RefTaskId, taskId, backup.RootDrives);
backup.AddHubNotificationEvent(707, "", "");
synthIndexSum = IndexManager.CheckSumIndex(taskId, false); // for synthetic backups
}
User.SendDoneBackup(taskId, backup.OriginalSize, backup.FinalSize, backup.TotalItems, indexChunk.Name, indexChunk.Sum, synthIndexSum, indexChunk.StorageDestinations, 100);
Logger.Append(Severity.INFO, "Task " + taskId + " has finished. " + backup.TotalItems + " items, " + backup.TotalChunks + " chunks. Original data size=" + Math.Round((double)backup.OriginalSize / 1024 / 1024, 1) + "MB, final=" + Math.Round((double)backup.FinalSize / 1024 / 1024, 1) + "MB");
string statsByKind = "Task " + taskId + " processed: ";
for (int i = 0; i < 10; i++)
{
statsByKind += backup.ItemsByType[i] + " " + ((FileType)i).ToString() + ", ";
}
Logger.Append(Severity.INFO, statsByKind);
#if DEBUG
Logger.Append(Severity.INFO, "DataProcessorStreams statistics : checksum=" + BenchmarkStats.Instance().ChecksumTime
+ "ms, dedup=" + BenchmarkStats.Instance().DedupTime
+ "ms, compress=" + BenchmarkStats.Instance().CompressTime
+ "ms, send=" + BenchmarkStats.Instance().SendTime + "ms.");
Logger.Append(Severity.INFO, "Dedup statistics : lookups=" + BenchmarkStats.Instance().DedupLookups
+ ", hotfound=" + BenchmarkStats.Instance().DedupHotFound
+ ", coldfound=" + BenchmarkStats.Instance().DedupColdFound
+ ", add=" + BenchmarkStats.Instance().DedupAdd + ".");
#endif
User.StorageSessionReceivedEvent -= new User.StorageSessionReceivedHandler(this.SendIndex);
//Console.WriteLine("IndexSessionReceived() : backup typre="+backup. .BackupTimes[0].Type);
backup.AddHubNotificationEvent(706, "", "");
backup.Terminate(true);
BackupDoneEvent(taskId);
}, TaskContinuationOptions.OnlyOnRanToCompletion | TaskContinuationOptions.ExecuteSynchronously
| TaskContinuationOptions.NotOnFaulted | TaskContinuationOptions.NotOnCanceled);
//consumeTask.Dispose();
}
internal void SendIndex(Session s)
{
// Terminate and free Dedup DB before saving it to ensure all its content has been flushed to disk
dedupIndex.Dispose();
System.Threading.Tasks.Task consumeIndexTask = System.Threading.Tasks.Task.Factory.StartNew(() => {
DataPipeline pipeline = new DataPipeline(PipelineMode.Write, DataProcessingFlags.CCompress | DataProcessingFlags.CChecksum);
ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup, new CancellationToken());
cp.Process(indexChunk, backup.BackupSet.MaxChunkSize * 10);
indexChunk.Size = pipeline.Stream.Length;
indexChunk.AddDestination(s.ToNode.Id);
}, TaskCreationOptions.LongRunning);
consumeIndexTask.ContinueWith(o => {
Logger.Append(Severity.INFO, "Processed and sent backup index");
backup.AddHubNotificationEvent(705, Math.Round((double)indexChunk.Size / 1024 / 1024, 1).ToString(), "");
// set final info : index location
backup.IndexStorageNodes = indexChunk.StorageDestinations;
Logger.Append(Severity.INFO, "Task " + s.TaskId + " has finished. " + backup.TotalItems + " items, " + backup.TotalChunks + " chunks. Original data size=" + Math.Round((double)backup.OriginalSize / 1024 / 1024, 1) + "MB, final=" + Math.Round((double)backup.FinalSize / 1024 / 1024, 1) + "MB");
string statsByKind = "Task " + s.TaskId + " processed: ";
for (int i = 0; i < 10; i++)
{
statsByKind += backup.ItemsByType[i] + " " + ((FileType)i).ToString() + ", ";
}
Logger.Append(Severity.INFO, statsByKind);
#if DEBUG
Logger.Append(Severity.INFO, "DataProcessorStreams statistics : checksum=" + BenchmarkStats.Instance().ChecksumTime
+ "ms, dedup=" + BenchmarkStats.Instance().DedupTime
+ "ms, compress=" + BenchmarkStats.Instance().CompressTime
+ "ms, send=" + BenchmarkStats.Instance().SendTime + "ms.");
Logger.Append(Severity.INFO, "Dedup statistics : lookups=" + BenchmarkStats.Instance().DedupLookups
+ ", hotfound=" + BenchmarkStats.Instance().DedupHotFound
+ ", coldfound=" + BenchmarkStats.Instance().DedupColdFound
+ ", add=" + BenchmarkStats.Instance().DedupAdd + ".");
#endif
//User.SessionReady -= new User.StorageSessionReceivedHandler(this.SendIndex);
backup.AddHubNotificationEvent(706, "", "");
backup.Terminate(true);
BackupDoneEvent(backup);
}, TaskContinuationOptions.OnlyOnRanToCompletion
| TaskContinuationOptions.ExecuteSynchronously
| TaskContinuationOptions.NotOnFaulted
| TaskContinuationOptions.NotOnCanceled
);
//consumeTask.Dispose();
}