private void SendIndex(long taskId, Session s /*, int budget*/)
{
System.Threading.Tasks.Task consumeIndexTask = System.Threading.Tasks.Task.Factory.StartNew(() => {
DataPipeline pipeline = new DataPipeline(PipelineMode.Write, DataProcessingFlags.CCompress | DataProcessingFlags.CChecksum);
//pipeline.Init();
ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup);
cp.Process(indexChunk, backup.MaxChunkSize * 10);
indexChunk.Size = pipeline.Stream.Length;
indexChunk.AddDestination(s.ClientId);
}, TaskCreationOptions.LongRunning);
consumeIndexTask.ContinueWith(o => {
Logger.Append(Severity.INFO, "Processed and sent backup index");
backup.AddHubNotificationEvent(705, Math.Round((double)indexChunk.Size / 1024 / 1024, 1).ToString(), "");
string synthIndexSum = indexChunk.Sum; // for Fulls
if (backup.Bs.ScheduleTimes[0].Level != P2PBackup.Common.BackupLevel.Full)
{
IndexManager idxManager = new IndexManager();
Logger.Append(Severity.INFO, "Building synthetic full index...");
idxManager.CreateSyntheticFullIndex(backup.RefTaskId, taskId, backup.RootDrives);
backup.AddHubNotificationEvent(707, "", "");
synthIndexSum = IndexManager.CheckSumIndex(taskId, false); // for synthetic backups
}
User.SendDoneBackup(taskId, backup.OriginalSize, backup.FinalSize, backup.TotalItems, indexChunk.Name, indexChunk.Sum, synthIndexSum, indexChunk.StorageDestinations, 100);
Logger.Append(Severity.INFO, "Task " + taskId + " has finished. " + backup.TotalItems + " items, " + backup.TotalChunks + " chunks. Original data size=" + Math.Round((double)backup.OriginalSize / 1024 / 1024, 1) + "MB, final=" + Math.Round((double)backup.FinalSize / 1024 / 1024, 1) + "MB");
string statsByKind = "Task " + taskId + " processed: ";
for (int i = 0; i < 10; i++)
{
statsByKind += backup.ItemsByType[i] + " " + ((FileType)i).ToString() + ", ";
}
Logger.Append(Severity.INFO, statsByKind);
#if DEBUG
Logger.Append(Severity.INFO, "DataProcessorStreams statistics : checksum=" + BenchmarkStats.Instance().ChecksumTime
+ "ms, dedup=" + BenchmarkStats.Instance().DedupTime
+ "ms, compress=" + BenchmarkStats.Instance().CompressTime
+ "ms, send=" + BenchmarkStats.Instance().SendTime + "ms.");
Logger.Append(Severity.INFO, "Dedup statistics : lookups=" + BenchmarkStats.Instance().DedupLookups
+ ", hotfound=" + BenchmarkStats.Instance().DedupHotFound
+ ", coldfound=" + BenchmarkStats.Instance().DedupColdFound
+ ", add=" + BenchmarkStats.Instance().DedupAdd + ".");
#endif
User.StorageSessionReceivedEvent -= new User.StorageSessionReceivedHandler(this.SendIndex);
//Console.WriteLine("IndexSessionReceived() : backup typre="+backup. .BackupTimes[0].Type);
backup.AddHubNotificationEvent(706, "", "");
backup.Terminate(true);
BackupDoneEvent(taskId);
}, TaskContinuationOptions.OnlyOnRanToCompletion | TaskContinuationOptions.ExecuteSynchronously
| TaskContinuationOptions.NotOnFaulted | TaskContinuationOptions.NotOnCanceled);
//consumeTask.Dispose();
}
internal void SendIndex(Session s)
{
// Terminate and free Dedup DB before saving it to ensure all its content has been flushed to disk
dedupIndex.Dispose();
System.Threading.Tasks.Task consumeIndexTask = System.Threading.Tasks.Task.Factory.StartNew(() => {
DataPipeline pipeline = new DataPipeline(PipelineMode.Write, DataProcessingFlags.CCompress | DataProcessingFlags.CChecksum);
ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup, new CancellationToken());
cp.Process(indexChunk, backup.BackupSet.MaxChunkSize * 10);
indexChunk.Size = pipeline.Stream.Length;
indexChunk.AddDestination(s.ToNode.Id);
}, TaskCreationOptions.LongRunning);
consumeIndexTask.ContinueWith(o => {
Logger.Append(Severity.INFO, "Processed and sent backup index");
backup.AddHubNotificationEvent(705, Math.Round((double)indexChunk.Size / 1024 / 1024, 1).ToString(), "");
// set final info : index location
backup.IndexStorageNodes = indexChunk.StorageDestinations;
Logger.Append(Severity.INFO, "Task " + s.TaskId + " has finished. " + backup.TotalItems + " items, " + backup.TotalChunks + " chunks. Original data size=" + Math.Round((double)backup.OriginalSize / 1024 / 1024, 1) + "MB, final=" + Math.Round((double)backup.FinalSize / 1024 / 1024, 1) + "MB");
string statsByKind = "Task " + s.TaskId + " processed: ";
for (int i = 0; i < 10; i++)
{
statsByKind += backup.ItemsByType[i] + " " + ((FileType)i).ToString() + ", ";
}
Logger.Append(Severity.INFO, statsByKind);
#if DEBUG
Logger.Append(Severity.INFO, "DataProcessorStreams statistics : checksum=" + BenchmarkStats.Instance().ChecksumTime
+ "ms, dedup=" + BenchmarkStats.Instance().DedupTime
+ "ms, compress=" + BenchmarkStats.Instance().CompressTime
+ "ms, send=" + BenchmarkStats.Instance().SendTime + "ms.");
Logger.Append(Severity.INFO, "Dedup statistics : lookups=" + BenchmarkStats.Instance().DedupLookups
+ ", hotfound=" + BenchmarkStats.Instance().DedupHotFound
+ ", coldfound=" + BenchmarkStats.Instance().DedupColdFound
+ ", add=" + BenchmarkStats.Instance().DedupAdd + ".");
#endif
//User.SessionReady -= new User.StorageSessionReceivedHandler(this.SendIndex);
backup.AddHubNotificationEvent(706, "", "");
backup.Terminate(true);
BackupDoneEvent(backup);
}, TaskContinuationOptions.OnlyOnRanToCompletion
| TaskContinuationOptions.ExecuteSynchronously
| TaskContinuationOptions.NotOnFaulted
| TaskContinuationOptions.NotOnCanceled
);
//consumeTask.Dispose();
}
public bool Delete(int id)
{
//if (_context == null)
var dbController = _appManager.DataController; //.Instance(appId: appId);
var canDeleteResult = (dbController.Entities.CanDeleteEntity(id)); // _context.EntCommands.CanDeleteEntity(id);
if (!canDeleteResult.Item1)
{
throw new Exception(canDeleteResult.Item2);
}
// Get the Entity describing the Pipeline and Pipeline Parts (DataSources)
// var source = DataSource.GetInitialDataSource(appId: appId);
var pipelineEntity = DataPipeline.GetPipelineEntity(id, _appManager.Cache);
var dataSources = DataPipeline.GetPipelineParts(_appManager.ZoneId, _appManager.AppId, pipelineEntity.EntityGuid);
var metaDataSource = DataSource.GetMetaDataSource(appId: _appManager.AppId);
// Delete Pipeline Parts
foreach (var dataSource in dataSources)
{
// Delete Configuration Entities (if any)
var dataSourceConfig = metaDataSource.GetAssignedEntities(Constants.MetadataForEntity /* .AssignmentObjectTypeEntity*/, dataSource.EntityGuid).FirstOrDefault();
if (dataSourceConfig != null)
{
dbController.Entities.DeleteEntity(dataSourceConfig.EntityId);
}
dbController.Entities.DeleteEntity(dataSource.EntityId);
}
// Delete Pipeline
dbController.Entities.DeleteEntity(id);
// flush cache
SystemManager.Purge(_appManager.AppId);
return(true);
}
public PassThruTargetAdapter(DataPipeline <TRow> .RowDelegate onProcess)
{
OnProcess = onProcess;
}
private void Consume(Session s /*, int budget*/)
{
// Filter client-side processing flags
DataProcessingFlags clientFlags = DataProcessingFlags.None;
foreach (DataProcessingFlags flag in Enum.GetValues(typeof(DataProcessingFlags)))
{
if ((int)flag < 512 && backup.DataFlags.HasFlag(flag))
{
clientFlags |= flag;
}
}
DataPipeline pipeline = new DataPipeline(PipelineMode.Write, clientFlags, this.backup.Bs.Id);
if (backup.DataFlags.HasFlag(DataProcessingFlags.CDedup))
{
pipeline.StorageNode = s.ClientId;
}
if (backup.DataFlags.HasFlag(DataProcessingFlags.CEncrypt))
{
//X509Certificate2 cert = new X509Certificate2(ConfigManager.GetValue("Security.CertificateFile"), "");
//pipeline.CryptoKey = (RSACryptoServiceProvider)cert.PublicKey.Key;
pipeline.CryptoKey = s.CryptoKey;
byte[] iv = new byte[16];
Array.Copy(System.BitConverter.GetBytes(backup.TaskId), iv, 8);
Array.Copy(System.BitConverter.GetBytes(backup.TaskId), 0, iv, 8, 8);
pipeline.IV = iv; //new byte[]{Convert.ToByte(backup.TaskId)};
}
//pipeline.Init();
ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup);
s.TransfertDoneEvent += new Session.TransfertDoneHandler(ManageChunkSent);
// We transfer chunks until reaching budget or there is no more chunks to send (backup done, or severe error)
while ((!chunkBuilderFeed.IsCompleted) && (s.Budget > 0))
{
if (cancellationTokenSource.IsCancellationRequested)
{
s.LoggerInstance.Log(Severity.INFO, "Received cancellation request for task #" + backup.TaskId + ", stop processing...");
s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent);
s.SendDisconnect();
s.Disconnect();
return;
}
BChunk chunk = null;
try{
lock (processingChunks){
chunk = chunkBuilderFeed.Take(cancellationTokenSource.Token);
s.LoggerInstance.Log(Severity.DEBUG2, "Processing chunk " + chunk.Name);
processingChunks.Add(chunk);
}
cp.Process(chunk, backup.MaxChunkSize);
/*backup.OriginalSize += chunk.OriginalSize;
* backup.FinalSize += chunk.Size;
* backup.TotalChunks ++;
* backup.TotalItems += chunk.Files.Count;*/
//if(chunk.Size > pipeline.HeaderLength)// an empty chunk doesn't count
// budget--;
/// TODO replace waitone with a cancellationtoken-aware impl : http://msdn.microsoft.com/en-us/library/ee191552.aspx
//if (chunk.SentEvent.WaitOne()){ // (60000, false)){
chunk.SentEvent.Wait(cancellationTokenSource.Token);
s.LoggerInstance.Log(Severity.DEBUG2, "Processed chunk " + chunk.Name + ", remaining budget=" + s.Budget);
chunk.SentEvent.Dispose();
/*}
* else{ // timeout waiting for storage node confirmation
* Logger.Append(Severity.WARNING, "Timeout waiting for storage node #"+s.ClientId+" ("+s.ClientIp+") confirmation, chunk "+chunk.Name);
* // TODO : but if have an error with one chunk, it's likely we will have one with next chunks too.
* // close session now instead of continuing???
* try{
* chunkBuilderFeed.Add(chunk, cancellationTokenSource.Token);
* }
* catch(InvalidOperationException){
* Logger.Append(Severity.ERROR, "Timeout waiting for storage node #"+s.ClientId+" : A session error occured, unable to use a new session to process chunk (queue is closed)");
* backup.AddHubNotificationEvent(811, chunk.Name, "Timeout waiting for storage node #"+s.ClientId+" : A session error occured, unable to use a new session to process chunk (queue is closed)");
* }
* }*/
}
catch (System.Net.Sockets.SocketException e) {
// error sending to storage node. Re-add chunk to list and ask another storage session to hub.
Console.WriteLine("############## Produce() : TAKE refused for chunk " + chunk.Name + ": " + e.Message + " ---- " + e.StackTrace);
backup.AddHubNotificationEvent(811, chunk.Name, e.Message);
if (chunk == null)
{
return;
}
RemoveChunk(chunk);
//s.Disconnect();
try{
User.AskAlternateDestination(backup.TaskId, s.ClientId);
chunkBuilderFeed.Add(chunk, cancellationTokenSource.Token);
}
catch (InvalidOperationException ioe) {
Logger.Append(Severity.ERROR, "A session error occured, unable to use a new session to process chunk (queue is closed) : " + ioe.Message);
backup.AddHubNotificationEvent(811, chunk.Name, "A session error occured, unable to use a new session to process chunk (queue is closed)");
}
//throw new Exception("Something went wrong with this consumer");
}
catch (OperationCanceledException) {
Logger.Append(Severity.DEBUG2, "Consumer task has been manually cancelled on purpose, stopping...");
s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent);
return;
}
/*Logger.Append(Severity.INFO, "DataProcessorStreams statistics : checksum="+BenchmarkStats.Instance().ChecksumTime
+"ms, dedup="+BenchmarkStats.Instance().DedupTime
+"ms, compress="+BenchmarkStats.Instance().CompressTime
+"ms, send="+BenchmarkStats.Instance().SendTime+"ms.");*/
}
s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent);
Logger.Append(Severity.DEBUG, "Session with node #" + s.ClientId + ": processed and transferred all data chunks, unused budget=" + s.Budget);
}
// TODO : split this into a dedicated consumer class, also merge ChunkProcessor in this future class
// using a class will allow to track what the consumer is doing, and cancel it if it blocks on an empty chunksFeeds.Take (when everything has yet been processed)
private void Consume(Session s /*, int budget*/)
{
if (/*chunksFeeds.IsCompleted*/ megaQueue.Count == 0 && doneBdhProducers == 0 && chunksFeeds.Count == 0)
{
Console.WriteLine("------------------------- CONSUME() : procducers queues marked complete and already processed, exiting");
return;
}
// Filter client-side processing flags
DataProcessingFlags clientFlags = DataProcessingFlags.None;
foreach (DataProcessingFlags flag in Enum.GetValues(typeof(DataProcessingFlags)))
{
if ((int)flag < 512 && backup.BackupSet.DataFlags.HasFlag(flag))
{
clientFlags |= flag;
}
}
DataPipeline pipeline = new DataPipeline(PipelineMode.Write, clientFlags, this.backup.BackupSet.Id, dedupIndex);
if (backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CDedup))
{
pipeline.StorageNode = s.ToNode.Id;
}
if (backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CEncrypt))
{
//X509Certificate2 cert = new X509Certificate2(ConfigManager.GetValue("Security.CertificateFile"), "");
//pipeline.CryptoKey = (RSACryptoServiceProvider)cert.PublicKey.Key;
pipeline.CryptoKey = s.CryptoKey;
byte[] iv = new byte[16];
Array.Copy(System.BitConverter.GetBytes(backup.Id), iv, 8);
Array.Copy(System.BitConverter.GetBytes(backup.Id), 0, iv, 8, 8);
pipeline.IV = iv; //new byte[]{Convert.ToByte(backup.TaskId)};
}
if (chunksFeeds.Count == 0)
{
return;
}
ChunkProcessor cp = new ChunkProcessor(s, pipeline, backup, this.cancellationTokenSource.Token);
s.TransfertDoneEvent += new Session.TransfertDoneHandler(ManageChunkSent);
Console.WriteLine("------------------------- Consume(" + s.Id + ") : taking chunks feed from queue which has " + chunksFeeds.Count + " elements yet to processe");
BlockingCollection <BChunk> myFeed = chunksFeeds.Take(this.cancellationTokenSource.Token);
Console.WriteLine("------------------------- Consume(" + s.Id + ") : got a new queue to process, yummy!");
// We transfer chunks until reaching budget or there is no more chunks to send (backup done, or severe error)
while ((!myFeed.IsCompleted) && (s.Budget > 0))
{
if (cancellationTokenSource.IsCancellationRequested)
{
s.LoggerInstance.Log(Severity.INFO, "Received cancellation request for task #" + backup.Id + ", stop processing...");
s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent);
s.SendDisconnect();
s.Disconnect();
return;
}
BChunk chunk = null;
try{
chunk = myFeed.Take(cancellationTokenSource.Token);
s.LoggerInstance.Log(Severity.TRIVIA, "Processing chunk " + chunk.Name);
lock (processingChunks){
processingChunks.Add(chunk);
}
cp.Process(chunk, backup.BackupSet.MaxChunkSize);
if (chunk.SentEvent.Wait(180000, cancellationTokenSource.Token))
{
s.LoggerInstance.Log(Severity.TRIVIA, "Processed chunk " + chunk.Name + ", remaining budget=" + s.Budget);
}
else // timeout waiting for storage node confirmation
{
Logger.Append(Severity.WARNING, "Timeout (3 minutes) waiting for storage node #" + s.ToNode.Id + " (" + s.ToNode.IP + ") confirmation, chunk " + chunk.Name);
backup.AddHubNotificationEvent(811, chunk.Name, "Timeout waiting for storage node #" + s.ToNode.Id + " transfer confirmation");
// TODO : but if have an error with one chunk, it's likely we will have one with next chunks too.
// close session now instead of continuing???
try{
myFeed.Add(chunk, cancellationTokenSource.Token);
}
catch (InvalidOperationException) {
Logger.Append(Severity.ERROR, "Timeout waiting for storage node #" + s.ToNode.Id + " : A session error occured, unable to use a new session to process chunk (queue is closed)");
backup.AddHubNotificationEvent(811, chunk.Name, "Timeout waiting for storage node #" + s.ToNode.Id + " : A session error occured, unable to use a new session to process chunk (queue is closed)");
}
}
if (chunk.SentEvent != null)
{
chunk.SentEvent.Dispose();
}
}
catch (System.Net.Sockets.SocketException e) {
// error sending to storage node. Re-add chunk to list and ask another storage session to hub.
Console.WriteLine("############## Produce() : TAKE refused for chunk " + chunk.Name + ": " + e.Message + " ---- " + e.StackTrace);
backup.AddHubNotificationEvent(811, chunk.Name, e.Message);
if (chunk == null)
{
return;
}
RemoveChunk(chunk);
try{
this.StorageNeeded(s, 1, false, true);
myFeed.Add(chunk, cancellationTokenSource.Token);
}
catch (InvalidOperationException ioe) {
Logger.Append(Severity.ERROR, "A session error occured, unable to use a new session to process chunk (queue is closed) : " + ioe.Message);
backup.AddHubNotificationEvent(811, chunk.Name, "A session error occured, unable to use a new session to process chunk (queue is closed)");
}
}
catch (OperationCanceledException) {
Logger.Append(Severity.TRIVIA, "Consumer task has been manually cancelled on purpose, stopping...");
s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent);
return;
}
catch (InvalidOperationException) {
Logger.Append(Severity.DEBUG, "Consumer task has done processing its chunks list");
}
}
if (!myFeed.IsCompleted)
{
// re-add this non-terminated chunk list on queue, to be processed by next session.
Console.WriteLine(" ---------- Consumer : feed not completed, re-adding to chunksFeeds");
chunksFeeds.Add(myFeed, cancellationTokenSource.Token);
Console.WriteLine(" ---------- Consumer : feed not completed, re-added, chunksFeeds count=" + chunksFeeds.Count);
}
s.TransfertDoneEvent -= new Session.TransfertDoneHandler(ManageChunkSent);
Logger.Append(Severity.DEBUG, "Session #" + s.Id + " with node #" + s.ToNode.Id + ": processed and transferred all data chunks, unused budget=" + s.Budget);
}
