private BChunk BuildIndexChunk() { backup.AddHubNotificationEvent(704, "", ""); BChunk iChunk = new BChunk(/*backup.TaskId, */ backup.Index.FullName, backup.Index.FullName, backup.TaskId); //string name, int bsid, string bPath, string snapPath) try{ //iChunk.Add(FileProvider.GetFile(index.FullName)); iChunk.Add(new MinimalFsItem(backup.Index.FullName)); if (backup.DataFlags.HasFlag(DataProcessingFlags.CDedup)) // backup the deduplication database { iChunk.Add(ItemProvider.GetProvider().GetItemByPath(DedupIndex.Instance().IndexDBName)); } /*string sumHash; * using(FileStream cksumFS = new FileStream(backup.Index.FullName, FileMode.Open, FileAccess.Read)){ * sumHash = BitConverter.ToString(SHA1.Create().ComputeHash(cksumFS)); * iChunk.Sum = sumHash; * }*/ iChunk.Sum = IndexManager.CheckSumIndex(backup.TaskId, (backup.Level != BackupLevel.Full)); // register for session received, to process index transfer User.StorageSessionReceivedEvent += new User.StorageSessionReceivedHandler(this.SendIndex); User.AskIndexDest(backup.TaskId, backup.Index.Name, iChunk.Sum); Logger.Append(Severity.DEBUG, "Asked index destination to hub"); return(iChunk); } catch (Exception e) { Logger.Append(Severity.ERROR, "Couldn't checksum index and/or ask destination to hub: " + e.Message + "---" + e.StackTrace); backup.AddHubNotificationEvent(808, e.Message, ""); } return(null); }
// splits a big file ( size > maxchunkfile) into multiple chunks private List <BChunk> GetBigFileChunks(IFSEntry bigFile, long filePosInChunk) { long pos = 0; long remaining = bigFile.FileSize; List <BChunk> chunks = new List <BChunk>(); while (remaining > 0) { chunkOrder++; IFSEntry f = bigFile.Clone(); BChunk chunk = new BChunk(this.TaskId); chunk.Order = chunkOrder; chunk.RootDriveId = this.backupRootDrive.ID; f.ChunkStartPos = 0; chunk.Add(f); chunks.Add(chunk); f.FileStartPos = pos; if (remaining > maxChunkSize) { pos += maxChunkSize; } else { pos += remaining; } remaining = bigFile.FileSize - pos; //if(remaining <0) Logger.Append(Severity.TRIVIA, "GetNextChunk() : splitted file " + f.SnapFullPath + " (size " + f.FileSize + ") , remaining=" + remaining + " to chunk - " + chunk.Name + " starting @ offset " + f.FileStartPos); } return(chunks); }
private BChunk BuildIndexChunk() { backup.AddHubNotificationEvent(704, "", ""); BChunk iChunk = null; string synthIndexFullPath = null; if (backup.Level != P2PBackup.Common.BackupLevel.Full && backup.Level != P2PBackup.Common.BackupLevel.SnapshotOnly) { IndexManager idxManager = new IndexManager(); Logger.Append(Severity.INFO, "Building synthetic full index..."); synthIndexFullPath = idxManager.CreateSyntheticFullIndex(backup.ParentTrackingId, backup.Id, backup.RootDrives); backup.AddHubNotificationEvent(707, "", ""); backup.SyntheticIndexSum = IndexManager.CheckSumIndex(backup.Id, false); // for synthetic backups } if (backup.Level == BackupLevel.Refresh) // backup the synth index { iChunk = new BChunk(backup.Id); iChunk.Add(new MinimalFsItem(synthIndexFullPath)); // minimalitem because we only care the data, not the rest } else // only backup the partial index { iChunk = new BChunk(backup.Id); //string name, int bsid, string bPath, string snapPath) iChunk.Add(new MinimalFsItem(backup.Index.FullName)); // minimalitem because we only care the data, not the rest } try{ //iChunk.Add(FileProvider.GetFile(index.FullName)); if (backup.Level != BackupLevel.SnapshotOnly && backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CDedup)) // backup the deduplication database { iChunk.Add(ItemProvider.GetProvider().GetItemByPath(dedupIndex.IndexDBName)); backup.DdbSum = dedupIndex.ChecksumDdb(); } iChunk.Sum = IndexManager.CheckSumIndex(backup.Id, (backup.Level == BackupLevel.Refresh)); if (backup.Level == BackupLevel.Full) { backup.SyntheticIndexSum = iChunk.Sum; // for Fulls } // stop waiting for sessions used for regular data transfer... //User.SessionReady -= this.SessionReceived; // ...but re-register for session that will transfer the index //User.SessionReady += this.SendIndex; StorageNeeded(new PeerSession { TaskId = backup.Id }, 1, true, false); Logger.Append(Severity.DEBUG, "Asked index destination to hub"); return(iChunk); } catch (Exception e) { Logger.Append(Severity.ERROR, "Couldn't checksum index and/or ask destination to hub: " + e.Message + "---" + e.StackTrace); backup.AddHubNotificationEvent(808, e.Message, ""); } return(null); }
internal IEnumerable <BChunk> GetNextChunk() { chunkOrder++; BChunk chunk = new BChunk(this.TaskId); chunk.Order = chunkOrder; chunk.RootDriveId = this.backupRootDrive.ID; uint filePosInChunk = 0; long currentSize = 0; while (itemIterator.MoveNext()) { //foreach(IFSEntry ent in GetFilesToBackup()){ IFSEntry ent = itemIterator.Current; if (ent == null) { continue; } // 1st case : we can add more files to the chunk if (ent.FileSize < maxChunkSize) { //try{ filePosInChunk += (uint)ent.FileSize; //IFSEntry f = ent; ent.ChunkStartPos = filePosInChunk; ent.FileStartPos = 0; chunk.Add(ent); currentSize += ent.FileSize; //Console.WriteLine("GetNextChunk() : added new file to chunk - "+itemIterator.Current.FileName); //} //catch(Exception e){ // Logger.Append(Severity.ERROR, "Could not add file "+itemIterator.Current.SnapFullPath+" : "+e.Message); //} } //2nd case : a file is too big to fit into one chunk, split it else { if (chunk.Items.Count > 0) { yield return(chunk); } /*chunk = new BChunk(currentPath.Path, snapshottedPath, this.TaskId); * chunk.Order = chunkOrder; * chunk.RootDriveId = this.backupRootDrive.ID; * filePosInChunk = 0; * currentSize = 0;*/ foreach (BChunk bigFileChunk in GetBigFileChunks(itemIterator.Current, filePosInChunk)) { yield return(bigFileChunk); } chunkOrder++; chunk = new BChunk(this.TaskId); chunk.Order = chunkOrder; chunk.RootDriveId = this.backupRootDrive.ID; filePosInChunk = 0; currentSize = 0; } // 3rd case : if a chunk reaches its max packSize, we create another one if (currentSize > maxChunkSize || chunk.Items.Count > 0 && currentSize > maxPackSize /*|| currentSize == 0 && chunk.Files.Count ==0 */ || chunk.Items.Count > maxChunkFiles) { //Console.WriteLine("GetNextChunk() : chunk reached max files or max size:currentsize="+currentSize+"/"+maxChunkSize // +",chunkfilescount="+chunk.Files.Count+"/"+maxChunkFiles); yield return(chunk); chunkOrder++; chunk = new BChunk(this.TaskId); chunk.Order = chunkOrder; chunk.RootDriveId = this.backupRootDrive.ID; filePosInChunk = 0; currentSize = 0; } } //4th case : // done processing file list but chunk not complete Logger.Append(Severity.TRIVIA, "GetNextChunk() : Done gathering files inside '" + snapshottedPath + "' without reaching chunk max size. " + chunk.Items.Count + " files, " + currentSize / 1024 + "k"); //if(currentSize > 0){ yield return(chunk); //itemIterator = GetFilesToBackup().GetEnumerator(); //yield break; //} }