private BChunk BuildIndexChunk()
{
backup.AddHubNotificationEvent(704, "", "");
BChunk iChunk = new BChunk(/*backup.TaskId, */ backup.Index.FullName, backup.Index.FullName, backup.TaskId); //string name, int bsid, string bPath, string snapPath)
try{
//iChunk.Add(FileProvider.GetFile(index.FullName));
iChunk.Add(new MinimalFsItem(backup.Index.FullName));
if (backup.DataFlags.HasFlag(DataProcessingFlags.CDedup)) // backup the deduplication database
{
iChunk.Add(ItemProvider.GetProvider().GetItemByPath(DedupIndex.Instance().IndexDBName));
}
/*string sumHash;
* using(FileStream cksumFS = new FileStream(backup.Index.FullName, FileMode.Open, FileAccess.Read)){
* sumHash = BitConverter.ToString(SHA1.Create().ComputeHash(cksumFS));
* iChunk.Sum = sumHash;
* }*/
iChunk.Sum = IndexManager.CheckSumIndex(backup.TaskId, (backup.Level != BackupLevel.Full));
// register for session received, to process index transfer
User.StorageSessionReceivedEvent += new User.StorageSessionReceivedHandler(this.SendIndex);
User.AskIndexDest(backup.TaskId, backup.Index.Name, iChunk.Sum);
Logger.Append(Severity.DEBUG, "Asked index destination to hub");
return(iChunk);
}
catch (Exception e) {
Logger.Append(Severity.ERROR, "Couldn't checksum index and/or ask destination to hub: " + e.Message + "---" + e.StackTrace);
backup.AddHubNotificationEvent(808, e.Message, "");
}
return(null);
}
// splits a big file ( size > maxchunkfile) into multiple chunks
private List <BChunk> GetBigFileChunks(IFSEntry bigFile, long filePosInChunk)
{
long pos = 0;
long remaining = bigFile.FileSize;
List <BChunk> chunks = new List <BChunk>();
while (remaining > 0)
{
chunkOrder++;
IFSEntry f = bigFile.Clone();
BChunk chunk = new BChunk(this.TaskId);
chunk.Order = chunkOrder;
chunk.RootDriveId = this.backupRootDrive.ID;
f.ChunkStartPos = 0;
chunk.Add(f);
chunks.Add(chunk);
f.FileStartPos = pos;
if (remaining > maxChunkSize)
{
pos += maxChunkSize;
}
else
{
pos += remaining;
}
remaining = bigFile.FileSize - pos;
//if(remaining <0)
Logger.Append(Severity.TRIVIA, "GetNextChunk() : splitted file " + f.SnapFullPath + " (size " + f.FileSize + ") , remaining=" + remaining + " to chunk - " + chunk.Name + " starting @ offset " + f.FileStartPos);
}
return(chunks);
}
private BChunk BuildIndexChunk()
{
backup.AddHubNotificationEvent(704, "", "");
BChunk iChunk = null;
string synthIndexFullPath = null;
if (backup.Level != P2PBackup.Common.BackupLevel.Full && backup.Level != P2PBackup.Common.BackupLevel.SnapshotOnly)
{
IndexManager idxManager = new IndexManager();
Logger.Append(Severity.INFO, "Building synthetic full index...");
synthIndexFullPath = idxManager.CreateSyntheticFullIndex(backup.ParentTrackingId, backup.Id, backup.RootDrives);
backup.AddHubNotificationEvent(707, "", "");
backup.SyntheticIndexSum = IndexManager.CheckSumIndex(backup.Id, false); // for synthetic backups
}
if (backup.Level == BackupLevel.Refresh) // backup the synth index
{
iChunk = new BChunk(backup.Id);
iChunk.Add(new MinimalFsItem(synthIndexFullPath)); // minimalitem because we only care the data, not the rest
}
else // only backup the partial index
{
iChunk = new BChunk(backup.Id); //string name, int bsid, string bPath, string snapPath)
iChunk.Add(new MinimalFsItem(backup.Index.FullName)); // minimalitem because we only care the data, not the rest
}
try{
//iChunk.Add(FileProvider.GetFile(index.FullName));
if (backup.Level != BackupLevel.SnapshotOnly && backup.BackupSet.DataFlags.HasFlag(DataProcessingFlags.CDedup)) // backup the deduplication database
{
iChunk.Add(ItemProvider.GetProvider().GetItemByPath(dedupIndex.IndexDBName));
backup.DdbSum = dedupIndex.ChecksumDdb();
}
iChunk.Sum = IndexManager.CheckSumIndex(backup.Id, (backup.Level == BackupLevel.Refresh));
if (backup.Level == BackupLevel.Full)
{
backup.SyntheticIndexSum = iChunk.Sum; // for Fulls
}
// stop waiting for sessions used for regular data transfer...
//User.SessionReady -= this.SessionReceived;
// ...but re-register for session that will transfer the index
//User.SessionReady += this.SendIndex;
StorageNeeded(new PeerSession {
TaskId = backup.Id
}, 1, true, false);
Logger.Append(Severity.DEBUG, "Asked index destination to hub");
return(iChunk);
}
catch (Exception e) {
Logger.Append(Severity.ERROR, "Couldn't checksum index and/or ask destination to hub: " + e.Message + "---" + e.StackTrace);
backup.AddHubNotificationEvent(808, e.Message, "");
}
return(null);
}
internal IEnumerable <BChunk> GetNextChunk()
{
chunkOrder++;
BChunk chunk = new BChunk(this.TaskId);
chunk.Order = chunkOrder;
chunk.RootDriveId = this.backupRootDrive.ID;
uint filePosInChunk = 0;
long currentSize = 0;
while (itemIterator.MoveNext())
{
//foreach(IFSEntry ent in GetFilesToBackup()){
IFSEntry ent = itemIterator.Current;
if (ent == null)
{
continue;
}
// 1st case : we can add more files to the chunk
if (ent.FileSize < maxChunkSize)
{
//try{
filePosInChunk += (uint)ent.FileSize;
//IFSEntry f = ent;
ent.ChunkStartPos = filePosInChunk;
ent.FileStartPos = 0;
chunk.Add(ent);
currentSize += ent.FileSize;
//Console.WriteLine("GetNextChunk() : added new file to chunk - "+itemIterator.Current.FileName);
//}
//catch(Exception e){
// Logger.Append(Severity.ERROR, "Could not add file "+itemIterator.Current.SnapFullPath+" : "+e.Message);
//}
}
//2nd case : a file is too big to fit into one chunk, split it
else
{
if (chunk.Items.Count > 0)
{
yield return(chunk);
}
/*chunk = new BChunk(currentPath.Path, snapshottedPath, this.TaskId);
* chunk.Order = chunkOrder;
* chunk.RootDriveId = this.backupRootDrive.ID;
* filePosInChunk = 0;
* currentSize = 0;*/
foreach (BChunk bigFileChunk in GetBigFileChunks(itemIterator.Current, filePosInChunk))
{
yield return(bigFileChunk);
}
chunkOrder++;
chunk = new BChunk(this.TaskId);
chunk.Order = chunkOrder;
chunk.RootDriveId = this.backupRootDrive.ID;
filePosInChunk = 0;
currentSize = 0;
}
// 3rd case : if a chunk reaches its max packSize, we create another one
if (currentSize > maxChunkSize || chunk.Items.Count > 0 && currentSize > maxPackSize
/*|| currentSize == 0 && chunk.Files.Count ==0 */
|| chunk.Items.Count > maxChunkFiles)
{
//Console.WriteLine("GetNextChunk() : chunk reached max files or max size:currentsize="+currentSize+"/"+maxChunkSize
// +",chunkfilescount="+chunk.Files.Count+"/"+maxChunkFiles);
yield return(chunk);
chunkOrder++;
chunk = new BChunk(this.TaskId);
chunk.Order = chunkOrder;
chunk.RootDriveId = this.backupRootDrive.ID;
filePosInChunk = 0;
currentSize = 0;
}
}
//4th case : // done processing file list but chunk not complete
Logger.Append(Severity.TRIVIA, "GetNextChunk() : Done gathering files inside '" + snapshottedPath + "' without reaching chunk max size. " + chunk.Items.Count + " files, " + currentSize / 1024 + "k");
//if(currentSize > 0){
yield return(chunk);
//itemIterator = GetFilesToBackup().GetEnumerator();
//yield break;
//}
}
