public IEnumerable <IFSEntry> GetNextEntry(BasePath path, string snapshottedPath)
{
// http://www.tummy.com/journals/entries/jafo_20101101_193113
fsprov = FSEnumeratorProvider.GetFSEnumeratorProvider().GetFSEnumerator(snapshottedPath);
yield return(prov.GetItemByPath(snapshottedPath));
}
/// <summary>
/// Gets the next incremental/diff entry to backup. To efficiently reuse existing code, we instanciate
/// a "phantom" BackupRootDriveHandler an use its "GetFull()" FS items provider as an items source to check
/// for modifications.
/// </summary>
/// <returns>
/// and IFSEntry
/// </returns>
/// <param name='path'>
/// Path.
/// </param>
/// <param name='snapshottedPath'>
/// Snapshotted path.
/// </param>
public IEnumerable <IFSEntry> GetNextEntry(BasePath path, string snapshottedPath)
{
idsToWatch = new List <long>();
fsProv = FSEnumeratorProvider.GetFSEnumeratorProvider().GetFSEnumerator(snapshottedPath);
phantomBrd = new BackupRootDriveHandler(rootDrive, this.taskId, 0, 0, 0, BackupLevel.Full, 0, 0, 0);
phantomBrd.SetCurrentPath(path);
phantomBrd.SubCompletionEvent += new BackupRootDriveHandler.SubCompletionHandler(BubbleUpSubCompletion);
bool moveRefEnumerator = true;
IFSEntry refEntry = null;
IFSEntry realRefEntry = null;
foreach (IFSEntry entry in phantomBrd.GetFull(true))
{
Console.WriteLine("PhantomBrd current entry : " + entry.Name);
// New File (inode/id > last refMaxId known inode/id)
if (entry.ID > refMaxId /*|| entry.CreateTime > refBackupTimeStart*/)
{
Console.WriteLine("Incremental GetFilesToBackup() added NEW entry : " + entry.SnapFullPath);
yield return(entry);
continue;
}
try{ //TOREMOVE
if (moveRefEnumerator || refEntry == null)
{
if (idxEnumerator.MoveNext()) // try to position to the same entry, previously backuped
/*if(refEntry != null && refEntry.ID == idxEnumerator.Current.ID){
* while(idxEnumerator.Current.ID == refEntry.ID) // in case we meet BigFiles (multiple same id entries), loop.
* if(!idxEnumerator.MoveNext())
* break;
* }*/
//else
{
refEntry = idxEnumerator.Current;
Console.WriteLine("Ref entry : " + refEntry.Name);
}
else
{
Console.WriteLine("CANNOT MoveNext() -- last entry is " + refEntry.Name);
}
//firstMove = false;
}
else
{
moveRefEnumerator = true;
Console.WriteLine("moveRef = false");
}
}
catch (Exception e) { // TOREMOVE
Console.WriteLine("Incremental GetFilesToBackup() ERROR : " + e.ToString());
}
//Console.WriteLine("Incremental GetFilesToBackup() ------- Refentry id="+refEntry.ID+", name="+refEntry.Name+", curid="+entry.ID+", curname="+entry.Name);
//Console.WriteLine ("RefEntry: "+refEntry.ToString()+", curEntry: "+entry.ToString());
//if(refEntry.ID == entry.ID)
// Console.WriteLine("Incremental GetFilesToBackup() entry "+entry.SnapFullPath+" __MATCHED__ in ref backup");
if (refEntry.ID != entry.ID)
{
Console.WriteLine("Incremental cur entry " + entry.ToString() + " DOESN'T_MATCH ref = " + refEntry.ToString());
long refItemPos = 0;
// new file potentially reusing an inode/ID number < ref max ID
if (entry.CreateTime > refBackupTimeStart || entry.LastModifiedTime > refBackupTimeStart)
{
moveRefEnumerator = false;
yield return(entry);
continue;
}
//other cases : File RenamedOrMovedItem RenamedOrMovedItem deleted
realRefEntry = refIndex.SearchItem(entry, rootDrive.SystemDrive.MountPoint, out refItemPos);
if (realRefEntry == null)
{
moveRefEnumerator = false;
yield return(entry);
continue;
}
// check if entry has been moved from outside to the current dir
else
{
refEntry = realRefEntry;
if (refEntry.ParentID != entry.ParentID) // moved
{
moveRefEnumerator = false;
//entry.ChangeStatus =
//continue;
}
}
/*// Check if entry is a newly created file (ctime > last backup start time) but with an ''old'' mtime
* if(refIndex.SearchItem(entry, rootDrive.SystemDrive.MountPoint, out searchRowid) == null
* && entry.LastMetadataModifiedTime > refBackupTimeStart){
* Console.WriteLine("Found (new?) entry with reused ID : "+entry.ToString());
*
* }*/
// else check if refEntry has been deleted
/*else{
* Console.WriteLine("Incremental GetFilesToBackup() entry "+refEntry.ToString()+" __DELETED__");
* refEntry.ChangeStatus = DataLayoutInfos.Deleted;
* yield return refEntry;
* continue;
*
* }*/
/*if(idsToWatch.Contains(entry.ID)){ // we found it! is was simply moved
* Logger.Append (Severity.DEBUG2, "Found wanted entry "+entry.ID+", "+entry.SnapFullPath);
* for(int i=idsToWatch.Count-1; i==0; i--)
* if(idsToWatch[i] == entry.ID)
* idsToWatch.RemoveAt(i);
* }*/
//Console.WriteLine("Incremental GetFilesToBackup() entry "+entry.SnapFullPath+"("+entry.ID+") DOES NOT MATCH, got "+refEntry.Name+" ("+refEntry.ID+")");
/*long searchRowid = 0;
* if((refEntry = refIndex.SearchItem(entry, rootDrive.SystemDrive.MountPoint, out searchRowid)) != null){
* long fsToIndexOffset = searchRowid - refIndex.RowId;
* Console.WriteLine("Incremental GetFilesToBackup() Found ref entry "+entry.SnapFullPath+" at __OFFSET__="+(searchRowid - refIndex.RowId));
* if( fsToIndexOffset < 100){
*
* for (int j = (int)fsToIndexOffset; j >0; j--){
* if(idxEnumerator.MoveNext())
* idsToWatch.Add(idxEnumerator.Current.ID);
* }
* }
* //if current FS entry and ref entry mismatch, but current FS entry exists in ref index ,
* // we put this ref entry on the 'to watch' list :
* // It may indicate that ref has been deleted, or, if we meet this 'watched' id during backup, that
* // the item has been moved
* else{
* //idsToWatch.Add(refEntry.ID);
* idxEnumerator.Dispose();
* idxEnumerator = refIndex.GetBaseItemsEnumerator(rootDrive.SystemDrive.OriginalMountPoint, refIndex.RowId).GetEnumerator();
* idxEnumerator.MoveNext();
* Console.WriteLine ("moved ref index enumerator to new root "+idxEnumerator.Current.ID+", name="+idxEnumerator.Current.Name);
* }
* }*/
/*else{ // new entry reusing "old" inode number/ID
* Console.WriteLine("\t NOT found matching ID for name="+entry.Name);
* yield return entry;
* // this new entry could explain the offset we get between current fs and ref index
* moveRefEnumerator = false;
* continue;
* }*/
}
//refEntry = srch;
// Entry already existed under the same id. if it also has the same name, continue checks.
// else, it may have been (1)renamed, or (2)deleted + inode reused for new file.
// (1) it it safe to assume it as only renamed if lastmetadata has changed but data hasn't.
// (2) if oldname!=newname, and lastwritetime has change, we cannot decide what happened. Consider
// it as a new entry, for safety.
//Console.WriteLine("\t Search found matching ID, ref name="+srch.Name);
/*if(srch.Name != entry.Name){
* Console.WriteLine("Incremental GetFilesToBackup() added RENAMED entry : "+entry.SnapFullPath);
* entry.BlockMetadata.BlockMetadata.Add(new RenamedOrMovedItem());
* yield return entry;
* continue;
* }*/
// Existing entry with modified data since ref backup
// lastmod < ref lastmod : rename(?). lastmod < ref lastmod : entry data modified
if (entry.LastModifiedTime != /*refBackupTimeStart*/ refEntry.LastModifiedTime ||
entry.LastMetadataModifiedTime != refEntry.LastMetadataModifiedTime)
{
Console.WriteLine("Incremental GetFilesToBackup() added MODIFIED (LastModifiedTime) entry : " + entry.SnapFullPath + ", entry.LastModifiedTime=" + entry.LastModifiedTime + ",refEntry.LastModifiedTime=" + refEntry.LastModifiedTime);
entry.ChangeStatus = DataLayoutInfos.HasChanges;
yield return(entry);
continue;
}
/*else if(entry.LastMetadataModifiedTime > refEntry.LastMetadataModifiedTime){
* entry.ChangeStatus = DataLayoutInfos.MetadaOnly;
* // moved entry
* //if(srch.ParentID != entry.ParentID || srch.Name != entry.Name){
* //entry.BlockMetadata.BlockMetadata.Add(new RenamedOrMovedItem());
* entry.ChangeStatus = DataLayoutInfos.MetadaOnly; // .RenameOnly;
* //}
* //else
* // entry.BlockMetadata.BlockMetadata.Add(new UnchangedDataItem());
* Console.WriteLine("Incremental GetFilesToBackup() added METADATACHANGE (LastMetadataModifiedTime) entry : "+entry.SnapFullPath);
* yield return entry;
* continue;
* }*/
// if we get there, entry hasn't changed.
//Console.WriteLine("Incremental GetFilesToBackup UNCHANGED "+entry.Name);
// }
// If we get there, FS entry didn't change since last backup.
// But if we are asked to perform a full refresh, return it anyway, with appropriate ChangeFlag
if (this.isFullRefreshBackup)
{
entry.ChangeStatus = DataLayoutInfos.NoChange;
yield return(entry);
}
} // // end foreach phantomBrdh
Console.WriteLine("FileCompareProvider GetNextEntry : end foreach");
// Fs has been enumerated. Now check if some ref index IDs remain unfound (== present inside idsToWatch)
// If so, either they have been deleted, or they have been moved out of scope (out of FS backup root directory)
// Anyways, we tag them as 'deleted', as, from the backup's root folder point of view, they are not part of
// the backup anymore.
if (idsToWatch.Count > 0)
{
Logger.Append(Severity.DEBUG, idsToWatch.Count + " entries seem to have been deleted.");
}
foreach (long id in idsToWatch)
{
long useless = 0;
IFSEntry deleted = refIndex.SearchItem(id, rootDrive.SystemDrive.MountPoint, out useless);
if (deleted != null)
{
Console.WriteLine("\t entry " + id + " has been deleted, was " + deleted.Name);
deleted.ChangeStatus = DataLayoutInfos.Deleted;
yield return(deleted);
}
else
{
Console.WriteLine("\t entry " + id + " has NOT BEEN DELETED - ERROR!!!");
}
}
yield break;
}
/// <summary>
/// Gets the next incremental/diff entry to backup. To efficiently reuse existing code, we instanciate
/// a "phantom" BackupRootDriveHandler an use its "GetFull()" FS items provider as an items source to check
/// for modifications.
/// </summary>
/// <returns>
/// and IFSEntry
/// </returns>
/// <param name='path'>
/// Path.
/// </param>
/// <param name='snapshottedPath'>
/// Snapshotted path.
/// </param>
public IEnumerable <IFSEntry> GetNextEntry(BasePath path, string snapshottedPath)
{
idsToWatch = new List <long>();
fsProv = FSEnumeratorProvider.GetFSEnumeratorProvider().GetFSEnumerator(snapshottedPath);
phantomBrd = new BackupRootDriveHandler(rootDrive, this.taskId, 0, 0, 0, BackupLevel.Full, 0, 0, 0);
phantomBrd.SetCurrentPath(path);
phantomBrd.SubCompletionEvent += new BackupRootDriveHandler.SubCompletionHandler(BubbleUpSubCompletion);
bool moveRefEnumerator = true;
IFSEntry refEntry = null;
IFSEntry realRefEntry = null;
foreach (IFSEntry entry in phantomBrd.GetFull(true))
{
Console.WriteLine("PhantomBrd current entry : " + entry.Name);
// New File (inode/id > last refMaxId known inode/id)
if (entry.ID > refMaxId /*|| entry.CreateTime > refBackupTimeStart*/)
{
Console.WriteLine("Incremental GetFilesToBackup() added NEW entry : " + entry.SnapFullPath);
yield return(entry);
continue;
}
//try{ //TOREMOVE
if (moveRefEnumerator || refEntry == null)
{
if (idxEnumerator.MoveNext()) // try to position to the same entry, previously backuped
/*if(refEntry != null && refEntry.ID == idxEnumerator.Current.ID){
* while(idxEnumerator.Current.ID == refEntry.ID) // in case we meet BigFiles (multiple same id entries), loop.
* if(!idxEnumerator.MoveNext())
* break;
* }*/
//else
{
refEntry = idxEnumerator.Current;
Console.WriteLine("Ref entry : " + refEntry.Name);
}
else
{
Console.WriteLine("CANNOT MoveNext() -- last entry is " + refEntry.Name);
}
//firstMove = false;
}
else
{
moveRefEnumerator = true;
Console.WriteLine("moveRef = false");
}
/*}
* catch(Exception e){ // TOREMOVE
* Console.WriteLine ("Incremental GetFilesToBackup() ERROR : "+e.ToString());
* }*/
//Console.WriteLine("Incremental GetFilesToBackup() ------- Refentry id="+refEntry.ID+", name="+refEntry.Name+", curid="+entry.ID+", curname="+entry.Name);
//Console.WriteLine ("RefEntry: "+refEntry.ToString()+", curEntry: "+entry.ToString());
//if(refEntry.ID == entry.ID)
// Console.WriteLine("Incremental GetFilesToBackup() entry "+entry.SnapFullPath+" __MATCHED__ in ref backup");
if (refEntry.ID != entry.ID)
{
Console.WriteLine("Incremental cur entry " + entry.ToString() + " DOESN'T_MATCH ref = " + refEntry.ToString());
long refItemPos = 0;
// new file potentially reusing an inode/ID number < ref max ID
if (entry.CreateTime > refBackupTimeStart || entry.LastModifiedTime > refBackupTimeStart)
{
moveRefEnumerator = false;
yield return(entry);
continue;
}
//other cases : File RenamedOrMovedItem RenamedOrMovedItem deleted
realRefEntry = refIndex.SearchItem(entry, rootDrive.SystemDrive.MountPoint, out refItemPos);
if (realRefEntry == null)
{
moveRefEnumerator = false;
yield return(entry);
continue;
}
// check if entry has been moved from outside to the current dir
else
{
refEntry = realRefEntry;
if (refEntry.ParentID != entry.ParentID) // moved
{
moveRefEnumerator = false;
//entry.ChangeStatus =
//continue;
}
}
}
//refEntry = srch;
// Entry already existed under the same id. if it also has the same name, continue checks.
// else, it may have been (1)renamed, or (2)deleted + inode reused for new file.
// (1) it it safe to assume it as only renamed if lastmetadata has changed but data hasn't.
// (2) if oldname!=newname, and lastwritetime has change, we cannot decide what happened. Consider
// it as a new entry, for safety.
//Console.WriteLine("\t Search found matching ID, ref name="+srch.Name);
/*if(srch.Name != entry.Name){
* Console.WriteLine("Incremental GetFilesToBackup() added RENAMED entry : "+entry.SnapFullPath);
* entry.BlockMetadata.BlockMetadata.Add(new RenamedOrMovedItem());
* yield return entry;
* continue;
* }*/
// Existing entry with modified data since ref backup
// lastmod < ref lastmod : rename(?). lastmod < ref lastmod : entry data modified
if (entry.LastModifiedTime != /*refBackupTimeStart*/ refEntry.LastModifiedTime ||
entry.FileSize != refEntry.FileSize)
{
Console.WriteLine("Incremental GetFilesToBackup() added MODIFIED (LastModifiedTime) entry : " + entry.SnapFullPath + ", entry.LastModifiedTime=" + entry.LastModifiedTime + ",refEntry.LastModifiedTime=" + refEntry.LastModifiedTime);
entry.ChangeStatus = DataLayoutInfos.HasChanges;
yield return(entry);
continue;
}
else if (entry.LastMetadataModifiedTime > refEntry.LastMetadataModifiedTime)
{
entry.ChangeStatus = DataLayoutInfos.MetadaOnly;
// moved entry
//if(srch.ParentID != entry.ParentID || srch.Name != entry.Name){
//entry.BlockMetadata.BlockMetadata.Add(new RenamedOrMovedItem());
//entry.ChangeStatus = DataLayoutInfos.MetadaOnly; // .RenameOnly;
//}
//else
// entry.BlockMetadata.BlockMetadata.Add(new UnchangedDataItem());
Console.WriteLine("Incremental GetFilesToBackup() added METADATACHANGE (LastMetadataModifiedTime) entry : " + entry.SnapFullPath);
yield return(entry);
continue;
}
// if we get there, entry hasn't changed.
//Console.WriteLine("Incremental GetFilesToBackup UNCHANGED "+entry.Name);
// }
// If we get there, FS entry didn't change since last backup.
// But if we are asked to perform a full refresh, return it anyway, with appropriate ChangeFlag
entry.ChangeStatus = DataLayoutInfos.NoChange;
yield return(entry);
} // // end foreach phantomBrdh
Console.WriteLine("FileCompareProvider GetNextEntry : end foreach");
// Fs has been enumerated. Now check if some ref index IDs remain unfound (== present inside idsToWatch)
// If so, either they have been deleted, or they have been moved out of scope (out of FS backup root directory)
// Anyways, we tag them as 'deleted', as, from the backup's root folder point of view, they are not part of
// the backup anymore.
if (idsToWatch.Count > 0)
{
Logger.Append(Severity.DEBUG, idsToWatch.Count + " entries seem to have been deleted.");
}
foreach (long id in idsToWatch)
{
long useless = 0;
IFSEntry deleted = refIndex.SearchItem(id, rootDrive.SystemDrive.MountPoint, out useless);
if (deleted != null)
{
Console.WriteLine("\t entry " + id + " has been deleted, was " + deleted.Name);
deleted.ChangeStatus = DataLayoutInfos.Deleted;
yield return(deleted);
}
else
{
Console.WriteLine("\t entry " + id + " has NOT BEEN DELETED - ERROR!!!");
}
}
yield break;
}
public IEnumerable <IFSEntry> GetNextEntry(BasePath path, string snapshottedPath)
{
//Console.WriteLine("Incremental GetFilesToBackup() 1");
fsProv = FSEnumeratorProvider.GetFSEnumeratorProvider().GetFSEnumerator(snapshottedPath);
//Console.WriteLine("Incremental GetFilesToBackup() 2");
bool exclude = false;
foreach (var item in fsProv)
{
//Console.WriteLine("Incremental GetFilesToBackup() 3");
IFSEntry entry = null;
IFSEntry refEntry = null;
try{
entry = prov.GetItem(item);
//Console.WriteLine ("FileCompareProvider.GetNextEntry() : "+entry.SnapshottedFullPath);
}
catch (Exception e) { // permission errors, deleted file...
Logger.Append(Severity.ERROR, "Could not add element '" + entry.SnapFullPath + "' to backup : " + e.Message);
Logger.Append(Severity.INFO, "TODO : report exception to hub for task logentry");
continue;
}
//Console.WriteLine("Incremental GetFilesToBackup() 4");
if (entry.ID > refMaxId || entry.CreateTime > refBackupTimeStart) // new File (inode/id > last refMaxId known inode/id)
{
Console.WriteLine("Incremental GetFilesToBackup() added NEW entry : " + entry.SnapFullPath);
yield return(entry);
continue; /// @@@@@@@@@@@@@@@ TO REMOVE to allow "if(entry.Kind == FileType.Directory){" to execute????
}
try{
/*else */ if (idxEnumerator.MoveNext()) // try to position to the same entry, previously backuped
{
refEntry = idxEnumerator.Current;
}
//Console.WriteLine("Incremental GetFilesToBackup() 5");
if (refEntry.ID == entry.ID)
{
Console.WriteLine("Incremental GetFilesToBackup() entry " + entry.SnapFullPath + " MATCHED in ref backup");
}
else
{
Console.WriteLine("Incremental GetFilesToBackup() entry " + entry.SnapFullPath + "(" + entry.ID + ") DOES NOT MATCH, got " + refEntry.Name + " (" + refEntry.ID + ")");
IFSEntry srch;
if ((srch = refIndex.SearchItem(entry, rootDrive.systemDrive.MountPoint)) != null)
{
Console.WriteLine("\t found matching ID, ref name=" + srch.Name);
}
else
{
Console.WriteLine("\t NOT found matching ID, ref name=" + entry.Name);
}
}
}
catch (Exception e) {
Console.WriteLine(" *** ERROR : " + e.ToString());
}
// entry modified (or created using a used-and-freed id)
if (entry.LastModifiedTime >= refBackupTimeStart)
{
//if(entry.Kind == FileType.Directory)
//Console.WriteLine("root path for "+entry.FileName+"="+Directory.GetDirectoryRoot(entry.FileName)+", rooted="+Path.IsPathRooted(entry.FileName));
Console.WriteLine("Incremental GetFilesToBackup() added MODIFIED (LastModifiedTime) entry : " + entry.SnapFullPath);
yield return(entry);
}
else if (entry.LastMetadataModifiedTime >= refBackupTimeStart)
{
// We will do our best to try to find if it's a renamed file
// on *nix, a file is renamed if its inode stays the same, but the ctime and filename changes and the mtime does nots
// if mtime and ctime change and inode is an already previously existing number, a file might have
// been deleted and another created, taking the freed inode number.
// This makes impossible for us to detect files renamed AND modifed, or modified AND renamed
// on NTFS a file has a unique ID, but it is hard to get from direftory.enumerate (does not return the right structure)
// add the file to the "to be checked" list
Console.WriteLine("Incremental GetFilesToBackup() : item " + entry.SnapFullPath + " has undergone METADATA (LastMetadataModifiedTime) change");
//itemsToCompare.Add(entry);
}
// check for new files with metadata dates in the past (such as packages installations...)
//else if(entry.ID >
if (entry.Kind == FileType.Directory)
{
//if(depth <2)
//Console.WriteLine ("FileCompareProvider.GetNextEntry() : entering "+entry.SnapshottedFullPath);
for (int i = path.ExcludedPaths.Count - 1; i >= 0; i--)
{
if (entry.SnapFullPath.IndexOf(path.ExcludedPaths[i]) == 0)
{
Logger.Append(Severity.INFO, "Ignoring path " + entry.SnapFullPath);
path.ExcludedPaths.RemoveAt(i);
exclude = true;
//yield return entry;
break;
}
}
depth++;
if (depth == 1)
{
//if(SubCompletionEvent != null) SubCompletionEvent(entry.SnapshottedFullPath.Replace(snapshottedPath, currentPath.Path));
//entry.FileName.Replace(snapshottedPath, currentPath
}
//snapshottedPath = entry.SnapshottedFullPath;
if (!exclude)
{
// recurse using found directory as basepath
snapshottedPath = entry.SnapFullPath;
yield return(entry); // return top-dir before entering and browse it
foreach (IFSEntry e in GetNextEntry(path, snapshottedPath))
{
yield return(e);
}
}
depth--;
}
//yield return entry;
exclude = false;
//}
}
//Logger.Append(Severity.DEBUG, "Done scanning file system, performing index compare...");
//IndexCompare();
}
