private void ViewChangesThreadStart()
{
uint reasonMask = Win32Api.USN_REASON_DATA_OVERWRITE |
Win32Api.USN_REASON_DATA_EXTEND |
Win32Api.USN_REASON_NAMED_DATA_OVERWRITE |
Win32Api.USN_REASON_NAMED_DATA_TRUNCATION |
Win32Api.USN_REASON_FILE_CREATE |
Win32Api.USN_REASON_FILE_DELETE |
Win32Api.USN_REASON_EA_CHANGE |
Win32Api.USN_REASON_SECURITY_CHANGE |
Win32Api.USN_REASON_RENAME_OLD_NAME |
Win32Api.USN_REASON_RENAME_NEW_NAME |
Win32Api.USN_REASON_INDEXABLE_CHANGE |
Win32Api.USN_REASON_BASIC_INFO_CHANGE |
Win32Api.USN_REASON_HARD_LINK_CHANGE |
Win32Api.USN_REASON_COMPRESSION_CHANGE |
Win32Api.USN_REASON_ENCRYPTION_CHANGE |
Win32Api.USN_REASON_OBJECT_ID_CHANGE |
Win32Api.USN_REASON_REPARSE_POINT_CHANGE |
Win32Api.USN_REASON_STREAM_CHANGE |
Win32Api.USN_REASON_CLOSE;
Win32Api.USN_JOURNAL_DATA newUsnState;
List <Win32Api.UsnEntry> usnEntries;
NtfsUsnJournal.UsnJournalReturnCode rtnCode = _usnJournal.GetUsnJournalEntries(_usnCurrentJournalState, reasonMask, out usnEntries, out newUsnState);
Dispatcher.Invoke(new FillListBoxDelegate(FillListBoxWithUsnEntries), rtnCode, usnEntries, newUsnState);
}
public void Execute(IJobExecutionContext context)
{
if (Singleton.Instance.SourceMountpoints == null || Singleton.Instance.SourceMountpoints.Count == 0)
{
return;
}
try
{
foreach (var sourceMount in Singleton.Instance.SourceMountpoints)
{
var construct = new DriveConstruct(sourceMount.MountPoint);
Win32Api.USN_JOURNAL_DATA newUsnState;
List <Win32Api.UsnEntry> usnEntries;
NtfsUsnJournal journal = new NtfsUsnJournal(construct.DriveLetter);
var drivePath = Path.Get(construct.DriveLetter);
logger.Trace("Polling for changes from " + sourceMount.CurrentUSNLocation);
var rtn = journal.GetUsnJournalEntries(construct.CurrentJournalData, reasonMask, out usnEntries, out newUsnState, OverrideLastUsn: sourceMount.CurrentUSNLocation);
if (rtn == NtfsUsnJournal.UsnJournalReturnCode.USN_JOURNAL_SUCCESS)
{
List <RawUSNEntry> entries = new List <RawUSNEntry>();
if (usnEntries.Any())
{
logger.Debug("USN returned with " + usnEntries.Count + " entries");
logger.Trace($"fsutil usn readjournal {construct.Volume} startusn={sourceMount.CurrentUSNLocation}");
}
List <USNChangeRange> changeRange = new List <USNChangeRange>();
foreach (var frn in usnEntries.Select(e => e.FileReferenceNumber).Distinct())
{
var entriesForFile = usnEntries.Where(f => f.FileReferenceNumber == frn).ToList();
if (entriesForFile.All(e => e.SourceInfo == Win32Api.UsnEntry.USNJournalSourceInfo.DataManagement || e.SourceInfo == Win32Api.UsnEntry.USNJournalSourceInfo.ReplicationManagement))
{
continue;
}
var actualPath = GetActualPath(journal, entriesForFile.FirstOrDefault());
if (actualPath == "Unavailable" || String.IsNullOrWhiteSpace(actualPath))
{
continue;
}
if (sourceMount.IgnoreList != null && sourceMount.IgnoreList.Any() && sourceMount.IgnoreList.Any(ignore => new Regex(ignore).IsMatch(actualPath)))
{
continue;
}
USNChangeRange range = new USNChangeRange
{
FRN = frn,
Min = entriesForFile.Min(e => e.TimeStamp),
Max = entriesForFile.Max(e => e.TimeStamp),
Closed = entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault() != null ? (entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault().Reason & Win32Api.USN_REASON_CLOSE) != 0 : false,
RenameFrom = entriesForFile.FirstOrDefault(e => (e.Reason & Win32Api.USN_REASON_RENAME_OLD_NAME) != 0),
Entry = entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault()
};
bool alreadyCopiedItem = entriesForFile.GroupBy(r => r.ParentFileReferenceNumber).Select(r => r.First()).Distinct().Any(pfrn => GetActualPath(journal, pfrn).Contains("\\.proximaTemp\\"));
if (alreadyCopiedItem || ShouldIgnore(actualPath, drivePath, range, journal))
{
continue;
}
changeRange.Add(range);
}
foreach (var item in changeRange)
{
var actualPath = GetActualPath(journal, item.Entry);
if (actualPath == "Unavailable")
{
continue;
}
string relativePath;
try
{
Uri drivePathUri = new Uri(drivePath.FullPath, UriKind.Absolute);
Uri actualPathUri = new Uri(actualPath, UriKind.Absolute);
relativePath = Uri.UnescapeDataString(drivePathUri.MakeRelativeUri(actualPathUri).ToString());
}
catch (Exception e)
{
relativePath = "#ERROR#";
}
if (relativePath == "#ERROR#" || relativePath.StartsWith("System Volume Information"))
{
continue;
}
string renameFromRelativePath = "";
if (item.RenameFrom != null)
{
string renameFromPath = GetActualPath(journal, ((Win32Api.UsnEntry)item.RenameFrom));
try
{
Uri drivePathUri = new Uri(drivePath.FullPath, UriKind.Absolute);
Uri actualPathUri = new Uri(actualPath, UriKind.Absolute);
renameFromRelativePath = Uri.UnescapeDataString(drivePathUri.MakeRelativeUri(actualPathUri).ToString());
}
catch (Exception e)
{
renameFromRelativePath = "";
}
}
if (!String.IsNullOrWhiteSpace(renameFromRelativePath) && renameFromRelativePath.StartsWith(".proximaTemp"))
{
continue;
}
var dbEntry = new RawUSNEntry();
PopulateFlags(dbEntry, item.Entry);
dbEntry.Path = actualPath;
dbEntry.RelativePath = relativePath;
dbEntry.File = item.Entry.IsFile;
dbEntry.Directory = item.Entry.IsFolder;
dbEntry.FRN = item.Entry.FileReferenceNumber;
dbEntry.PFRN = item.Entry.ParentFileReferenceNumber;
dbEntry.RecordLength = item.Entry.RecordLength;
dbEntry.USN = item.Entry.USN;
dbEntry.Mountpoint = sourceMount;
dbEntry.TimeStamp = item.Entry.TimeStamp.Truncate(TimeSpan.TicksPerMillisecond);
dbEntry.SourceInfo = item.Entry.SourceInfo.ToString();
dbEntry.ChangeRange = item;
if (actualPath != null && actualPath != "Unavailable" && actualPath.ToLowerInvariant().StartsWith($"{journal.MountPoint.TrimEnd('\\')}\\$".ToLowerInvariant()))
{
dbEntry.SystemFile = true;
}
if (item.RenameFrom != null)
{
dbEntry.RenameFromPath = GetActualPath(journal, ((Win32Api.UsnEntry)item.RenameFrom));
if (!string.IsNullOrWhiteSpace(dbEntry.RenameFromPath) && dbEntry.RenameFromPath != "Unavailable")
{
dbEntry.RenameFromRelativePath = new Regex(Regex.Escape(drivePath.FullPath), RegexOptions.IgnoreCase).Replace(dbEntry.RenameFromPath, "", 1);
}
}
entries.Add(dbEntry);
}
if (changeRange.Any())
{
Singleton.Instance.Repository.Add <USNChangeRange>(changeRange);
Singleton.Instance.Repository.Add <RawUSNEntry>(entries);
var performRollup = RollupService.PerformRollup(entries, sourceMount, Singleton.Instance.Repository);
logger.Info(string.Format("[{3}] Adding [{2}CHANGE/{1}USN/{0}File]", performRollup.Count, entries.Count, changeRange.Count, sourceMount.Id));
foreach (var fileAction in performRollup)
{
// logger.Trace("ADD: " + fileAction.RelativePath + ", USN:" + fileAction.USN);
}
Singleton.Instance.Repository.Add <FileAction>(performRollup);
//performRollup.ForEach(f=> logger.Debug("Added " + f.Id));
}
construct.CurrentJournalData = newUsnState;
sourceMount.CurrentUSNLocation = newUsnState.NextUsn;
sourceMount.Volume = construct.Volume;
Singleton.Instance.Repository.Update(sourceMount);
}
else
{
logger.Error("Error on Monitor - " + rtn.ToString());
throw new UsnJournalException(rtn);
}
}
}
catch (Exception e)
{
logger.Error(e, "Error in USNJournalMonitor");
}
}
private Dictionary <int, Win32Api.UsnEntry> GetUsnRecordsDictionary()
{
PrivilegesManager pm = new PrivilegesManager();
pm.Grant();
Dictionary <int, Win32Api.UsnEntry> uEntries = new Dictionary <int, Win32Api.UsnEntry>();
using (usnJ = new NtfsUsnJournal(/*brd.SystemDrive.MountPoint*/ brd /*.Snapshot.MountPoint*/)){
Logger.Append(Severity.DEBUG, "Reading USN journal " + journalId + " for '" + brd.SystemDrive.MountPoint
+ "' from seq " + prevTransactionId + " to seq " + transactionId
+ " (changed entries from " + Utilities.Utils.GetLocalDateTimeFromUnixTime(refTimeStamp).ToString()
+ " to " + Utilities.Utils.GetLocalDateTimeFromUnixTime(brd.Snapshot.TimeStamp).ToLocalTime().ToString() + ")");
Win32Api.USN_JOURNAL_DATA stateJd = new Win32Api.USN_JOURNAL_DATA();
stateJd.UsnJournalID = journalId;
stateJd.NextUsn = prevTransactionId;
Win32Api.USN_JOURNAL_DATA newState = new Win32Api.USN_JOURNAL_DATA(); // unused, as we maintain our own state
List <Win32Api.UsnEntry> changedUsnEntries = new List <Win32Api.UsnEntry>();
usnJ.GetUsnJournalState(ref newState);
NtfsUsnJournal.UsnJournalReturnCode retCode = usnJ.GetUsnJournalEntries(stateJd, refTimeStamp, 0xFFFFFFFF, out changedUsnEntries, out newState);
if (retCode != NtfsUsnJournal.UsnJournalReturnCode.USN_JOURNAL_SUCCESS)
{
throw new Exception(retCode.ToString());
}
int entryId = 0;
foreach (Win32Api.UsnEntry ue in changedUsnEntries)
{
if (ue != null && ue.Reason > 0)
{
entryId = (int)(ue.FileReferenceNumber);
//if(ue.Name.StartsWith("grut"))
//Console.WriteLine ("|--------| USN seq="+ue.USN+", item "+entryId+" ("+ue.Name+") "+((NtfsUsnJournal.UsnReasonCode)ue.Reason).ToString());
if (!uEntries.ContainsKey(entryId))
{
uEntries[entryId] = ue;
}
else // cumulate reason flags
// ignore created+deleted (temporary or short-lived (between 2 backups) items
{
if (
((Win32Api.UsnReasonCode)ue.Reason).HasFlag(Win32Api.UsnReasonCode.USN_REASON_FILE_DELETE) &&
((Win32Api.UsnReasonCode)uEntries[entryId].Reason).HasFlag(Win32Api.UsnReasonCode.USN_REASON_FILE_CREATE)
)
{
Console.WriteLine("*** item " + ue.Name + " CREATED+DELETED");
continue;
}
// file ID reused (file delete + new create) : totally replace previous entry
else if (
((Win32Api.UsnReasonCode)ue.Reason).HasFlag(Win32Api.UsnReasonCode.USN_REASON_FILE_CREATE) &&
((Win32Api.UsnReasonCode)uEntries[entryId].Reason).HasFlag(Win32Api.UsnReasonCode.USN_REASON_FILE_DELETE)
)
{
uEntries[entryId] = ue;
}
// cumulate flags
else if (!((Win32Api.UsnReasonCode)uEntries[entryId].Reason).HasFlag(((Win32Api.UsnReasonCode)ue.Reason)))
{
Win32Api.UsnReasonCode newReason = ((Win32Api.UsnReasonCode)uEntries[entryId].Reason) | ((Win32Api.UsnReasonCode)ue.Reason);
uEntries[entryId] = ue;
uEntries[entryId].Reason = (uint)newReason;
}
// only keep the last rename operation
/*if(((NtfsUsnJournal.UsnReasonCode)ue.Reason).HasFlag(NtfsUsnJournal.UsnReasonCode.USN_REASON_RENAME_NEW_NAME) ){
* Console.WriteLine ("*** item "+ue.Name+" RENAMED (reasons="+((NtfsUsnJournal.UsnReasonCode)ue.Reason).ToString());
* NtfsUsnJournal.UsnReasonCode newReason = ((NtfsUsnJournal.UsnReasonCode)entries[entryId].Reason) ;
* if(!((NtfsUsnJournal.UsnReasonCode)entries[entryId].Reason).HasFlag(NtfsUsnJournal.UsnReasonCode.USN_REASON_RENAME_NEW_NAME) )
* newReason |= NtfsUsnJournal.UsnReasonCode.USN_REASON_RENAME_NEW_NAME;
* entries[entryId] = ue;
* entries[entryId].Reason = (uint)newReason;
* }*/
}
}
}
Logger.Append(Severity.TRIVIA, "Done reading USN journal " + journalId + " for '" + brd.SystemDrive.MountPoint);
} //end using
return(uEntries);
}
public void Execute(IJobExecutionContext context)
{
if (Singleton.Instance.SourceMountpoints == null || Singleton.Instance.SourceMountpoints.Count == 0)
{
return;
}
try
{
using (Repository repo = new Repository())
{
foreach (var sourceMount in Singleton.Instance.SourceMountpoints)
{
var construct = new DriveConstruct(sourceMount.MountPoint);
Win32Api.USN_JOURNAL_DATA newUsnState;
List<Win32Api.UsnEntry> usnEntries;
NtfsUsnJournal journal = new NtfsUsnJournal(construct.DriveLetter);
var drivePath = Path.Get(construct.DriveLetter);
logger.Trace("Polling for changes from " + sourceMount.CurrentUSNLocation);
var rtn = journal.GetUsnJournalEntries(construct.CurrentJournalData, reasonMask, out usnEntries, out newUsnState, OverrideLastUsn: sourceMount.CurrentUSNLocation);
if (rtn == NtfsUsnJournal.UsnJournalReturnCode.USN_JOURNAL_SUCCESS)
{
List<RawUSNEntry> entries = new List<RawUSNEntry>();
if (usnEntries.Any())
{
logger.Debug("USN returned with " + usnEntries.Count + " entries");
}
List<USNChangeRange> changeRange = new List<USNChangeRange>();
foreach (var frn in usnEntries.Select(e=>e.FileReferenceNumber).Distinct())
{
var entriesForFile = usnEntries.Where(f => f.FileReferenceNumber == frn).ToList();
if (entriesForFile.All(e => e.SourceInfo == Win32Api.UsnEntry.USNJournalSourceInfo.DataManagement || e.SourceInfo == Win32Api.UsnEntry.USNJournalSourceInfo.ReplicationManagement))
{
continue;
}
var actualPath = GetActualPath(journal, entriesForFile.FirstOrDefault());
if (actualPath == "Unavailable" || String.IsNullOrWhiteSpace(actualPath) )
{
continue;
}
if (sourceMount.IgnoreList != null && sourceMount.IgnoreList.Any() && sourceMount.IgnoreList.Any(ignore => new Regex(ignore).IsMatch(actualPath)))
{
continue;
}
USNChangeRange range = new USNChangeRange
{
FRN = frn,
Min = entriesForFile.Min(e => e.TimeStamp),
Max = entriesForFile.Max(e => e.TimeStamp),
Closed = entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault() != null ? (entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault().Reason & Win32Api.USN_REASON_CLOSE) != 0 : false,
RenameFrom = entriesForFile.FirstOrDefault(e => (e.Reason & Win32Api.USN_REASON_RENAME_OLD_NAME) != 0),
Entry = entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault()
};
changeRange.Add(range);
}
//logger.Trace("ChangeRange : " + changeRange.Count);
foreach (var item in changeRange)
{
var actualPath = GetActualPath(journal, item.Entry as Win32Api.UsnEntry );
if (actualPath == "Unavailable")
{
continue;
}
string relativePath;
try
{
Uri drivePathUri = new Uri(drivePath.FullPath, UriKind.Absolute);
Uri actualPathUri = new Uri(actualPath, UriKind.Absolute);
relativePath = Uri.UnescapeDataString(drivePathUri.MakeRelativeUri(actualPathUri).ToString());
}
catch (Exception e)
{
relativePath = "#ERROR#";
}
if (relativePath == "#ERROR#" || relativePath.StartsWith("System Volume Information"))
{
continue;
}
var itemMin = item.Min.Truncate(TimeSpan.TicksPerMillisecond);
var itemMax = item.Max.Truncate(TimeSpan.TicksPerMillisecond);
var count = repo.Count<USNJournalSyncLog>(f =>
f.Action.RelativePath == relativePath && f.DestinationMachine == Singleton.Instance.CurrentServer &&
(
(
(f.ActionStartDate.HasValue && f.ActionStartDate <= itemMin)
&&
(f.ActionFinishDate.HasValue && f.ActionFinishDate >= itemMax)
)
||
(
(
( ( f.ActionStartDate.HasValue && f.ActionStartDate >= itemMin) && f.ActionFinishDate.HasValue && f.ActionFinishDate <= itemMin )
&&
(f.ActionFinishDate.HasValue && f.ActionFinishDate >= itemMax)
)
||
(
(f.ActionStartDate.HasValue && f.ActionStartDate <= itemMin)
&&
((f.ActionFinishDate.HasValue && f.ActionFinishDate <= itemMax) && f.ActionStartDate.HasValue && f.ActionStartDate >= itemMin )
)
||
(
(f.ActionStartDate.HasValue && f.ActionStartDate >= itemMin)
&&
(f.ActionFinishDate.HasValue && f.ActionFinishDate <= itemMax)
)
)
)
);
if (count > 0)
{
//logger.Info("Count is " + count);
continue;
}
var dbEntry = new RawUSNEntry();
PopulateFlags(dbEntry, item.Entry);
dbEntry.Path = actualPath;
dbEntry.RelativePath = relativePath;
dbEntry.File = item.Entry.IsFile;
dbEntry.Directory = item.Entry.IsFolder;
dbEntry.FRN = item.Entry.FileReferenceNumber;
dbEntry.PFRN = item.Entry.ParentFileReferenceNumber;
dbEntry.RecordLength = item.Entry.RecordLength;
dbEntry.USN = item.Entry.USN;
dbEntry.Mountpoint = sourceMount;
dbEntry.TimeStamp = item.Entry.TimeStamp.Truncate(TimeSpan.TicksPerMillisecond);
dbEntry.SourceInfo = item.Entry.SourceInfo.ToString();
dbEntry.ChangeRange = item;
if ( actualPath != null && actualPath != "Unavailable" && actualPath.ToLowerInvariant().StartsWith($"{journal.MountPoint.TrimEnd('\\')}\\$".ToLowerInvariant()))
{
dbEntry.SystemFile = true;
}
if (item.RenameFrom != null)
{
dbEntry.RenameFromPath = GetActualPath(journal, ((Win32Api.UsnEntry) item.RenameFrom));
if (!string.IsNullOrWhiteSpace(dbEntry.RenameFromPath ) && dbEntry.RenameFromPath != "Unavailable")
{
dbEntry.RenameFromRelativePath = new Regex(Regex.Escape(drivePath.FullPath), RegexOptions.IgnoreCase).Replace(dbEntry.RenameFromPath, "", 1);
}
}
entries.Add(dbEntry);
}
if (changeRange.Any())
{
repo.Add<USNChangeRange>(changeRange);
repo.Add<RawUSNEntry>(entries);
var performRollup = RollupService.PerformRollup(entries, sourceMount, Singleton.Instance.Repository);
logger.Info(string.Format("Adding [{2}CHANGE/{1}USN/{0}File]", performRollup.Count, entries.Count, changeRange.Count));
foreach (var fileAction in performRollup)
{
// logger.Trace("ADD: " + fileAction.RelativePath + ", USN:" + fileAction.USN);
}
repo.Add<FileAction>(performRollup);
//performRollup.ForEach(f=> logger.Debug("Added " + f.Id));
}
construct.CurrentJournalData = newUsnState;
sourceMount.CurrentUSNLocation = newUsnState.NextUsn;
sourceMount.Volume = construct.Volume;
repo.Update(sourceMount);
}
else
{
logger.Error("Error on Monitor - " + rtn.ToString());
throw new UsnJournalException(rtn);
}
}
}
}
catch (Exception e)
{
logger.Error(e, "Error in USNJournalMonitor");
}
}
public void BeginScan()
{
//clear
parentFileReferenceIdentifiers.Clear();
USNEntries.Clear();
USNDirectories.Clear();
usnCurrentJournalState = new Win32Api.USN_JOURNAL_DATA();
//1 phase; handle
try
{
usnJournal = new NtfsUsnJournal(selectedVolume);
OnEntryAmountUpdate(true);
}
catch (Exception)
{
OnEntryAmountUpdate(false);
return;
}
//2 phase; current state
Win32Api.USN_JOURNAL_DATA journalState = new Win32Api.USN_JOURNAL_DATA();
NtfsUsnJournal.UsnJournalReturnCode rtn = usnJournal.GetUsnJournalState(ref journalState);
if (rtn == NtfsUsnJournal.UsnJournalReturnCode.USN_JOURNAL_SUCCESS)
{
usnCurrentJournalState = journalState;
OnEntryAmountUpdate(true);
}
else
{
OnEntryAmountUpdate(false);
return;
}
//3 phase; query
uint reasonMask = Win32Api.USN_REASON_DATA_OVERWRITE |
Win32Api.USN_REASON_DATA_EXTEND |
Win32Api.USN_REASON_NAMED_DATA_OVERWRITE |
Win32Api.USN_REASON_NAMED_DATA_TRUNCATION |
Win32Api.USN_REASON_FILE_CREATE |
Win32Api.USN_REASON_FILE_DELETE |
Win32Api.USN_REASON_EA_CHANGE |
Win32Api.USN_REASON_SECURITY_CHANGE |
Win32Api.USN_REASON_RENAME_OLD_NAME |
Win32Api.USN_REASON_RENAME_NEW_NAME |
Win32Api.USN_REASON_INDEXABLE_CHANGE |
Win32Api.USN_REASON_BASIC_INFO_CHANGE |
Win32Api.USN_REASON_HARD_LINK_CHANGE |
Win32Api.USN_REASON_COMPRESSION_CHANGE |
Win32Api.USN_REASON_ENCRYPTION_CHANGE |
Win32Api.USN_REASON_OBJECT_ID_CHANGE |
Win32Api.USN_REASON_REPARSE_POINT_CHANGE |
Win32Api.USN_REASON_STREAM_CHANGE |
Win32Api.USN_REASON_CLOSE;
OldestUSN = usnCurrentJournalState.FirstUsn;
NtfsUsnJournal.UsnJournalReturnCode rtnCode = usnJournal.GetUsnJournalEntries(usnCurrentJournalState, reasonMask, out List <Win32Api.UsnEntry> usnEntries, out usnCurrentJournalState);
if (rtnCode == NtfsUsnJournal.UsnJournalReturnCode.USN_JOURNAL_SUCCESS)
{
OnEntryAmountUpdate(true);
//4 phase
ResolveIdentifiers(usnEntries);
OnEntryAmountUpdate(true);
//5 phase
AddEntries(usnEntries);
OnEntryAmountUpdate(true);
OnWorkEnded();
}
else
{
OnEntryAmountUpdate(false);
return;
}
}
