public void Execute(IJobExecutionContext context) { if (Singleton.Instance.SourceMountpoints == null || Singleton.Instance.SourceMountpoints.Count == 0) { return; } try { foreach (var sourceMount in Singleton.Instance.SourceMountpoints) { var construct = new DriveConstruct(sourceMount.MountPoint); Win32Api.USN_JOURNAL_DATA newUsnState; List <Win32Api.UsnEntry> usnEntries; NtfsUsnJournal journal = new NtfsUsnJournal(construct.DriveLetter); var drivePath = Path.Get(construct.DriveLetter); logger.Trace("Polling for changes from " + sourceMount.CurrentUSNLocation); var rtn = journal.GetUsnJournalEntries(construct.CurrentJournalData, reasonMask, out usnEntries, out newUsnState, OverrideLastUsn: sourceMount.CurrentUSNLocation); if (rtn == NtfsUsnJournal.UsnJournalReturnCode.USN_JOURNAL_SUCCESS) { List <RawUSNEntry> entries = new List <RawUSNEntry>(); if (usnEntries.Any()) { logger.Debug("USN returned with " + usnEntries.Count + " entries"); logger.Trace($"fsutil usn readjournal {construct.Volume} startusn={sourceMount.CurrentUSNLocation}"); } List <USNChangeRange> changeRange = new List <USNChangeRange>(); foreach (var frn in usnEntries.Select(e => e.FileReferenceNumber).Distinct()) { var entriesForFile = usnEntries.Where(f => f.FileReferenceNumber == frn).ToList(); if (entriesForFile.All(e => e.SourceInfo == Win32Api.UsnEntry.USNJournalSourceInfo.DataManagement || e.SourceInfo == Win32Api.UsnEntry.USNJournalSourceInfo.ReplicationManagement)) { continue; } var actualPath = GetActualPath(journal, entriesForFile.FirstOrDefault()); if (actualPath == "Unavailable" || String.IsNullOrWhiteSpace(actualPath)) { continue; } if (sourceMount.IgnoreList != null && sourceMount.IgnoreList.Any() && sourceMount.IgnoreList.Any(ignore => new Regex(ignore).IsMatch(actualPath))) { continue; } USNChangeRange range = new USNChangeRange { FRN = frn, Min = entriesForFile.Min(e => e.TimeStamp), Max = entriesForFile.Max(e => e.TimeStamp), Closed = entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault() != null ? (entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault().Reason & Win32Api.USN_REASON_CLOSE) != 0 : false, RenameFrom = entriesForFile.FirstOrDefault(e => (e.Reason & Win32Api.USN_REASON_RENAME_OLD_NAME) != 0), Entry = entriesForFile.OrderBy(f => f.TimeStamp).LastOrDefault() }; bool alreadyCopiedItem = entriesForFile.GroupBy(r => r.ParentFileReferenceNumber).Select(r => r.First()).Distinct().Any(pfrn => GetActualPath(journal, pfrn).Contains("\\.proximaTemp\\")); if (alreadyCopiedItem || ShouldIgnore(actualPath, drivePath, range, journal)) { continue; } changeRange.Add(range); } foreach (var item in changeRange) { var actualPath = GetActualPath(journal, item.Entry); if (actualPath == "Unavailable") { continue; } string relativePath; try { Uri drivePathUri = new Uri(drivePath.FullPath, UriKind.Absolute); Uri actualPathUri = new Uri(actualPath, UriKind.Absolute); relativePath = Uri.UnescapeDataString(drivePathUri.MakeRelativeUri(actualPathUri).ToString()); } catch (Exception e) { relativePath = "#ERROR#"; } if (relativePath == "#ERROR#" || relativePath.StartsWith("System Volume Information")) { continue; } string renameFromRelativePath = ""; if (item.RenameFrom != null) { string renameFromPath = GetActualPath(journal, ((Win32Api.UsnEntry)item.RenameFrom)); try { Uri drivePathUri = new Uri(drivePath.FullPath, UriKind.Absolute); Uri actualPathUri = new Uri(actualPath, UriKind.Absolute); renameFromRelativePath = Uri.UnescapeDataString(drivePathUri.MakeRelativeUri(actualPathUri).ToString()); } catch (Exception e) { renameFromRelativePath = ""; } } if (!String.IsNullOrWhiteSpace(renameFromRelativePath) && renameFromRelativePath.StartsWith(".proximaTemp")) { continue; } var dbEntry = new RawUSNEntry(); PopulateFlags(dbEntry, item.Entry); dbEntry.Path = actualPath; dbEntry.RelativePath = relativePath; dbEntry.File = item.Entry.IsFile; dbEntry.Directory = item.Entry.IsFolder; dbEntry.FRN = item.Entry.FileReferenceNumber; dbEntry.PFRN = item.Entry.ParentFileReferenceNumber; dbEntry.RecordLength = item.Entry.RecordLength; dbEntry.USN = item.Entry.USN; dbEntry.Mountpoint = sourceMount; dbEntry.TimeStamp = item.Entry.TimeStamp.Truncate(TimeSpan.TicksPerMillisecond); dbEntry.SourceInfo = item.Entry.SourceInfo.ToString(); dbEntry.ChangeRange = item; if (actualPath != null && actualPath != "Unavailable" && actualPath.ToLowerInvariant().StartsWith($"{journal.MountPoint.TrimEnd('\\')}\\$".ToLowerInvariant())) { dbEntry.SystemFile = true; } if (item.RenameFrom != null) { dbEntry.RenameFromPath = GetActualPath(journal, ((Win32Api.UsnEntry)item.RenameFrom)); if (!string.IsNullOrWhiteSpace(dbEntry.RenameFromPath) && dbEntry.RenameFromPath != "Unavailable") { dbEntry.RenameFromRelativePath = new Regex(Regex.Escape(drivePath.FullPath), RegexOptions.IgnoreCase).Replace(dbEntry.RenameFromPath, "", 1); } } entries.Add(dbEntry); } if (changeRange.Any()) { Singleton.Instance.Repository.Add <USNChangeRange>(changeRange); Singleton.Instance.Repository.Add <RawUSNEntry>(entries); var performRollup = RollupService.PerformRollup(entries, sourceMount, Singleton.Instance.Repository); logger.Info(string.Format("[{3}] Adding [{2}CHANGE/{1}USN/{0}File]", performRollup.Count, entries.Count, changeRange.Count, sourceMount.Id)); foreach (var fileAction in performRollup) { // logger.Trace("ADD: " + fileAction.RelativePath + ", USN:" + fileAction.USN); } Singleton.Instance.Repository.Add <FileAction>(performRollup); //performRollup.ForEach(f=> logger.Debug("Added " + f.Id)); } construct.CurrentJournalData = newUsnState; sourceMount.CurrentUSNLocation = newUsnState.NextUsn; sourceMount.Volume = construct.Volume; Singleton.Instance.Repository.Update(sourceMount); } else { logger.Error("Error on Monitor - " + rtn.ToString()); throw new UsnJournalException(rtn); } } } catch (Exception e) { logger.Error(e, "Error in USNJournalMonitor"); } }
public void Execute(IJobExecutionContext context) { logger.Trace("USNJournalSync Execution"); if (Singleton.Instance.DestinationMountpoints == null || Singleton.Instance.DestinationMountpoints.Count == 0) { logger.Trace("No destination points"); return; } try { foreach (var syncFrom in Singleton.Instance.DestinationMountpoints) { syncFrom.DestinationServer.Fetch(Singleton.Instance.Servers); syncFrom.Mountpoint.Reference = Singleton.Instance.Repository.ById <MonitoredMountpoint>(syncFrom.Mountpoint.ReferenceId); if (syncFrom.Mountpoint.Reference.PublicPath == null) { logger.Warn($"PublicPath for DestinationMountPoint:{syncFrom.Id} is null! Aborting copy."); continue; } List <FileAction> rawEntries; if (String.IsNullOrWhiteSpace(syncFrom.RelativePathStartFilter)) { rawEntries = Singleton.Instance.Repository.Many <FileAction>(f => f.Mountpoint == syncFrom.Mountpoint && f.USN > syncFrom.LastUSN, limit: Singleton.Instance.CurrentServer.NormalCopyLimit, AscendingSort: e => e.USN).ToList(); } else { rawEntries = Singleton.Instance.Repository.Many <FileAction>(f => f.RelativePath.StartsWith(syncFrom.RelativePathStartFilter) && f.Mountpoint == syncFrom.Mountpoint && f.USN > syncFrom.LastUSN, limit: Singleton.Instance.CurrentServer.NormalCopyLimit, AscendingSort: e => e.USN).ToList(); } var changedFiles = RollupService.PerformRollup(rawEntries).ToList(); if (rawEntries.Count == 0) { logger.Trace("No changes found"); continue; } logger.Trace($"{changedFiles.Count} changed files for {syncFrom.Id} since USN {syncFrom.LastUSN}"); long lastUsn = rawEntries.Max(f => f.USN); Parallel.ForEach(changedFiles, new ParallelOptions { MaxDegreeOfParallelism = Singleton.Instance.CurrentServer.MaxThreads }, fileAction => { USNJournalSyncLog log = new USNJournalSyncLog(); log.Enqueued = DateTime.Now; log.DestinationMachine = Singleton.Instance.CurrentServer; log.SourceMachine = syncFrom.Mountpoint.Reference.Server; log.Entry = fileAction.USNEntry; log.Action = fileAction; Singleton.Instance.Repository.Add(log); TransferItem(log, syncFrom); }); IQueryable <USNJournalSyncLog> failedSync; if (String.IsNullOrWhiteSpace(syncFrom.RelativePathStartFilter)) { failedSync = Singleton.Instance.Repository.Many <USNJournalSyncLog>(f => !f.Successfull && f.Action.Mountpoint == syncFrom.Mountpoint && !f.RequiresManualIntervention, limit: Singleton.Instance.CurrentServer.FailedCopyLimit); } else { failedSync = Singleton.Instance.Repository.Many <USNJournalSyncLog>(f => f.Action.RelativePath.StartsWith(syncFrom.RelativePathStartFilter) && !f.Successfull && f.Action.Mountpoint == syncFrom.Mountpoint && !f.RequiresManualIntervention, limit: Singleton.Instance.CurrentServer.FailedCopyLimit); } Parallel.ForEach(failedSync, new ParallelOptions { MaxDegreeOfParallelism = Singleton.Instance.CurrentServer.MaxThreads }, failedItem => { if (failedItem.Retries == null) { failedItem.Retries = new List <DateTime> { failedItem.ActionStartDate ?? DateTime.Now }; } else { if (failedItem.Retries.Count == 20) { failedItem.RequiresManualIntervention = true; Singleton.Instance.Repository.Update(failedItem); return; } failedItem.Retries.Add(DateTime.Now); } TransferItem(failedItem, syncFrom); }); syncFrom.LastUSN = lastUsn; Singleton.Instance.Repository.Update(syncFrom); } } catch (Exception e) { logger.Error(e, "Error during JournalSync"); } }