public override RFProcessingResult Process()
{
var result = new RFProcessingResult();
if (!mConfig.SourceSite.Enabled)
{
Log.Debug("Not checking site {0} as it's disabled.", mConfig.SourceSite.SiteKey);
return(result);
}
if (!mConfig.DestinationSite.Enabled)
{
Log.Debug("Not checking site {0} as it's disabled.", mConfig.DestinationSite.SiteKey);
return(result);
}
var stateKey = GenerateStateKey("SeenFiles");
var seenFiles = new RFSeenFiles();
var seenFilesEntry = Context.LoadEntry(stateKey);
if (seenFilesEntry != null)
{
seenFiles = (seenFilesEntry as RFDocument).GetContent <RFSeenFiles>();
}
int newFiles = 0;
var destinationOpen = false;
Log.Info("Checking for files to move from {0} to {1}", mConfig.SourceSite, mConfig.DestinationSite);
try
{
mConfig.SourceSite.Open(Context);
var availableFiles = mConfig.SourceSite.CheckSite(mConfig.MonitoredFiles);
var utcNow = DateTime.UtcNow;
var filesToRemove = new List <RFFileAvailableEvent>();
foreach (var availableFile in availableFiles)
{
if (RFSeenFiles.IsExpired(availableFile.FileAttributes, utcNow, mConfig.SourceSite.MaxAge))
{
var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
if (monitoredFile.RemoveExpired)
{
filesToRemove.Add(availableFile);
}
continue;
}
if (!HaveSeenFile(seenFiles, availableFile))
{
try
{
if (!IsCancelling)
{
Log.Info("Retrieving new file {0} from {1}", availableFile.FileAttributes.FileName, mConfig.SourceSite);
var data = mConfig.SourceSite.GetFile(availableFile);
var fileAttributes = availableFile.FileAttributes;
var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
if (monitoredFile.FileKey == RFInternalFileKey.ZIPArchive)
{
// unpack and process each file if it matches monitored files
Log.Info("Attempting to download and unpack archive {0}", fileAttributes.FileName);
using (var ms = new MemoryStream(data))
{
foreach (var file in RIFF.Interfaces.Compression.ZIP.ZIPUtils.UnzipArchive(ms))
{
foreach (var candidateFile in mConfig.MonitoredFiles)
{
var isMatch = false;
if (!string.IsNullOrWhiteSpace(candidateFile.FileNameRegex))
{
var regex = new Regex(candidateFile.FileNameRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled);
isMatch |= regex.IsMatch(file.Item1.FileName);
}
if (!isMatch && !string.IsNullOrWhiteSpace(candidateFile.FileNameWildcard))
{
var regex = new Regex(RFRegexHelpers.WildcardToRegex(candidateFile.FileNameWildcard), RegexOptions.IgnoreCase | RegexOptions.Compiled);
isMatch |= regex.IsMatch(file.Item1.FileName);
}
if (isMatch)
{
try
{
// .zip does not contain timezone information
// so set file's update time to archive's
// update time
var newAttrs = new RFFileTrackedAttributes
{
FileName = file.Item1.FileName,
FileSize = file.Item1.FileSize,
FullPath = file.Item1.FullPath,
ModifiedDate = fileAttributes.ModifiedDate
};
if (!destinationOpen)
{
mConfig.DestinationSite.Open(Context);
destinationOpen = true;
}
if (ProcessFile(new RFFileAvailableEvent
{
FileKey = candidateFile.FileKey,
FileAttributes = newAttrs,
SourceSite = availableFile.SourceSite
}, file.Item2, candidateFile, seenFiles))
{
newFiles++;
}
}
catch (Exception ex)
{
Log.UserError("Error extracting file {0} from archive {1}: {2}", file.Item1.FileName, availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
}
// add archive to seen files
seenFiles.MarkSeenFile(monitoredFile.FileKey, fileAttributes);
}
else
{
if (!destinationOpen)
{
mConfig.DestinationSite.Open(Context);
destinationOpen = true;
}
if (ProcessFile(availableFile, data, monitoredFile, seenFiles))
{
newFiles++;
}
}
// archive
try
{
mConfig.SourceSite.ArchiveFile(availableFile);
} catch (Exception ex)
{
Log.Warning("Unable to archive file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
}
}
catch (Exception ex)
{
var errorCode = ex.HResult & 0xFFFF;
if (errorCode == ERROR_SHARING_VIOLATION || errorCode == ERROR_LOCK_VIOLATION)
{
Log.Info("Unable to download file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
else
{
Log.UserError("Error downloading file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
if (!IsCancelling)
{
seenFiles.CleanUpMaxAge(utcNow, mConfig.SourceSite.MaxAge);
}
Context.SaveEntry(RFDocument.Create(stateKey, seenFiles));
// try to remove
if (!IsCancelling)
{
foreach (var fileToRemove in filesToRemove)
{
try
{
mConfig.SourceSite.DeleteFile(fileToRemove);
}
catch (Exception ex)
{
Log.UserError("Error deleting file {0}: {1}", fileToRemove.FileAttributes.FullPath, ex.Message);
}
}
Log.Info("Finished checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
}
else
{
Log.Info("Interrupted when checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
}
}
catch (RFTransientSystemException)
{
throw;
}
catch (Exception ex)
{
Log.UserError("Error checking file site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
}
finally
{
try
{
if (mConfig.SourceSite != null)
{
mConfig.SourceSite.Close();
}
if (mConfig.DestinationSite != null && destinationOpen)
{
mConfig.DestinationSite.Close();
}
}
catch (Exception ex)
{
Log.Warning("Error closing site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
}
}
result.WorkDone = newFiles > 0;
return(result);
}
protected static void MarkSeenFile(RFSeenFiles seenFiles, RFFileAvailableEvent e)
{
seenFiles.MarkSeenFile(e.FileKey, e.FileAttributes);
}