protected byte[] DecryptFile(byte[] data, RFFileTrackedAttributes attributes, out string finalName)
{
finalName = attributes.FileName;
if (!string.IsNullOrWhiteSpace(mConfig.SourceSite.PGPSuffixes))
{
try
{
var pgpExtensions = mConfig.SourceSite.PGPSuffixes.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
var matchedExtension = pgpExtensions.FirstOrDefault(e => attributes.FileName.ToLower().EndsWith(e, StringComparison.OrdinalIgnoreCase));
if (matchedExtension != null)
{
Log.Info("Attempting to PGP decrypt file {0} as it matched suffix {1}", finalName, matchedExtension);
var inputStream = new MemoryStream(data);
var outputStream = new MemoryStream();
PGPUtils.Decrypt(inputStream, mConfig.SourceSite.PGPKeyPath, mConfig.SourceSite.PGPKeyPassword).CopyTo(outputStream);
data = outputStream.ToArray();
finalName = finalName.Substring(0, finalName.Length - matchedExtension.Length);
finalName = finalName.TrimEnd('.');
}
}
catch (Exception ex)
{
Log.UserError("Error PGP decrypting file {0}: {1}", attributes.FileName, ex.Message);
}
}
return(data);
}
public static bool IsExpired(RFFileTrackedAttributes attributes, DateTime utcNow, decimal?maxAge)
{
if (maxAge.HasValue)
{
return((decimal)(utcNow - attributes.ModifiedDate).TotalHours > maxAge);
}
return(false);
}
protected bool IsStillWrittenTo(RFFileTrackedAttributes candidate)
{
if (WriteCooldown.HasValue)
{
var now = DateTime.Now;
var writtenAgo = (now - candidate.ModifiedDate).TotalSeconds;
if (writtenAgo < WriteCooldown.Value)
{
RFStatic.Log.Warning(this, "Ignoring file {0} as it's been written to {1} seconds ago.", candidate.FileName, writtenAgo);
return(true);
}
}
return(false);
}
protected RFFileAvailableEvent ProcessCandidate(RFMonitoredFile file, RFFileTrackedAttributes candidate, ref List <RFFileAvailableEvent> foundFiles)
{
if (!IsStillWrittenTo(candidate))
{
var fae = new RFFileAvailableEvent
{
FileKey = file.FileKey,
FileAttributes = candidate,
SourceSite = SiteKey
};
foundFiles.Add(fae);
return(fae);
}
return(null);
}
public void MarkSeenFile(string fileKey, RFFileTrackedAttributes attributes)
{
if (SeenAttributes.ContainsKey(fileKey))
{
var seenEntry = SeenAttributes[fileKey];
seenEntry.Add(attributes);
}
else
{
SeenAttributes.Add(fileKey, new List <RFFileTrackedAttributes>()
{
attributes
});
}
}
public virtual RFDate ExtractValueDate(RFFileTrackedAttributes fileAttributes, RFRawReport content)
{
if (!string.IsNullOrWhiteSpace(Format))
{
try
{
if (fileAttributes != null && !string.IsNullOrWhiteSpace(fileAttributes.FileName) && fileAttributes.FileName.Length >= (Start + Format.Length))
{
return(new RFDate(DateTime.ParseExact(fileAttributes.FileName.Substring(Start, Format.Length), Format, CultureInfo.InvariantCulture)).OffsetWeekdays(Offset));
}
}
catch (Exception)
{
throw new RFLogicException(this, "Unable to extract date from file name {0} - has the format changed?", fileAttributes.FileName);
}
}
if (DefaultDate != null)
{
return(DefaultDate());
}
return(RFDate.NullDate);
}
public bool HaveSeenFile(string fileKey, RFFileTrackedAttributes attributes)
{
if (SeenAttributes.ContainsKey(fileKey))
{
var seenAttributes = SeenAttributes[fileKey];
var haveSeen = seenAttributes.Any(a => a.Equals(attributes));
if (!haveSeen)
{
RFStatic.Log.Debug(this, "File {0} ({1}) is new. None of existing {2} have-seens match:", fileKey, RFXMLSerializer.SerializeContract(attributes), seenAttributes.Count);
/*foreach(var a in seenAttributes)
* {
* RFStatic.Log.Debug(this, RFXMLSerializer.SerializeContract(a));
* }*/
}
return(haveSeen);
}
else
{
RFStatic.Log.Debug(this, "No seen files entries for file {0}", fileKey);
}
return(false);
}
public void Expand(RFMirrorSourceConfig config)
{
// get all files and check if they can be expanded
var dataTable = new DataTable();
using (var sqlConn = new SqlConnection(_connectionString))
{
sqlConn.Open();
var getCommand = new SqlCommand("RIFF.GetMirroredFile", sqlConn);
using (var reader = getCommand.ExecuteReader(System.Data.CommandBehavior.SingleResult))
{
dataTable.Load(reader);
}
}
foreach (DataRow row in dataTable.Rows)
{
var archiveName = row["FileName"].ToString();
var isExtracted = (bool)row["IsExtracted"];
if (!isExtracted && sArchives.Any(a => archiveName.EndsWith(a, StringComparison.OrdinalIgnoreCase)))
{
// it's an archive, try to extract it
var archivePath = row["MirrorPath"].ToString();
var archive = File.ReadAllBytes(Path.Combine(RootDirectory, archivePath));
using (var ms = new MemoryStream(archive))
{
foreach (var file in RIFF.Interfaces.Compression.ZIP.ZIPUtils.UnzipArchive(ms))
{
try
{
var mirrorDirectory = Path.Combine(Path.GetDirectoryName(archivePath), Path.GetFileNameWithoutExtension(archivePath));
Directory.CreateDirectory(mirrorDirectory);
var mirrorPath = Path.Combine(mirrorDirectory, file.Item1.FileName);
PutMirroredFile(
row["SourceSite"].ToString(),
file.Item1.FileSize,
file.Item1.FileName,
Path.Combine(row["SourcePath"].ToString(), file.Item1.FileName),
(DateTime)row["ModifiedDate"],
(DateTime)row["ReceivedDate"],
false,
mirrorPath);
// .zip does not contain timezone information
// so set file's update time to archive's
// update time
var newAttrs = new RFFileTrackedAttributes
{
FileName = file.Item1.FileName,
FileSize = file.Item1.FileSize,
FullPath = file.Item1.FullPath,
ModifiedDate = fileAttributes.ModifiedDate
};
if (ProcessFile(new RFFileAvailableEvent
{
FileKey = candidateFile.FileKey,
FileAttributes = newAttrs,
SourceSite = mConfig.SourceSite.SiteKey
}, file.Item2, candidateFile, seenFiles))
{
newFiles++;
}
}
catch (Exception ex)
{
//Log.UserError("Error extracting file {0} from archive {1}: {2}", file.Item1.FileName, availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
}
}
public static RFRawReport LoadFromStream(MemoryStream stream, RFFileTrackedAttributes attributes, RFDate?valueDate, RFReportParserConfig config, IRFReportBuilder builder)
{
RFReportParserFormat actualFormat = config.Format;
if (config.Format == RFReportParserFormat.AutoDetect && config.CustomLoader == null)
{
var extension = Path.GetExtension(attributes.FileName).ToLower();
if (!sExtensionFormats.TryGetValue(extension, out actualFormat))
{
throw new RFSystemException(typeof(RFReportParserProcessor), "Unable to auto-detect file format of file {0}", attributes.FileName);
}
}
var loader = config.CustomLoader ?? GetLoader(actualFormat, config);
var tables = loader.Load(stream);
if (tables == null || tables.Count == 0)
{
RFStatic.Log.Warning(typeof(RFReportParserProcessor), "No data loaded from file {0}", attributes.FileName);
return(null);
}
var rawReport = new RFRawReportBuilder().BuildReport(tables, builder, config);
if (rawReport == null)
{
RFStatic.Log.Warning(typeof(RFReportParserProcessor), "No data extracted from file {0}", attributes.FileName);
return(null);
}
if (valueDate.HasValue) // override
{
rawReport.ValueDate = valueDate.Value;
}
else
{
rawReport.ValueDate = builder.ExtractValueDate(attributes, rawReport);
}
if (rawReport.ValueDate == RFDate.NullDate)
{
throw new RFLogicException(typeof(RFReportParserProcessor), "Unable to derive value date for file {0}.", attributes.FileName);
}
rawReport.ReportCode = config.ReportCode;
rawReport.UpdateTime = attributes.ModifiedDate;
rawReport.PostDeserialize();
if (config.ValidatorFunc != null)
{
var errorMessage = config.ValidatorFunc(rawReport);
if (!string.IsNullOrWhiteSpace(errorMessage))
{
if (errorMessage == sIgnoreReport)
{
return(null);
}
throw new RFLogicException(typeof(RFReportParserProcessor), "Report validation failed - incorrect file? ({0})", errorMessage);
}
}
if (config.RequiredColumns != null && config.RequiredColumns.Any())
{
var cols = new SortedSet <string>(rawReport.GetFirstSection().Columns);
var missingColumns = config.RequiredColumns.Where(rc => !cols.Contains(rc));
if (missingColumns.Any())
{
throw new RFLogicException(typeof(RFReportParserProcessor), "Missing {0} mandatory columns - incorrect file? ({1})", missingColumns.Count(), string.Join(",", missingColumns));
}
}
return(rawReport);
}
public override RFProcessingResult Process()
{
var result = new RFProcessingResult();
if (!mConfig.SourceSite.Enabled)
{
Log.Debug("Not checking site {0} as it's disabled.", mConfig.SourceSite.SiteKey);
return(result);
}
if (!mConfig.DestinationSite.Enabled)
{
Log.Debug("Not checking site {0} as it's disabled.", mConfig.DestinationSite.SiteKey);
return(result);
}
var stateKey = GenerateStateKey("SeenFiles");
var seenFiles = new RFSeenFiles();
var seenFilesEntry = Context.LoadEntry(stateKey);
if (seenFilesEntry != null)
{
seenFiles = (seenFilesEntry as RFDocument).GetContent <RFSeenFiles>();
}
int newFiles = 0;
var destinationOpen = false;
Log.Info("Checking for files to move from {0} to {1}", mConfig.SourceSite, mConfig.DestinationSite);
try
{
mConfig.SourceSite.Open(Context);
var availableFiles = mConfig.SourceSite.CheckSite(mConfig.MonitoredFiles);
var utcNow = DateTime.UtcNow;
var filesToRemove = new List <RFFileAvailableEvent>();
foreach (var availableFile in availableFiles)
{
if (RFSeenFiles.IsExpired(availableFile.FileAttributes, utcNow, mConfig.SourceSite.MaxAge))
{
var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
if (monitoredFile.RemoveExpired)
{
filesToRemove.Add(availableFile);
}
continue;
}
if (!HaveSeenFile(seenFiles, availableFile))
{
try
{
if (!IsCancelling)
{
Log.Info("Retrieving new file {0} from {1}", availableFile.FileAttributes.FileName, mConfig.SourceSite);
var data = mConfig.SourceSite.GetFile(availableFile);
var fileAttributes = availableFile.FileAttributes;
var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
if (monitoredFile.FileKey == RFInternalFileKey.ZIPArchive)
{
// unpack and process each file if it matches monitored files
Log.Info("Attempting to download and unpack archive {0}", fileAttributes.FileName);
using (var ms = new MemoryStream(data))
{
foreach (var file in RIFF.Interfaces.Compression.ZIP.ZIPUtils.UnzipArchive(ms))
{
foreach (var candidateFile in mConfig.MonitoredFiles)
{
var isMatch = false;
if (!string.IsNullOrWhiteSpace(candidateFile.FileNameRegex))
{
var regex = new Regex(candidateFile.FileNameRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled);
isMatch |= regex.IsMatch(file.Item1.FileName);
}
if (!isMatch && !string.IsNullOrWhiteSpace(candidateFile.FileNameWildcard))
{
var regex = new Regex(RFRegexHelpers.WildcardToRegex(candidateFile.FileNameWildcard), RegexOptions.IgnoreCase | RegexOptions.Compiled);
isMatch |= regex.IsMatch(file.Item1.FileName);
}
if (isMatch)
{
try
{
// .zip does not contain timezone information
// so set file's update time to archive's
// update time
var newAttrs = new RFFileTrackedAttributes
{
FileName = file.Item1.FileName,
FileSize = file.Item1.FileSize,
FullPath = file.Item1.FullPath,
ModifiedDate = fileAttributes.ModifiedDate
};
if (!destinationOpen)
{
mConfig.DestinationSite.Open(Context);
destinationOpen = true;
}
if (ProcessFile(new RFFileAvailableEvent
{
FileKey = candidateFile.FileKey,
FileAttributes = newAttrs,
SourceSite = availableFile.SourceSite
}, file.Item2, candidateFile, seenFiles))
{
newFiles++;
}
}
catch (Exception ex)
{
Log.UserError("Error extracting file {0} from archive {1}: {2}", file.Item1.FileName, availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
}
// add archive to seen files
seenFiles.MarkSeenFile(monitoredFile.FileKey, fileAttributes);
}
else
{
if (!destinationOpen)
{
mConfig.DestinationSite.Open(Context);
destinationOpen = true;
}
if (ProcessFile(availableFile, data, monitoredFile, seenFiles))
{
newFiles++;
}
}
// archive
try
{
mConfig.SourceSite.ArchiveFile(availableFile);
} catch (Exception ex)
{
Log.Warning("Unable to archive file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
}
}
catch (Exception ex)
{
var errorCode = ex.HResult & 0xFFFF;
if (errorCode == ERROR_SHARING_VIOLATION || errorCode == ERROR_LOCK_VIOLATION)
{
Log.Info("Unable to download file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
else
{
Log.UserError("Error downloading file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
if (!IsCancelling)
{
seenFiles.CleanUpMaxAge(utcNow, mConfig.SourceSite.MaxAge);
}
Context.SaveEntry(RFDocument.Create(stateKey, seenFiles));
// try to remove
if (!IsCancelling)
{
foreach (var fileToRemove in filesToRemove)
{
try
{
mConfig.SourceSite.DeleteFile(fileToRemove);
}
catch (Exception ex)
{
Log.UserError("Error deleting file {0}: {1}", fileToRemove.FileAttributes.FullPath, ex.Message);
}
}
Log.Info("Finished checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
}
else
{
Log.Info("Interrupted when checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
}
}
catch (RFTransientSystemException)
{
throw;
}
catch (Exception ex)
{
Log.UserError("Error checking file site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
}
finally
{
try
{
if (mConfig.SourceSite != null)
{
mConfig.SourceSite.Close();
}
if (mConfig.DestinationSite != null && destinationOpen)
{
mConfig.DestinationSite.Close();
}
}
catch (Exception ex)
{
Log.Warning("Error closing site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
}
}
result.WorkDone = newFiles > 0;
return(result);
}