protected byte[] DecryptFile(byte[] data, RFFileTrackedAttributes attributes, out string finalName)
        {
            finalName = attributes.FileName;
            if (!string.IsNullOrWhiteSpace(mConfig.SourceSite.PGPSuffixes))
            {
                try
                {
                    var pgpExtensions    = mConfig.SourceSite.PGPSuffixes.Trim().Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                    var matchedExtension = pgpExtensions.FirstOrDefault(e => attributes.FileName.ToLower().EndsWith(e, StringComparison.OrdinalIgnoreCase));
                    if (matchedExtension != null)
                    {
                        Log.Info("Attempting to PGP decrypt file {0} as it matched suffix {1}", finalName, matchedExtension);
                        var inputStream  = new MemoryStream(data);
                        var outputStream = new MemoryStream();
                        PGPUtils.Decrypt(inputStream, mConfig.SourceSite.PGPKeyPath, mConfig.SourceSite.PGPKeyPassword).CopyTo(outputStream);
                        data = outputStream.ToArray();

                        finalName = finalName.Substring(0, finalName.Length - matchedExtension.Length);
                        finalName = finalName.TrimEnd('.');
                    }
                }
                catch (Exception ex)
                {
                    Log.UserError("Error PGP decrypting file {0}: {1}", attributes.FileName, ex.Message);
                }
            }
            return(data);
        }
예제 #2
0
 public static bool IsExpired(RFFileTrackedAttributes attributes, DateTime utcNow, decimal?maxAge)
 {
     if (maxAge.HasValue)
     {
         return((decimal)(utcNow - attributes.ModifiedDate).TotalHours > maxAge);
     }
     return(false);
 }
예제 #3
0
 protected bool IsStillWrittenTo(RFFileTrackedAttributes candidate)
 {
     if (WriteCooldown.HasValue)
     {
         var now        = DateTime.Now;
         var writtenAgo = (now - candidate.ModifiedDate).TotalSeconds;
         if (writtenAgo < WriteCooldown.Value)
         {
             RFStatic.Log.Warning(this, "Ignoring file {0} as it's been written to {1} seconds ago.", candidate.FileName, writtenAgo);
             return(true);
         }
     }
     return(false);
 }
예제 #4
0
 protected RFFileAvailableEvent ProcessCandidate(RFMonitoredFile file, RFFileTrackedAttributes candidate, ref List <RFFileAvailableEvent> foundFiles)
 {
     if (!IsStillWrittenTo(candidate))
     {
         var fae = new RFFileAvailableEvent
         {
             FileKey        = file.FileKey,
             FileAttributes = candidate,
             SourceSite     = SiteKey
         };
         foundFiles.Add(fae);
         return(fae);
     }
     return(null);
 }
예제 #5
0
 public void MarkSeenFile(string fileKey, RFFileTrackedAttributes attributes)
 {
     if (SeenAttributes.ContainsKey(fileKey))
     {
         var seenEntry = SeenAttributes[fileKey];
         seenEntry.Add(attributes);
     }
     else
     {
         SeenAttributes.Add(fileKey, new List <RFFileTrackedAttributes>()
         {
             attributes
         });
     }
 }
 public virtual RFDate ExtractValueDate(RFFileTrackedAttributes fileAttributes, RFRawReport content)
 {
     if (!string.IsNullOrWhiteSpace(Format))
     {
         try
         {
             if (fileAttributes != null && !string.IsNullOrWhiteSpace(fileAttributes.FileName) && fileAttributes.FileName.Length >= (Start + Format.Length))
             {
                 return(new RFDate(DateTime.ParseExact(fileAttributes.FileName.Substring(Start, Format.Length), Format, CultureInfo.InvariantCulture)).OffsetWeekdays(Offset));
             }
         }
         catch (Exception)
         {
             throw new RFLogicException(this, "Unable to extract date from file name {0} - has the format changed?", fileAttributes.FileName);
         }
     }
     if (DefaultDate != null)
     {
         return(DefaultDate());
     }
     return(RFDate.NullDate);
 }
예제 #7
0
        public bool HaveSeenFile(string fileKey, RFFileTrackedAttributes attributes)
        {
            if (SeenAttributes.ContainsKey(fileKey))
            {
                var seenAttributes = SeenAttributes[fileKey];
                var haveSeen       = seenAttributes.Any(a => a.Equals(attributes));
                if (!haveSeen)
                {
                    RFStatic.Log.Debug(this, "File {0} ({1}) is new. None of existing {2} have-seens match:", fileKey, RFXMLSerializer.SerializeContract(attributes), seenAttributes.Count);

                    /*foreach(var a in seenAttributes)
                     * {
                     *  RFStatic.Log.Debug(this, RFXMLSerializer.SerializeContract(a));
                     * }*/
                }
                return(haveSeen);
            }
            else
            {
                RFStatic.Log.Debug(this, "No seen files entries for file {0}", fileKey);
            }
            return(false);
        }
예제 #8
0
        public void Expand(RFMirrorSourceConfig config)
        {
            // get all files and check if they can be expanded
            var dataTable = new DataTable();

            using (var sqlConn = new SqlConnection(_connectionString))
            {
                sqlConn.Open();
                var getCommand = new SqlCommand("RIFF.GetMirroredFile", sqlConn);
                using (var reader = getCommand.ExecuteReader(System.Data.CommandBehavior.SingleResult))
                {
                    dataTable.Load(reader);
                }
            }

            foreach (DataRow row in dataTable.Rows)
            {
                var archiveName = row["FileName"].ToString();
                var isExtracted = (bool)row["IsExtracted"];
                if (!isExtracted && sArchives.Any(a => archiveName.EndsWith(a, StringComparison.OrdinalIgnoreCase)))
                {
                    // it's an archive, try to extract it
                    var archivePath = row["MirrorPath"].ToString();
                    var archive     = File.ReadAllBytes(Path.Combine(RootDirectory, archivePath));

                    using (var ms = new MemoryStream(archive))
                    {
                        foreach (var file in RIFF.Interfaces.Compression.ZIP.ZIPUtils.UnzipArchive(ms))
                        {
                            try
                            {
                                var mirrorDirectory = Path.Combine(Path.GetDirectoryName(archivePath), Path.GetFileNameWithoutExtension(archivePath));
                                Directory.CreateDirectory(mirrorDirectory);

                                var mirrorPath = Path.Combine(mirrorDirectory, file.Item1.FileName);

                                PutMirroredFile(
                                    row["SourceSite"].ToString(),
                                    file.Item1.FileSize,
                                    file.Item1.FileName,
                                    Path.Combine(row["SourcePath"].ToString(), file.Item1.FileName),
                                    (DateTime)row["ModifiedDate"],
                                    (DateTime)row["ReceivedDate"],
                                    false,
                                    mirrorPath);

                                // .zip does not contain timezone information
                                // so set file's update time to archive's
                                // update time
                                var newAttrs = new RFFileTrackedAttributes
                                {
                                    FileName     = file.Item1.FileName,
                                    FileSize     = file.Item1.FileSize,
                                    FullPath     = file.Item1.FullPath,
                                    ModifiedDate = fileAttributes.ModifiedDate
                                };
                                if (ProcessFile(new RFFileAvailableEvent
                                {
                                    FileKey = candidateFile.FileKey,
                                    FileAttributes = newAttrs,
                                    SourceSite = mConfig.SourceSite.SiteKey
                                }, file.Item2, candidateFile, seenFiles))
                                {
                                    newFiles++;
                                }
                            }
                            catch (Exception ex)
                            {
                                //Log.UserError("Error extracting file {0} from archive {1}: {2}", file.Item1.FileName, availableFile.FileAttributes.FileName, ex.Message);
                            }
                        }
                    }
                }
            }
        }
예제 #9
0
        public static RFRawReport LoadFromStream(MemoryStream stream, RFFileTrackedAttributes attributes, RFDate?valueDate, RFReportParserConfig config, IRFReportBuilder builder)
        {
            RFReportParserFormat actualFormat = config.Format;

            if (config.Format == RFReportParserFormat.AutoDetect && config.CustomLoader == null)
            {
                var extension = Path.GetExtension(attributes.FileName).ToLower();
                if (!sExtensionFormats.TryGetValue(extension, out actualFormat))
                {
                    throw new RFSystemException(typeof(RFReportParserProcessor), "Unable to auto-detect file format of file {0}", attributes.FileName);
                }
            }

            var loader = config.CustomLoader ?? GetLoader(actualFormat, config);
            var tables = loader.Load(stream);

            if (tables == null || tables.Count == 0)
            {
                RFStatic.Log.Warning(typeof(RFReportParserProcessor), "No data loaded from file {0}", attributes.FileName);
                return(null);
            }

            var rawReport = new RFRawReportBuilder().BuildReport(tables, builder, config);

            if (rawReport == null)
            {
                RFStatic.Log.Warning(typeof(RFReportParserProcessor), "No data extracted from file {0}", attributes.FileName);
                return(null);
            }

            if (valueDate.HasValue) // override
            {
                rawReport.ValueDate = valueDate.Value;
            }
            else
            {
                rawReport.ValueDate = builder.ExtractValueDate(attributes, rawReport);
            }
            if (rawReport.ValueDate == RFDate.NullDate)
            {
                throw new RFLogicException(typeof(RFReportParserProcessor), "Unable to derive value date for file {0}.", attributes.FileName);
            }
            rawReport.ReportCode = config.ReportCode;
            rawReport.UpdateTime = attributes.ModifiedDate;
            rawReport.PostDeserialize();

            if (config.ValidatorFunc != null)
            {
                var errorMessage = config.ValidatorFunc(rawReport);
                if (!string.IsNullOrWhiteSpace(errorMessage))
                {
                    if (errorMessage == sIgnoreReport)
                    {
                        return(null);
                    }
                    throw new RFLogicException(typeof(RFReportParserProcessor), "Report validation failed - incorrect file? ({0})", errorMessage);
                }
            }

            if (config.RequiredColumns != null && config.RequiredColumns.Any())
            {
                var cols           = new SortedSet <string>(rawReport.GetFirstSection().Columns);
                var missingColumns = config.RequiredColumns.Where(rc => !cols.Contains(rc));
                if (missingColumns.Any())
                {
                    throw new RFLogicException(typeof(RFReportParserProcessor), "Missing {0} mandatory columns - incorrect file? ({1})", missingColumns.Count(), string.Join(",", missingColumns));
                }
            }
            return(rawReport);
        }
        public override RFProcessingResult Process()
        {
            var result = new RFProcessingResult();

            if (!mConfig.SourceSite.Enabled)
            {
                Log.Debug("Not checking site {0} as it's disabled.", mConfig.SourceSite.SiteKey);
                return(result);
            }
            if (!mConfig.DestinationSite.Enabled)
            {
                Log.Debug("Not checking site {0} as it's disabled.", mConfig.DestinationSite.SiteKey);
                return(result);
            }
            var stateKey = GenerateStateKey("SeenFiles");

            var seenFiles      = new RFSeenFiles();
            var seenFilesEntry = Context.LoadEntry(stateKey);

            if (seenFilesEntry != null)
            {
                seenFiles = (seenFilesEntry as RFDocument).GetContent <RFSeenFiles>();
            }

            int newFiles        = 0;
            var destinationOpen = false;

            Log.Info("Checking for files to move from {0} to {1}", mConfig.SourceSite, mConfig.DestinationSite);
            try
            {
                mConfig.SourceSite.Open(Context);

                var availableFiles = mConfig.SourceSite.CheckSite(mConfig.MonitoredFiles);
                var utcNow         = DateTime.UtcNow;
                var filesToRemove  = new List <RFFileAvailableEvent>();
                foreach (var availableFile in availableFiles)
                {
                    if (RFSeenFiles.IsExpired(availableFile.FileAttributes, utcNow, mConfig.SourceSite.MaxAge))
                    {
                        var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
                        if (monitoredFile.RemoveExpired)
                        {
                            filesToRemove.Add(availableFile);
                        }
                        continue;
                    }
                    if (!HaveSeenFile(seenFiles, availableFile))
                    {
                        try
                        {
                            if (!IsCancelling)
                            {
                                Log.Info("Retrieving new file {0} from {1}", availableFile.FileAttributes.FileName, mConfig.SourceSite);

                                var data           = mConfig.SourceSite.GetFile(availableFile);
                                var fileAttributes = availableFile.FileAttributes;
                                var monitoredFile  = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);

                                if (monitoredFile.FileKey == RFInternalFileKey.ZIPArchive)
                                {
                                    // unpack and process each file if it matches monitored files
                                    Log.Info("Attempting to download and unpack archive {0}", fileAttributes.FileName);
                                    using (var ms = new MemoryStream(data))
                                    {
                                        foreach (var file in RIFF.Interfaces.Compression.ZIP.ZIPUtils.UnzipArchive(ms))
                                        {
                                            foreach (var candidateFile in mConfig.MonitoredFiles)
                                            {
                                                var isMatch = false;
                                                if (!string.IsNullOrWhiteSpace(candidateFile.FileNameRegex))
                                                {
                                                    var regex = new Regex(candidateFile.FileNameRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled);
                                                    isMatch |= regex.IsMatch(file.Item1.FileName);
                                                }
                                                if (!isMatch && !string.IsNullOrWhiteSpace(candidateFile.FileNameWildcard))
                                                {
                                                    var regex = new Regex(RFRegexHelpers.WildcardToRegex(candidateFile.FileNameWildcard), RegexOptions.IgnoreCase | RegexOptions.Compiled);
                                                    isMatch |= regex.IsMatch(file.Item1.FileName);
                                                }
                                                if (isMatch)
                                                {
                                                    try
                                                    {
                                                        // .zip does not contain timezone information
                                                        // so set file's update time to archive's
                                                        // update time
                                                        var newAttrs = new RFFileTrackedAttributes
                                                        {
                                                            FileName     = file.Item1.FileName,
                                                            FileSize     = file.Item1.FileSize,
                                                            FullPath     = file.Item1.FullPath,
                                                            ModifiedDate = fileAttributes.ModifiedDate
                                                        };
                                                        if (!destinationOpen)
                                                        {
                                                            mConfig.DestinationSite.Open(Context);
                                                            destinationOpen = true;
                                                        }
                                                        if (ProcessFile(new RFFileAvailableEvent
                                                        {
                                                            FileKey = candidateFile.FileKey,
                                                            FileAttributes = newAttrs,
                                                            SourceSite = availableFile.SourceSite
                                                        }, file.Item2, candidateFile, seenFiles))
                                                        {
                                                            newFiles++;
                                                        }
                                                    }
                                                    catch (Exception ex)
                                                    {
                                                        Log.UserError("Error extracting file {0} from archive {1}: {2}", file.Item1.FileName, availableFile.FileAttributes.FileName, ex.Message);
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    // add archive to seen files
                                    seenFiles.MarkSeenFile(monitoredFile.FileKey, fileAttributes);
                                }
                                else
                                {
                                    if (!destinationOpen)
                                    {
                                        mConfig.DestinationSite.Open(Context);
                                        destinationOpen = true;
                                    }

                                    if (ProcessFile(availableFile, data, monitoredFile, seenFiles))
                                    {
                                        newFiles++;
                                    }
                                }

                                // archive
                                try
                                {
                                    mConfig.SourceSite.ArchiveFile(availableFile);
                                } catch (Exception ex)
                                {
                                    Log.Warning("Unable to archive file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
                                }
                            }
                        }
                        catch (Exception ex)
                        {
                            var errorCode = ex.HResult & 0xFFFF;

                            if (errorCode == ERROR_SHARING_VIOLATION || errorCode == ERROR_LOCK_VIOLATION)
                            {
                                Log.Info("Unable to download file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
                            }
                            else
                            {
                                Log.UserError("Error downloading file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
                            }
                        }
                    }
                }
                if (!IsCancelling)
                {
                    seenFiles.CleanUpMaxAge(utcNow, mConfig.SourceSite.MaxAge);
                }
                Context.SaveEntry(RFDocument.Create(stateKey, seenFiles));

                // try to remove
                if (!IsCancelling)
                {
                    foreach (var fileToRemove in filesToRemove)
                    {
                        try
                        {
                            mConfig.SourceSite.DeleteFile(fileToRemove);
                        }
                        catch (Exception ex)
                        {
                            Log.UserError("Error deleting file {0}: {1}", fileToRemove.FileAttributes.FullPath, ex.Message);
                        }
                    }
                    Log.Info("Finished checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
                }
                else
                {
                    Log.Info("Interrupted when checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
                }
            }
            catch (RFTransientSystemException)
            {
                throw;
            }
            catch (Exception ex)
            {
                Log.UserError("Error checking file site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
            }
            finally
            {
                try
                {
                    if (mConfig.SourceSite != null)
                    {
                        mConfig.SourceSite.Close();
                    }
                    if (mConfig.DestinationSite != null && destinationOpen)
                    {
                        mConfig.DestinationSite.Close();
                    }
                }
                catch (Exception ex)
                {
                    Log.Warning("Error closing site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
                }
            }
            result.WorkDone = newFiles > 0;
            return(result);
        }