public override RFProcessingResult Process() { var inputFile = Context.LoadDocumentContent <RFFile>(InstanceParams.Key); RFRawReport rawReport = null; using (var ms = new MemoryStream(inputFile.Data)) { rawReport = LoadFromStream(ms, inputFile.Attributes, inputFile.ValueDate, _config, _builder); } if (rawReport != null) { rawReport.SourceUniqueKey = inputFile.UniqueKey; rawReport.SourceFilename = inputFile.Attributes.FileName; Context.SaveEntry(RFDocument.Create(RFRawReportKey.Create(KeyDomain, _config.ReportCode, new RFGraphInstance { ValueDate = rawReport.ValueDate, Name = _config.GraphInstance }), rawReport)); } return(new RFProcessingResult { WorkDone = true }); }
public RFProcessingTracker SaveDocument(RFCatalogKey key, object content, bool raiseEvent, RFUserLogEntry userLogEntry) { if (raiseEvent) { return(SaveEntry(RFDocument.Create(key, content), userLogEntry)); } else { SaveEntry(RFDocument.Create(key, content), false, userLogEntry); return(new RFProcessingTracker("dummy")); } }
public override void PutFile(RFFileAvailableEvent file, RFMonitoredFile fileConfig, byte[] data) { var uniqueKey = GenerateUniqueKey(file); var fileEntry = new RFFile { Attributes = file.FileAttributes, FileKey = file.FileKey, Data = data, UniqueKey = uniqueKey }; mContext.SaveEntry(RFDocument.Create( RFFileKey.Create(mKeyDomain, file.FileKey, uniqueKey), fileEntry)); }
public void QueuedTriggerTest() { var triggerKey = RFGenericCatalogKey.Create(_fixture.KeyDomain, "Trigger Key", TestEngine.TestKeys.Key1, null); _fixture.Context.SaveEntry(RFDocument.Create(triggerKey, new RFScheduleTrigger { LastTriggerTime = DateTime.Now }), true); Thread.Sleep(TimeSpan.FromSeconds(1)); _fixture.Context.SaveEntry(RFDocument.Create(triggerKey, new RFScheduleTrigger { LastTriggerTime = DateTime.Now }), true); Thread.Sleep(TimeSpan.FromSeconds(1)); _fixture.Context.SaveEntry(RFDocument.Create(triggerKey, new RFScheduleTrigger { LastTriggerTime = DateTime.Now }), true); Thread.Sleep(TimeSpan.FromSeconds(20)); var result = _fixture.Context.LoadDocumentContent <string>(RFGenericCatalogKey.Create(_fixture.KeyDomain, "Queued Trigger", TestEngine.TestKeys.Key1, null)); Assert.Equal("StartStopStartStop", result); // we expect 2x, not 1x or 3x }
public ActionResult SubmitFiles( RFDate?valueDate = null, string returnUrl = null, string instance = null ) { try { if (Request.Files == null || Request.Files.Count == 0) { throw new RFSystemException(this, "No files submitted."); } var processKey = String.Format("{0}_web", DateTime.Now.ToString("yyyyMMdd_HHmmss")); var submittable = new List <RFCatalogEntryDTO>(); int uqIdx = 1; foreach (var knownFileEnum in RIFF.Web.Core.App_Start.RIFFStart.Config.GetInputFileKeys()) { // match up by string (or could have registered an enum type...) foreach (string inputFileKey in Request.Files.Keys) { if (knownFileEnum.ToString() == inputFileKey) { var inputFile = Request.Files[inputFileKey] as HttpPostedFileBase; if (inputFile != null && inputFile.ContentLength > 0) { var uniqueKey = String.Format("{0}_web_{1}", DateTime.Now.ToString("yyyyMMdd_HHmmss"), uqIdx++); var doc = RFDocument.Create( RFFileKey.Create( EngineConfig.KeyDomain, knownFileEnum, uniqueKey), new RFFile { Attributes = new RFFileTrackedAttributes { FileName = inputFile.FileName, FullPath = inputFile.FileName, FileSize = inputFile.ContentLength, ModifiedDate = DateTime.Now }, Data = RFStreamHelpers.ReadBytes(inputFile.InputStream), FileKey = knownFileEnum, ValueDate = valueDate, UniqueKey = uniqueKey }); if (instance.NotBlank() && valueDate.HasValue) { doc.Key.GraphInstance = new RFGraphInstance { Name = instance, ValueDate = valueDate.Value }; } submittable.Add(new RFCatalogEntryDTO(doc)); } } } } using (var rfService = new RFServiceClient()) { RFProcessingTrackerHandle trackerKey = null; trackerKey = rfService.RFService.SubmitAndProcess(submittable, new RFUserLogEntry { Action = "Upload Files", Description = String.Format("Uploaded {0} files for processing", submittable.Count), IsUserAction = true, IsWarning = false, Processor = null, ValueDate = valueDate.HasValue ? valueDate.Value : RFDate.NullDate, Username = Username }); //lock (sSync) { AddModelToCache(processKey, new ProcessingModel { Tracker = trackerKey, ProcessingKey = processKey, ReturnUrl = returnUrl }); } return(RedirectToAction("ProcessingStatus", new { processKey = processKey })); } } catch (Exception ex) { Log.Exception(this, "SubmitFiles", ex); return(Error("InputFiles", "System", null, "Error submitting files: {0}", ex.Message)); } }
public ActionResult SubmitFile( string fileKey, HttpPostedFileBase fileData, RFDate?valueDate, string returnUrl = null, string instance = null) { try { if (fileData == null && Request.Files != null && Request.Files.Count > 0) { fileData = Request.Files[0]; } var uniqueKey = String.Format("{0}_web", DateTime.Now.ToString("yyyyMMdd_HHmmss")); if (fileData == null || fileData.FileName == null || fileData.InputStream == null) { throw new RFSystemException(this, "No file submitted."); } var fileName = Path.GetFileName(fileData.FileName); var newFileEntry = RFDocument.Create( RFFileKey.Create( EngineConfig.KeyDomain, RFEnum.FromString(fileKey), uniqueKey), new RFFile { Attributes = new RFFileTrackedAttributes { FileName = fileName, FullPath = fileData.FileName, FileSize = fileData.ContentLength, ModifiedDate = DateTime.Now }, Data = RFStreamHelpers.ReadBytes(fileData.InputStream), FileKey = RFEnum.FromString(fileKey), ValueDate = valueDate, UniqueKey = uniqueKey }); // the file will have graph instance attached if (instance.NotBlank() && valueDate.HasValue) { newFileEntry.Key.GraphInstance = new RFGraphInstance { Name = instance, ValueDate = valueDate.Value }; } using (var rfService = new RFServiceClient()) { RFProcessingTrackerHandle trackerKey = null; trackerKey = rfService.RFService.SubmitAndProcess(new List <RFCatalogEntryDTO> { new RFCatalogEntryDTO(newFileEntry) }, new RFUserLogEntry { Action = "Upload File", Description = String.Format("Uploaded file {0} for processing", fileName), IsUserAction = true, IsWarning = false, Processor = null, ValueDate = valueDate.HasValue ? valueDate.Value : RFDate.NullDate, Username = Username }); //lock (sSync) { AddModelToCache(uniqueKey, new ProcessingModel { Tracker = trackerKey, ProcessingKey = uniqueKey, FileKey = fileKey, FileName = fileName, FileSize = fileData.ContentLength, ReturnUrl = returnUrl }); } return(RedirectToAction("ProcessingStatus", new { processKey = uniqueKey })); } } catch (Exception ex) { Log.Exception(this, "SubmitFile", ex); return(Error("InputFiles", "System", null, "Error submitting file: {0}", ex.Message)); } }
public RFProcessingTrackerHandle SaveDocumentAsync(RFCatalogKey key, object content, RFUserLogEntry userLogEntry) { return(SaveEntryAsync(RFDocument.Create(key, content), userLogEntry)); }
public override RFProcessingResult Process() { var result = new RFProcessingResult(); if (!mConfig.SourceSite.Enabled) { Log.Debug("Not checking site {0} as it's disabled.", mConfig.SourceSite.SiteKey); return(result); } if (!mConfig.DestinationSite.Enabled) { Log.Debug("Not checking site {0} as it's disabled.", mConfig.DestinationSite.SiteKey); return(result); } var stateKey = GenerateStateKey("SeenFiles"); var seenFiles = new RFSeenFiles(); var seenFilesEntry = Context.LoadEntry(stateKey); if (seenFilesEntry != null) { seenFiles = (seenFilesEntry as RFDocument).GetContent <RFSeenFiles>(); } int newFiles = 0; var destinationOpen = false; Log.Info("Checking for files to move from {0} to {1}", mConfig.SourceSite, mConfig.DestinationSite); try { mConfig.SourceSite.Open(Context); var availableFiles = mConfig.SourceSite.CheckSite(mConfig.MonitoredFiles); var utcNow = DateTime.UtcNow; var filesToRemove = new List <RFFileAvailableEvent>(); foreach (var availableFile in availableFiles) { if (RFSeenFiles.IsExpired(availableFile.FileAttributes, utcNow, mConfig.SourceSite.MaxAge)) { var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey); if (monitoredFile.RemoveExpired) { filesToRemove.Add(availableFile); } continue; } if (!HaveSeenFile(seenFiles, availableFile)) { try { if (!IsCancelling) { Log.Info("Retrieving new file {0} from {1}", availableFile.FileAttributes.FileName, mConfig.SourceSite); var data = mConfig.SourceSite.GetFile(availableFile); var fileAttributes = availableFile.FileAttributes; var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey); if (monitoredFile.FileKey == RFInternalFileKey.ZIPArchive) { // unpack and process each file if it matches monitored files Log.Info("Attempting to download and unpack archive {0}", fileAttributes.FileName); using (var ms = new MemoryStream(data)) { foreach (var file in RIFF.Interfaces.Compression.ZIP.ZIPUtils.UnzipArchive(ms)) { foreach (var candidateFile in mConfig.MonitoredFiles) { var isMatch = false; if (!string.IsNullOrWhiteSpace(candidateFile.FileNameRegex)) { var regex = new Regex(candidateFile.FileNameRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled); isMatch |= regex.IsMatch(file.Item1.FileName); } if (!isMatch && !string.IsNullOrWhiteSpace(candidateFile.FileNameWildcard)) { var regex = new Regex(RFRegexHelpers.WildcardToRegex(candidateFile.FileNameWildcard), RegexOptions.IgnoreCase | RegexOptions.Compiled); isMatch |= regex.IsMatch(file.Item1.FileName); } if (isMatch) { try { // .zip does not contain timezone information // so set file's update time to archive's // update time var newAttrs = new RFFileTrackedAttributes { FileName = file.Item1.FileName, FileSize = file.Item1.FileSize, FullPath = file.Item1.FullPath, ModifiedDate = fileAttributes.ModifiedDate }; if (!destinationOpen) { mConfig.DestinationSite.Open(Context); destinationOpen = true; } if (ProcessFile(new RFFileAvailableEvent { FileKey = candidateFile.FileKey, FileAttributes = newAttrs, SourceSite = availableFile.SourceSite }, file.Item2, candidateFile, seenFiles)) { newFiles++; } } catch (Exception ex) { Log.UserError("Error extracting file {0} from archive {1}: {2}", file.Item1.FileName, availableFile.FileAttributes.FileName, ex.Message); } } } } } // add archive to seen files seenFiles.MarkSeenFile(monitoredFile.FileKey, fileAttributes); } else { if (!destinationOpen) { mConfig.DestinationSite.Open(Context); destinationOpen = true; } if (ProcessFile(availableFile, data, monitoredFile, seenFiles)) { newFiles++; } } // archive try { mConfig.SourceSite.ArchiveFile(availableFile); } catch (Exception ex) { Log.Warning("Unable to archive file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message); } } } catch (Exception ex) { var errorCode = ex.HResult & 0xFFFF; if (errorCode == ERROR_SHARING_VIOLATION || errorCode == ERROR_LOCK_VIOLATION) { Log.Info("Unable to download file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message); } else { Log.UserError("Error downloading file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message); } } } } if (!IsCancelling) { seenFiles.CleanUpMaxAge(utcNow, mConfig.SourceSite.MaxAge); } Context.SaveEntry(RFDocument.Create(stateKey, seenFiles)); // try to remove if (!IsCancelling) { foreach (var fileToRemove in filesToRemove) { try { mConfig.SourceSite.DeleteFile(fileToRemove); } catch (Exception ex) { Log.UserError("Error deleting file {0}: {1}", fileToRemove.FileAttributes.FullPath, ex.Message); } } Log.Info("Finished checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count); } else { Log.Info("Interrupted when checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count); } } catch (RFTransientSystemException) { throw; } catch (Exception ex) { Log.UserError("Error checking file site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message); } finally { try { if (mConfig.SourceSite != null) { mConfig.SourceSite.Close(); } if (mConfig.DestinationSite != null && destinationOpen) { mConfig.DestinationSite.Close(); } } catch (Exception ex) { Log.Warning("Error closing site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message); } } result.WorkDone = newFiles > 0; return(result); }