public override RFProcessingResult Process()
{
var inputFile = Context.LoadDocumentContent <RFFile>(InstanceParams.Key);
RFRawReport rawReport = null;
using (var ms = new MemoryStream(inputFile.Data))
{
rawReport = LoadFromStream(ms, inputFile.Attributes, inputFile.ValueDate, _config, _builder);
}
if (rawReport != null)
{
rawReport.SourceUniqueKey = inputFile.UniqueKey;
rawReport.SourceFilename = inputFile.Attributes.FileName;
Context.SaveEntry(RFDocument.Create(RFRawReportKey.Create(KeyDomain, _config.ReportCode, new RFGraphInstance {
ValueDate = rawReport.ValueDate, Name = _config.GraphInstance
}),
rawReport));
}
return(new RFProcessingResult {
WorkDone = true
});
}
public RFProcessingTracker SaveDocument(RFCatalogKey key, object content, bool raiseEvent, RFUserLogEntry userLogEntry)
{
if (raiseEvent)
{
return(SaveEntry(RFDocument.Create(key, content), userLogEntry));
}
else
{
SaveEntry(RFDocument.Create(key, content), false, userLogEntry);
return(new RFProcessingTracker("dummy"));
}
}
public override void PutFile(RFFileAvailableEvent file, RFMonitoredFile fileConfig, byte[] data)
{
var uniqueKey = GenerateUniqueKey(file);
var fileEntry = new RFFile
{
Attributes = file.FileAttributes,
FileKey = file.FileKey,
Data = data,
UniqueKey = uniqueKey
};
mContext.SaveEntry(RFDocument.Create(
RFFileKey.Create(mKeyDomain, file.FileKey, uniqueKey),
fileEntry));
}
public void QueuedTriggerTest()
{
var triggerKey = RFGenericCatalogKey.Create(_fixture.KeyDomain, "Trigger Key", TestEngine.TestKeys.Key1, null);
_fixture.Context.SaveEntry(RFDocument.Create(triggerKey, new RFScheduleTrigger {
LastTriggerTime = DateTime.Now
}), true);
Thread.Sleep(TimeSpan.FromSeconds(1));
_fixture.Context.SaveEntry(RFDocument.Create(triggerKey, new RFScheduleTrigger {
LastTriggerTime = DateTime.Now
}), true);
Thread.Sleep(TimeSpan.FromSeconds(1));
_fixture.Context.SaveEntry(RFDocument.Create(triggerKey, new RFScheduleTrigger {
LastTriggerTime = DateTime.Now
}), true);
Thread.Sleep(TimeSpan.FromSeconds(20));
var result = _fixture.Context.LoadDocumentContent <string>(RFGenericCatalogKey.Create(_fixture.KeyDomain, "Queued Trigger", TestEngine.TestKeys.Key1, null));
Assert.Equal("StartStopStartStop", result); // we expect 2x, not 1x or 3x
}
public ActionResult SubmitFiles(
RFDate?valueDate = null,
string returnUrl = null,
string instance = null
)
{
try
{
if (Request.Files == null || Request.Files.Count == 0)
{
throw new RFSystemException(this, "No files submitted.");
}
var processKey = String.Format("{0}_web", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
var submittable = new List <RFCatalogEntryDTO>();
int uqIdx = 1;
foreach (var knownFileEnum in RIFF.Web.Core.App_Start.RIFFStart.Config.GetInputFileKeys())
{
// match up by string (or could have registered an enum type...)
foreach (string inputFileKey in Request.Files.Keys)
{
if (knownFileEnum.ToString() == inputFileKey)
{
var inputFile = Request.Files[inputFileKey] as HttpPostedFileBase;
if (inputFile != null && inputFile.ContentLength > 0)
{
var uniqueKey = String.Format("{0}_web_{1}", DateTime.Now.ToString("yyyyMMdd_HHmmss"), uqIdx++);
var doc = RFDocument.Create(
RFFileKey.Create(
EngineConfig.KeyDomain,
knownFileEnum,
uniqueKey),
new RFFile
{
Attributes = new RFFileTrackedAttributes
{
FileName = inputFile.FileName,
FullPath = inputFile.FileName,
FileSize = inputFile.ContentLength,
ModifiedDate = DateTime.Now
},
Data = RFStreamHelpers.ReadBytes(inputFile.InputStream),
FileKey = knownFileEnum,
ValueDate = valueDate,
UniqueKey = uniqueKey
});
if (instance.NotBlank() && valueDate.HasValue)
{
doc.Key.GraphInstance = new RFGraphInstance {
Name = instance, ValueDate = valueDate.Value
};
}
submittable.Add(new RFCatalogEntryDTO(doc));
}
}
}
}
using (var rfService = new RFServiceClient())
{
RFProcessingTrackerHandle trackerKey = null;
trackerKey = rfService.RFService.SubmitAndProcess(submittable, new RFUserLogEntry
{
Action = "Upload Files",
Description = String.Format("Uploaded {0} files for processing", submittable.Count),
IsUserAction = true,
IsWarning = false,
Processor = null,
ValueDate = valueDate.HasValue ? valueDate.Value : RFDate.NullDate,
Username = Username
});
//lock (sSync)
{
AddModelToCache(processKey, new ProcessingModel
{
Tracker = trackerKey,
ProcessingKey = processKey,
ReturnUrl = returnUrl
});
}
return(RedirectToAction("ProcessingStatus", new { processKey = processKey }));
}
}
catch (Exception ex)
{
Log.Exception(this, "SubmitFiles", ex);
return(Error("InputFiles", "System", null, "Error submitting files: {0}", ex.Message));
}
}
public ActionResult SubmitFile(
string fileKey,
HttpPostedFileBase fileData,
RFDate?valueDate,
string returnUrl = null,
string instance = null)
{
try
{
if (fileData == null && Request.Files != null && Request.Files.Count > 0)
{
fileData = Request.Files[0];
}
var uniqueKey = String.Format("{0}_web", DateTime.Now.ToString("yyyyMMdd_HHmmss"));
if (fileData == null || fileData.FileName == null || fileData.InputStream == null)
{
throw new RFSystemException(this, "No file submitted.");
}
var fileName = Path.GetFileName(fileData.FileName);
var newFileEntry = RFDocument.Create(
RFFileKey.Create(
EngineConfig.KeyDomain,
RFEnum.FromString(fileKey),
uniqueKey),
new RFFile
{
Attributes = new RFFileTrackedAttributes
{
FileName = fileName,
FullPath = fileData.FileName,
FileSize = fileData.ContentLength,
ModifiedDate = DateTime.Now
},
Data = RFStreamHelpers.ReadBytes(fileData.InputStream),
FileKey = RFEnum.FromString(fileKey),
ValueDate = valueDate,
UniqueKey = uniqueKey
});
// the file will have graph instance attached
if (instance.NotBlank() && valueDate.HasValue)
{
newFileEntry.Key.GraphInstance = new RFGraphInstance
{
Name = instance,
ValueDate = valueDate.Value
};
}
using (var rfService = new RFServiceClient())
{
RFProcessingTrackerHandle trackerKey = null;
trackerKey = rfService.RFService.SubmitAndProcess(new List <RFCatalogEntryDTO> {
new RFCatalogEntryDTO(newFileEntry)
}, new RFUserLogEntry
{
Action = "Upload File",
Description = String.Format("Uploaded file {0} for processing", fileName),
IsUserAction = true,
IsWarning = false,
Processor = null,
ValueDate = valueDate.HasValue ? valueDate.Value : RFDate.NullDate,
Username = Username
});
//lock (sSync)
{
AddModelToCache(uniqueKey, new ProcessingModel
{
Tracker = trackerKey,
ProcessingKey = uniqueKey,
FileKey = fileKey,
FileName = fileName,
FileSize = fileData.ContentLength,
ReturnUrl = returnUrl
});
}
return(RedirectToAction("ProcessingStatus", new { processKey = uniqueKey }));
}
}
catch (Exception ex)
{
Log.Exception(this, "SubmitFile", ex);
return(Error("InputFiles", "System", null, "Error submitting file: {0}", ex.Message));
}
}
public RFProcessingTrackerHandle SaveDocumentAsync(RFCatalogKey key, object content, RFUserLogEntry userLogEntry)
{
return(SaveEntryAsync(RFDocument.Create(key, content), userLogEntry));
}
public override RFProcessingResult Process()
{
var result = new RFProcessingResult();
if (!mConfig.SourceSite.Enabled)
{
Log.Debug("Not checking site {0} as it's disabled.", mConfig.SourceSite.SiteKey);
return(result);
}
if (!mConfig.DestinationSite.Enabled)
{
Log.Debug("Not checking site {0} as it's disabled.", mConfig.DestinationSite.SiteKey);
return(result);
}
var stateKey = GenerateStateKey("SeenFiles");
var seenFiles = new RFSeenFiles();
var seenFilesEntry = Context.LoadEntry(stateKey);
if (seenFilesEntry != null)
{
seenFiles = (seenFilesEntry as RFDocument).GetContent <RFSeenFiles>();
}
int newFiles = 0;
var destinationOpen = false;
Log.Info("Checking for files to move from {0} to {1}", mConfig.SourceSite, mConfig.DestinationSite);
try
{
mConfig.SourceSite.Open(Context);
var availableFiles = mConfig.SourceSite.CheckSite(mConfig.MonitoredFiles);
var utcNow = DateTime.UtcNow;
var filesToRemove = new List <RFFileAvailableEvent>();
foreach (var availableFile in availableFiles)
{
if (RFSeenFiles.IsExpired(availableFile.FileAttributes, utcNow, mConfig.SourceSite.MaxAge))
{
var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
if (monitoredFile.RemoveExpired)
{
filesToRemove.Add(availableFile);
}
continue;
}
if (!HaveSeenFile(seenFiles, availableFile))
{
try
{
if (!IsCancelling)
{
Log.Info("Retrieving new file {0} from {1}", availableFile.FileAttributes.FileName, mConfig.SourceSite);
var data = mConfig.SourceSite.GetFile(availableFile);
var fileAttributes = availableFile.FileAttributes;
var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
if (monitoredFile.FileKey == RFInternalFileKey.ZIPArchive)
{
// unpack and process each file if it matches monitored files
Log.Info("Attempting to download and unpack archive {0}", fileAttributes.FileName);
using (var ms = new MemoryStream(data))
{
foreach (var file in RIFF.Interfaces.Compression.ZIP.ZIPUtils.UnzipArchive(ms))
{
foreach (var candidateFile in mConfig.MonitoredFiles)
{
var isMatch = false;
if (!string.IsNullOrWhiteSpace(candidateFile.FileNameRegex))
{
var regex = new Regex(candidateFile.FileNameRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled);
isMatch |= regex.IsMatch(file.Item1.FileName);
}
if (!isMatch && !string.IsNullOrWhiteSpace(candidateFile.FileNameWildcard))
{
var regex = new Regex(RFRegexHelpers.WildcardToRegex(candidateFile.FileNameWildcard), RegexOptions.IgnoreCase | RegexOptions.Compiled);
isMatch |= regex.IsMatch(file.Item1.FileName);
}
if (isMatch)
{
try
{
// .zip does not contain timezone information
// so set file's update time to archive's
// update time
var newAttrs = new RFFileTrackedAttributes
{
FileName = file.Item1.FileName,
FileSize = file.Item1.FileSize,
FullPath = file.Item1.FullPath,
ModifiedDate = fileAttributes.ModifiedDate
};
if (!destinationOpen)
{
mConfig.DestinationSite.Open(Context);
destinationOpen = true;
}
if (ProcessFile(new RFFileAvailableEvent
{
FileKey = candidateFile.FileKey,
FileAttributes = newAttrs,
SourceSite = availableFile.SourceSite
}, file.Item2, candidateFile, seenFiles))
{
newFiles++;
}
}
catch (Exception ex)
{
Log.UserError("Error extracting file {0} from archive {1}: {2}", file.Item1.FileName, availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
}
// add archive to seen files
seenFiles.MarkSeenFile(monitoredFile.FileKey, fileAttributes);
}
else
{
if (!destinationOpen)
{
mConfig.DestinationSite.Open(Context);
destinationOpen = true;
}
if (ProcessFile(availableFile, data, monitoredFile, seenFiles))
{
newFiles++;
}
}
// archive
try
{
mConfig.SourceSite.ArchiveFile(availableFile);
} catch (Exception ex)
{
Log.Warning("Unable to archive file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
}
}
catch (Exception ex)
{
var errorCode = ex.HResult & 0xFFFF;
if (errorCode == ERROR_SHARING_VIOLATION || errorCode == ERROR_LOCK_VIOLATION)
{
Log.Info("Unable to download file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
else
{
Log.UserError("Error downloading file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
if (!IsCancelling)
{
seenFiles.CleanUpMaxAge(utcNow, mConfig.SourceSite.MaxAge);
}
Context.SaveEntry(RFDocument.Create(stateKey, seenFiles));
// try to remove
if (!IsCancelling)
{
foreach (var fileToRemove in filesToRemove)
{
try
{
mConfig.SourceSite.DeleteFile(fileToRemove);
}
catch (Exception ex)
{
Log.UserError("Error deleting file {0}: {1}", fileToRemove.FileAttributes.FullPath, ex.Message);
}
}
Log.Info("Finished checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
}
else
{
Log.Info("Interrupted when checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
}
}
catch (RFTransientSystemException)
{
throw;
}
catch (Exception ex)
{
Log.UserError("Error checking file site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
}
finally
{
try
{
if (mConfig.SourceSite != null)
{
mConfig.SourceSite.Close();
}
if (mConfig.DestinationSite != null && destinationOpen)
{
mConfig.DestinationSite.Close();
}
}
catch (Exception ex)
{
Log.Warning("Error closing site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
}
}
result.WorkDone = newFiles > 0;
return(result);
}
