public override RFProcessingResult Process()
{
Context.SaveDocument(_testKey1, "Written", true);
Context.SaveDocument(_testKey2, "Written", true);
Log.Info("Written under transaction.");
return(RFProcessingResult.Success(true));
}
public RFProcessingTracker GetProcessStatus(RFProcessingTrackerHandle trackerHandle)
{
try
{
Log.Debug(this, "GetProcessStatus {0}", trackerHandle.TrackerCode);
LogRequest();
RFProcessingTracker tracker = null;
lock (_sync)
{
_trackers.TryGetValue(trackerHandle.TrackerCode, out tracker);
if (tracker == null)
{
Log.Warning(this, "Unable to find tracker for {0}; current cache size {1}", trackerHandle.TrackerCode, _trackers.Count);
}
}
return(tracker);
}
catch (Exception ex)
{
Log.Exception(this, "GetProcessStatus", ex);
var tracker = new RFProcessingTracker(trackerHandle.TrackerCode);
tracker.CycleFinished("dummy", RFProcessingResult.Error(new string[] { ex.Message }, false));
tracker.SetComplete();
return(tracker);
}
}
public override RFProcessingResult Process()
{
var result = new RFProcessingResult();
if (!_config.Enabled)
{
return(result);
}
try
{
int numRows = 0;
var sqlQuery = _config.QueryFunc();
var fileName = _config.FileNameFunc();
var path = System.IO.Path.Combine(_config.DestinationPath, fileName);
using (var connection = new SqlConnection(_config.ConnectionString))
{
connection.Open();
try
{
var knownColumns = new Dictionary <string, int>();
using (var queryCommand = new SqlCommand(sqlQuery, connection))
{
using (var reader = queryCommand.ExecuteReader(System.Data.CommandBehavior.SingleResult))
{
var dataTable = new DataTable();
dataTable.Load(reader);
if (dataTable != null && dataTable.Rows != null)
{
System.IO.File.WriteAllText(
path,
CSVBuilder.FromDataTable(dataTable));
numRows = dataTable.Rows.Count;
}
}
}
}
catch (Exception ex)
{
throw new RFSystemException(this, ex, "Error running SQL report for file {0}", fileName);
}
}
Context.SystemLog.Info(this, "Saved {0} rows from an SQL query into {1}", numRows, fileName);
}
catch (Exception ex)
{
throw new RFSystemException(this, ex, "Error running SQL report for path {0}", _config.DestinationPath);
}
result.WorkDone = true;
return(result);
}
public override RFProcessingResult Process()
{
var result = new RFProcessingResult();
using (var activity = _config.Activity(Context))
{
if (_config.ShouldRun != null && !_config.ShouldRun(Context, InstanceParams))
{
Log.Info("Not running activity {0} as marked for not running in this instance.", typeof(A).ToString());
return(result);
}
var trackerKey = _config.TrackerKey != null?_config.TrackerKey(InstanceParams) : null;
if (trackerKey != null)
{
var tracker = Context.LoadDocumentContent <ActionTracker>(trackerKey);
if (tracker != null)
{
if (tracker.AlreadyRun)
{
Log.Info("Not running activity {0} as already run.", typeof(A).ToString());
return(result);
}
}
}
if (_config.Action(activity, InstanceParams)) // return true to indicate it has run
{
if (trackerKey != null)
{
Context.SaveDocument(trackerKey, new ActionTracker
{
AlreadyRun = true,
LastRunBy = "system",
LastRunTime = DateTimeOffset.Now
});
}
result.WorkDone = true;
return(result);
}
}
return(result); // no work done
}
public override RFProcessingResult Process()
{
var result = new RFProcessingResult();
if (!string.IsNullOrWhiteSpace(_config.ConnectionString) && _config.MaintainCatalog)
{
try
{
using (var connection = new SqlConnection(_config.ConnectionString))
{
connection.Open();
using (var transaction = connection.BeginTransaction())
{
int rows1 = 0, rows2 = 0, rows3 = 0;
using (var command1 = new SqlCommand("DELETE FROM RIFF.CatalogDocument WHERE CatalogEntryID NOT IN ( SELECT CatalogEntryID FROM RIFF.DocumentLatestView WHERE IsValid = 1 )", connection, transaction))
{
rows1 = command1.ExecuteNonQuery();
}
using (var command2 = new SqlCommand("DELETE FROM RIFF.CatalogEntry WHERE CatalogEntryID NOT IN ( SELECT CatalogEntryID FROM RIFF.DocumentLatestView WHERE IsValid = 1 )", connection, transaction))
{
rows2 = command2.ExecuteNonQuery();
}
using (var command3 = new SqlCommand("DELETE FROM RIFF.CatalogKey WHERE CatalogKeyID NOT IN ( SELECT CatalogKeyID FROM RIFF.DocumentLatestView )", connection, transaction))
{
rows3 = command3.ExecuteNonQuery();
}
transaction.Commit();
Log.Info("Cleaned up {0} rows from CatalogDocument, {1} rows from CatalogEntry and {2} rows from CatalogKey.", rows1, rows2, rows3);
}
}
}
catch (Exception ex)
{
Log.SystemError(ex, "Error maintaining SQL catalog.", ex);
result.IsError = true;
result.AddMessage(ex.Message);
}
result.WorkDone = true;
}
return(result);
}
public override RFProcessingResult Process()
{
var result = new RFProcessingResult();
if (_config.Enabled && !string.IsNullOrWhiteSpace(_config.ArchivePath))
{
var inputReport = Context.LoadDocumentContent <RFRawReport>(InstanceParams.Key);
if (inputReport != null)
{
var csvBuilder = new StringBuilder();
foreach (var section in inputReport.Sections)
{
csvBuilder.Append(CSVBuilder.FromDataTable(section.AsDataTable()));
}
var outputDirectory = Path.Combine(_config.ArchivePath, inputReport.ValueDate.ToString("yyyy-MM-dd"));
Directory.CreateDirectory(outputDirectory);
var outputPath = Path.Combine(outputDirectory, String.Format("{0}_{1}.csv", inputReport.SourceUniqueKey, inputReport.UpdateTime.ToLocalTime().ToString("yyyyMMdd_HHmmss")));
File.WriteAllBytes(outputPath, Encoding.UTF8.GetBytes(csvBuilder.ToString()));
}
result.WorkDone = true;
}
return(result);
}
public override RFProcessingResult Process()
{
_action(Context);
return(RFProcessingResult.Success(true));
}
public override RFProcessingResult Process()
{
var result = new RFProcessingResult();
if (!mConfig.SourceSite.Enabled)
{
Log.Debug("Not checking site {0} as it's disabled.", mConfig.SourceSite.SiteKey);
return(result);
}
if (!mConfig.DestinationSite.Enabled)
{
Log.Debug("Not checking site {0} as it's disabled.", mConfig.DestinationSite.SiteKey);
return(result);
}
var stateKey = GenerateStateKey("SeenFiles");
var seenFiles = new RFSeenFiles();
var seenFilesEntry = Context.LoadEntry(stateKey);
if (seenFilesEntry != null)
{
seenFiles = (seenFilesEntry as RFDocument).GetContent <RFSeenFiles>();
}
int newFiles = 0;
var destinationOpen = false;
Log.Info("Checking for files to move from {0} to {1}", mConfig.SourceSite, mConfig.DestinationSite);
try
{
mConfig.SourceSite.Open(Context);
var availableFiles = mConfig.SourceSite.CheckSite(mConfig.MonitoredFiles);
var utcNow = DateTime.UtcNow;
var filesToRemove = new List <RFFileAvailableEvent>();
foreach (var availableFile in availableFiles)
{
if (RFSeenFiles.IsExpired(availableFile.FileAttributes, utcNow, mConfig.SourceSite.MaxAge))
{
var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
if (monitoredFile.RemoveExpired)
{
filesToRemove.Add(availableFile);
}
continue;
}
if (!HaveSeenFile(seenFiles, availableFile))
{
try
{
if (!IsCancelling)
{
Log.Info("Retrieving new file {0} from {1}", availableFile.FileAttributes.FileName, mConfig.SourceSite);
var data = mConfig.SourceSite.GetFile(availableFile);
var fileAttributes = availableFile.FileAttributes;
var monitoredFile = mConfig.MonitoredFiles.FirstOrDefault(m => m.FileKey == availableFile.FileKey);
if (monitoredFile.FileKey == RFInternalFileKey.ZIPArchive)
{
// unpack and process each file if it matches monitored files
Log.Info("Attempting to download and unpack archive {0}", fileAttributes.FileName);
using (var ms = new MemoryStream(data))
{
foreach (var file in RIFF.Interfaces.Compression.ZIP.ZIPUtils.UnzipArchive(ms))
{
foreach (var candidateFile in mConfig.MonitoredFiles)
{
var isMatch = false;
if (!string.IsNullOrWhiteSpace(candidateFile.FileNameRegex))
{
var regex = new Regex(candidateFile.FileNameRegex, RegexOptions.IgnoreCase | RegexOptions.Compiled);
isMatch |= regex.IsMatch(file.Item1.FileName);
}
if (!isMatch && !string.IsNullOrWhiteSpace(candidateFile.FileNameWildcard))
{
var regex = new Regex(RFRegexHelpers.WildcardToRegex(candidateFile.FileNameWildcard), RegexOptions.IgnoreCase | RegexOptions.Compiled);
isMatch |= regex.IsMatch(file.Item1.FileName);
}
if (isMatch)
{
try
{
// .zip does not contain timezone information
// so set file's update time to archive's
// update time
var newAttrs = new RFFileTrackedAttributes
{
FileName = file.Item1.FileName,
FileSize = file.Item1.FileSize,
FullPath = file.Item1.FullPath,
ModifiedDate = fileAttributes.ModifiedDate
};
if (!destinationOpen)
{
mConfig.DestinationSite.Open(Context);
destinationOpen = true;
}
if (ProcessFile(new RFFileAvailableEvent
{
FileKey = candidateFile.FileKey,
FileAttributes = newAttrs,
SourceSite = availableFile.SourceSite
}, file.Item2, candidateFile, seenFiles))
{
newFiles++;
}
}
catch (Exception ex)
{
Log.UserError("Error extracting file {0} from archive {1}: {2}", file.Item1.FileName, availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
}
// add archive to seen files
seenFiles.MarkSeenFile(monitoredFile.FileKey, fileAttributes);
}
else
{
if (!destinationOpen)
{
mConfig.DestinationSite.Open(Context);
destinationOpen = true;
}
if (ProcessFile(availableFile, data, monitoredFile, seenFiles))
{
newFiles++;
}
}
// archive
try
{
mConfig.SourceSite.ArchiveFile(availableFile);
} catch (Exception ex)
{
Log.Warning("Unable to archive file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
}
}
catch (Exception ex)
{
var errorCode = ex.HResult & 0xFFFF;
if (errorCode == ERROR_SHARING_VIOLATION || errorCode == ERROR_LOCK_VIOLATION)
{
Log.Info("Unable to download file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
else
{
Log.UserError("Error downloading file {0}: {1}", availableFile.FileAttributes.FileName, ex.Message);
}
}
}
}
if (!IsCancelling)
{
seenFiles.CleanUpMaxAge(utcNow, mConfig.SourceSite.MaxAge);
}
Context.SaveEntry(RFDocument.Create(stateKey, seenFiles));
// try to remove
if (!IsCancelling)
{
foreach (var fileToRemove in filesToRemove)
{
try
{
mConfig.SourceSite.DeleteFile(fileToRemove);
}
catch (Exception ex)
{
Log.UserError("Error deleting file {0}: {1}", fileToRemove.FileAttributes.FullPath, ex.Message);
}
}
Log.Info("Finished checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
}
else
{
Log.Info("Interrupted when checking for files on {0} - {1} new files of {2} total", mConfig.SourceSite, newFiles, availableFiles.Count);
}
}
catch (RFTransientSystemException)
{
throw;
}
catch (Exception ex)
{
Log.UserError("Error checking file site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
}
finally
{
try
{
if (mConfig.SourceSite != null)
{
mConfig.SourceSite.Close();
}
if (mConfig.DestinationSite != null && destinationOpen)
{
mConfig.DestinationSite.Close();
}
}
catch (Exception ex)
{
Log.Warning("Error closing site {0}: {1}", mConfig.SourceSite.SiteKey, ex.Message);
}
}
result.WorkDone = newFiles > 0;
return(result);
}
