public static long ExportCatalogUpdates(IRFProcessingContext context, string path, RFDate startDate, RFDate?endDate = null, string password = null)
{
long c = 0;
var keysInScope = context.SearchKeys(typeof(RFCatalogKey), startDate, endDate, 99999, null, false).Where(k => k.IsValid && k.Key.Plane == RFPlane.User).ToList();
foreach (var keyDate in keysInScope.GroupBy(k => k.UpdateTime.Date))
{
var fileName = Path.Combine(path, String.Format("RIFF_{0}_Updates_{1}.zip", context.Environment, keyDate.Key.ToString("yyyyMMdd")));
context.SystemLog.Info(typeof(RFCatalogMaintainer), "Exporting {0} documents into {1}", keyDate.Count(), fileName);
var exportableDocuments = new Dictionary <string, byte[]>();
long cnt = 1;
foreach (var key in keyDate)
{
var doc = context.LoadEntry(key.Key) as RFDocument;
if (doc != null)
{
var docName = RFFileHelpers.SanitizeFileName(string.Format("{0}_{1}_{2}_{3}_{4}.xml",
doc.Key.GraphInstance?.ValueDate?.ToString() ?? "none",
doc.Key.GraphInstance?.Name ?? "none",
doc.Key.GetType().Name,
doc.Key.FriendlyString(),
cnt++));
exportableDocuments.Add(docName, Encoding.UTF8.GetBytes(RFXMLSerializer.PrettySerializeContract(doc)));
c++;
}
}
ZIPUtils.ZipFiles(fileName, exportableDocuments, password);
}
return(c);
}
public ActionResult ImportEntry(HttpPostedFileBase fileData)
{
try
{
if (fileData == null || fileData.FileName == null || fileData.InputStream == null)
{
throw new RFSystemException(this, "No file submitted.");
}
var xml = System.Text.Encoding.UTF8.GetString(RFStreamHelpers.ReadBytes(fileData.InputStream));
var document = RFXMLSerializer.DeserializeContract(typeof(RFDocument).FullName, xml) as RFDocument;
if (document == null)
{
return(Error("ImportEntry", "System", null, "Unable to deserialize object."));
}
else
{
Context.SaveDocument(document.Key, document.Content);
return(RedirectToAction("DataEditor", "System"));
}
}
catch (Exception ex)
{
Log.Exception(this, "ImportEntry", ex);
return(Error("ImportEntry", "System", null, "Error submitting entry: {0}", ex.Message));
}
}
public static long ImportCatalogUpdates(IRFProcessingContext context, string path)
{
long c = 0;
foreach (var f in Directory.GetFiles(path, "*.zip").OrderBy(f => f))
{
context.SystemLog.Info(typeof(RFCatalogMaintainer), "Importing updates from {0}", f);
using (var fs = new FileStream(f, FileMode.Open, FileAccess.Read))
{
try
{
foreach (var entry in ZIPUtils.UnzipArchive(fs))
{
try
{
var document = RFXMLSerializer.DeserializeContract(typeof(RFDocument).FullName, new string(Encoding.UTF8.GetChars(entry.Item2))) as RFDocument;
if (document != null && context.SaveEntry(document, false, false))
{
c++;
}
}
catch (Exception ex)
{
context.SystemLog.Error(typeof(RFCatalogMaintainer), "Error importing entry {0}: {1}", entry.Item1, ex.Message);
}
}
}
catch (Exception ex)
{
context.SystemLog.Error(typeof(RFCatalogMaintainer), "Error processing .zip file {0}: {1}", f, ex.Message);
}
}
}
return(c);
}
public RFCatalogEntryDTO(RFCatalogEntry e)
{
if (e != null)
{
TypeName = e.GetType().FullName;
Content = RFXMLSerializer.SerializeContract(e);
}
}
public FileResult GetProcessDomain(string processName, string instanceName, DateTime?instanceDate)
{
using (var activity = new RFEngineActivity(Context, EngineConfig))
{
RFGraphProcessorDomain domain = null;
var engineConfig = activity.GetEngineConfig();
var graphInstance = new RFGraphInstance
{
Name = string.IsNullOrWhiteSpace(instanceName) ? null : instanceName,
ValueDate = instanceDate.HasValue ? new RFDate(instanceDate.Value.Date) : RFDate.NullDate
};
foreach (var graph in engineConfig.Graphs.Values)
{
var process = graph.Processes.Values.FirstOrDefault(p => RFGraphDefinition.GetFullName(graph.GraphName, p.Name) == processName);
if (process != null)
{
var processor = process.Processor();
domain = processor.CreateDomain();
if (domain != null)
{
foreach (var propertyInfo in domain.GetType().GetProperties())
{
var ioBehaviourAttribute = (propertyInfo.GetCustomAttributes(typeof(RFIOBehaviourAttribute), true).FirstOrDefault() as RFIOBehaviourAttribute);
RFCatalogKey ioKey = null;
if (ioBehaviourAttribute != null)
{
var ioBehaviour = ioBehaviourAttribute.IOBehaviour;
var ioMapping = process.IOMappings.FirstOrDefault(m => m.PropertyName == propertyInfo.Name);
if (ioMapping != null)
{
ioKey = ioMapping.Key.CreateForInstance(graphInstance);
var options = RFGraphInstance.ImplyOptions(ioMapping);
var entry = Context.LoadEntry(ioKey, options) as RFDocument;
if (entry != null)
{
propertyInfo.SetValue(domain, entry.Content);
}
}
}
}
}
break;
}
}
if (domain != null)
{
var xmlString = RFXMLSerializer.PrettySerializeContract(domain);
return(File(
Encoding.UTF8.GetBytes(xmlString),
"text/xml",
String.Format("{0}_{1}_{2}.xml", processName, instanceName ?? String.Empty, instanceDate.Value.ToString("yyyy-MM-dd"))
));
}
}
return(null);
}
public bool SaveDocument(string keyType, string contentType, string keyData, string contentData)
{
var key = RFXMLSerializer.DeserializeContract(keyType, keyData) as RFCatalogKey;
var content = RFXMLSerializer.DeserializeContract(contentType, contentData);
var metadata = new RFMetadata();
metadata.Properties.Add("Creator", "webuser");
Context.SaveDocument(key, content, false, CreateUserLogEntry("Save Document", String.Format("Updated document {0}", key.FriendlyString()), null)); // silent
return(true);
}
public bool UpdateDocument(string type, long keyReference, string data)
{
var allKeys = Context.GetKeysByType(RFReflectionHelpers.GetTypeByFullName(type));
if (allKeys.ContainsKey(keyReference))
{
var existingDocument = Context.LoadEntry(allKeys[keyReference]) as RFDocument;
existingDocument.Content = RFXMLSerializer.DeserializeContract(existingDocument.Type, data);
Context.SaveDocument(existingDocument.Key, existingDocument.Content, false, CreateUserLogEntry("Update Document", String.Format("Saved document {0}", existingDocument.Key.FriendlyString()), null));
}
return(true);
}
public FileResult DownloadEntry(string type, long keyReference)
{
using (var dataEditor = new RFDataEditorActivity(Context, Username))
{
var entry = dataEditor.GetDocumentForDownload(type, keyReference);
if (entry != null)
{
var shortType = System.IO.Path.GetExtension(type).TrimStart('.');
var content = entry.Content;
if (content is IRFDataSet)
{
var namePart = entry.Key.FriendlyString();
Array.ForEach(Path.GetInvalidFileNameChars(), c => namePart = namePart.Replace(c.ToString(), String.Empty));
var date = (entry.Key.GraphInstance != null ? entry.Key.GraphInstance.ValueDate : null) ?? DateTime.Now;
return(File(RIFF.Interfaces.Formats.XLSX.XLSXGenerator.ExportToXLSX(type, content as IRFDataSet), "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
String.Format("{0}_{3}-{1}-{2}.xlsx", namePart, shortType, keyReference, date.ToString("yyyyMMdd"))));
}
else if (entry.Content is RFFile)
{
var file = entry.Content as RFFile;
return(File(file.Data, ImplyContentType(file.ContentName), file.Attributes.FileName));
}
else if (entry.Content is RFRawReport)
{
var report = entry.Content as RFRawReport;
var csvBuilder = new StringBuilder();
foreach (var section in report.Sections)
{
try
{
csvBuilder.Append(CSVBuilder.FromDataTable(section.AsDataTable()));
}
catch (Exception ex)
{
Log.Warning(this, "Error exporting section {0} of report {1} to .csv: {2}", section.Name, report.ReportCode, ex.Message);
}
}
return(File(Encoding.UTF8.GetBytes(csvBuilder.ToString()), "text/csv", String.Format("{0}_{1}.csv", report.ReportCode, report.ValueDate.ToString("yyyy-MM-dd"))));
}
else
{
var namePart = entry.Key.FriendlyString();
Array.ForEach(Path.GetInvalidFileNameChars(), c => namePart = namePart.Replace(c.ToString(), String.Empty));
var date = (entry.Key.GraphInstance != null ? entry.Key.GraphInstance.ValueDate : null) ?? DateTime.Now;
var xml = RFXMLSerializer.PrettySerializeContract(content);
return(File(Encoding.UTF8.GetBytes(xml), "text/xml", String.Format("{0}_{3}-{1}-{2}.xml", namePart, shortType, keyReference, date.ToString("yyyyMMdd"))));
}
}
}
return(null);
}
public FileResult ExportEntry(string type, long keyReference)
{
using (var dataEditor = new RFDataEditorActivity(Context, Username))
{
var entry = dataEditor.GetDocumentForDownload(type, keyReference);
if (entry != null)
{
var shortType = System.IO.Path.GetExtension(type).TrimStart('.');
var xml = RFXMLSerializer.PrettySerializeContract(entry);
return(File(Encoding.UTF8.GetBytes(xml), "text/xml", String.Format("RFDocument_{0}-{1}-{2}.xml", shortType, keyReference, DateTime.Now.ToString("yyyyMMdd_HHmmss"))));
}
}
return(null);
}
public object GetDocumentForEdit(string type, long keyReference)
{
var entry = GetDocumentForDownload(type, keyReference);
if (entry != null)
{
return(new
{
Key = RFXMLSerializer.PrettySerializeContract(entry.Key),
Content = RFXMLSerializer.PrettySerializeContract(entry.Content)
});
}
return(null);
}
public bool RequiresApply(RFDate valueDate)
{
try
{
var template = LoadTemplateEntry(valueDate);
var latest = LoadLatestEntry(valueDate);
if (template != null && latest != null)
{
return(RFXMLSerializer.SerializeContract(template.Content) != RFXMLSerializer.SerializeContract(latest.Content));
}
}
catch (Exception ex)
{
Log.Warning(this, ex.Message);
}
return(false);
}
public void SerializationTests()
{
// type-aware serialization
var testDoc = CreateTestDocument();
var reserializeText = RFXMLSerializer.DeserializeContract(typeof(TestDocument).FullName, RFXMLSerializer.PrettySerializeContract(testDoc));
var reserializeBinary = RFXMLSerializer.BinaryDeserializeContract(typeof(TestDocument).FullName, RFXMLSerializer.BinarySerializeContract(testDoc));
CompareTestDoc(reserializeText as TestDocument);
CompareTestDoc(reserializeBinary as TestDocument);
// generic XML serialization, we reserialize to avoid namespace declaration order issues
var originalXmlString = RFXMLSerializer.BinaryDeserializeXML(RFXMLSerializer.BinarySerializeContract(testDoc));
Assert.True(originalXmlString.NotBlank());
Assert.True(originalXmlString.Length > 500);
var binaryRepresentation = RFXMLSerializer.BinarySerializeXML(originalXmlString);
var reserializedXmlString = RFXMLSerializer.BinaryDeserializeXML(binaryRepresentation);
Assert.Equal(originalXmlString, reserializedXmlString);
}
public void CleanUpMaxAge(DateTime utcNow, decimal?maxAge)
{
if (maxAge.HasValue)
{
foreach (var fileEntry in SeenAttributes)
{
var toRemove = fileEntry.Value.Where(e => IsExpired(e, utcNow, maxAge)).ToList();
if (toRemove.Count > 0)
{
foreach (var tr in toRemove)
{
RFStatic.Log.Debug(this, "Removing max aged entry: {0}", RFXMLSerializer.SerializeContract(tr));
}
}
foreach (var remove in toRemove)
{
fileEntry.Value.Remove(remove);
}
}
}
}
public void SerializationTests()
{
var ds1 = new TestSet
{
Rows = new List <TestSet.Row>
{
new TestSet.Row {
A = "b", B = 2
},
new TestSet.Row {
A = "a", B = 1
},
new TestSet.Row {
A = "c", B = 3
},
}
};
var ds2 = new TestSet
{
Rows = new List <TestSet.Row>
{
new TestSet.Row {
A = "a", B = 1
},
new TestSet.Row {
A = "c", B = 3
},
new TestSet.Row {
A = "b", B = 2
},
}
};
var ds1text = RFXMLSerializer.SerializeContract(ds1);
var ds2text = RFXMLSerializer.SerializeContract(ds2);
Assert.Equal(ds1text, ds2text);
}
public RFCatalogEntry Deserialize()
{
return(RFXMLSerializer.DeserializeContract(TypeName, Content) as RFCatalogEntry);
}
public JsonResult GetProcesses(string instanceName, DateTime?instanceDate)
{
using (var activity = new RFEngineActivity(Context, EngineConfig))
{
var engineConfig = activity.GetEngineConfig();
var graphInstance = new RFGraphInstance
{
Name = string.IsNullOrWhiteSpace(instanceName) ? RFGraphInstance.DEFAULT_INSTANCE : instanceName,
ValueDate = instanceDate.HasValue ? new RFDate(instanceDate.Value.Date) : RFDate.NullDate
};
var stats = activity.GetEngineStats(graphInstance);
var allProcesses = new List <object>();
foreach (var process in engineConfig.Processes.Values)
{
var processor = process.Processor();
var stat = stats.GetStat(process.Name);
allProcesses.Add(new
{
Graph = "(none)",
Name = process.Name,
FullName = process.Name,
Description = process.Description,
Type = processor.GetType().FullName,
RequiresIdle = false,
IsGraph = false,
LastRun = stat != null ? (DateTimeOffset?)stat.LastRun : null,
LastDuration = stat != null ? (long?)stat.LastDuration : null,
});
}
foreach (var graph in engineConfig.Graphs.Values)
{
foreach (var process in graph.Processes.Values)
{
var fullName = RFGraphDefinition.GetFullName(graph.GraphName, process.Name);
var stat = stats.GetStat(fullName);
var processor = process.Processor();
var domain = processor.CreateDomain();
var io = new List <object>();
if (domain != null)
{
foreach (var propertyInfo in domain.GetType().GetProperties())
{
var ioBehaviourAttribute = (propertyInfo.GetCustomAttributes(typeof(RFIOBehaviourAttribute), true).FirstOrDefault() as RFIOBehaviourAttribute);
string direction = "-";
string dateType = "-";
RFCatalogKey ioKey = null;
if (ioBehaviourAttribute != null)
{
var ioBehaviour = ioBehaviourAttribute.IOBehaviour;
direction = ioBehaviour.ToString();
var ioMapping = process.IOMappings.FirstOrDefault(m => m.PropertyName == propertyInfo.Name);
if (ioMapping != null)
{
ioKey = ioMapping.Key.CreateForInstance(graphInstance);
var options = RFGraphInstance.ImplyOptions(ioMapping);
dateType = options.DateBehaviour.ToString();
}
}
io.Add(new
{
Field = propertyInfo.Name,
Direction = direction,
DateType = dateType,
DataType = propertyInfo.PropertyType.Name,
KeyType = ioKey != null ? ioKey.GetType().Name : null,
ShortKey = ioKey != null ? ioKey.FriendlyString() : null,
FullKey = ioKey != null ? RFXMLSerializer.SerializeContract(ioKey) : null
});
}
}
allProcesses.Add(new
{
Graph = graph.GraphName,
Name = process.Name,
FullName = fullName,
IsGraph = true,
Description = process.Description,
Type = processor.GetType().FullName,
IO = io,
LastRun = stat != null ? (DateTimeOffset?)stat.LastRun : null,
LastDuration = stat != null ? (long?)stat.LastDuration : null
});
}
}
return(Json(allProcesses));
}
}
private void OnSerializing(StreamingContext ctx)
{
// resort rows by serialized representation - slow but ensures dataset equality
Rows = Rows?.Select(r => new { Key = RFXMLSerializer.SerializeContract(r), Row = r }).OrderBy(r => r.Key).Select(r => r.Row).ToList();
}
public bool HaveSeenFile(string fileKey, RFFileTrackedAttributes attributes)
{
if (SeenAttributes.ContainsKey(fileKey))
{
var seenAttributes = SeenAttributes[fileKey];
var haveSeen = seenAttributes.Any(a => a.Equals(attributes));
if (!haveSeen)
{
RFStatic.Log.Debug(this, "File {0} ({1}) is new. None of existing {2} have-seens match:", fileKey, RFXMLSerializer.SerializeContract(attributes), seenAttributes.Count);
/*foreach(var a in seenAttributes)
* {
* RFStatic.Log.Debug(this, RFXMLSerializer.SerializeContract(a));
* }*/
}
return(haveSeen);
}
else
{
RFStatic.Log.Debug(this, "No seen files entries for file {0}", fileKey);
}
return(false);
}
