public void Select(SelectFilter filter, StatementSink result) {
if (result == null) throw new ArgumentNullException();
foreach (Entity[] s in SplitArray(filter.Subjects))
foreach (Entity[] p in SplitArray(filter.Predicates))
foreach (Resource[] o in SplitArray(filter.Objects))
foreach (Entity[] m in SplitArray(filter.Metas))
{
Select(
ToMultiRes(s),
ToMultiRes(p),
ToMultiRes(o),
ToMultiRes(m),
filter.LiteralFilters,
result,
filter.Limit); // hmm, repeated
}
}
public void Select(SelectFilter filter, StatementSink sink) {
output.WriteLine("SELECT: " + filter);
source.Select(filter, sink);
public void Select(SelectFilter filter, StatementSink sink) {
if (!selfilterresults.ContainsKey(filter)) {
MemoryStore s = new MemoryStore();
source.Select(filter, s);
selfilterresults[filter] = s;
}
((MemoryStore)selfilterresults[filter]).Select(sink);
public override void Select(SelectFilter filter, StatementSink result) {
Entity[] scanMetas = filter.Metas;
filter.Metas = null;
if (scanMetas == null || namedgraphs.Count == 0) scanMetas = new Entity[] { null };
foreach (Entity meta in scanMetas) {
SelectableSource[] sources = GetSources(meta);
if (sources == null) continue;
foreach (SelectableSource s in sources)
s.Select(filter, result);
}
public void Select(SelectFilter filter, StatementSink sink) {
Store.DefaultSelect(this, filter, sink);
public void Select(SelectFilter filter, StatementSink sink)
{
if (filter.Predicates == null || filter.LiteralFilters != null)
{
data.Select(filter, sink);
return;
}
ResSet remainingPredicates = new ResSet();
Entity[] subjects = filter.Subjects;
Entity[] predicates = filter.Predicates;
Resource[] objects = filter.Objects;
Entity[] metas = filter.Metas;
foreach (Entity p in predicates)
{
if (p == type)
{
if (objects != null)
{
// Do the subjects have any of the types listed in the objects,
// or what things have those types?
// Expand objects by the subclass closure of the objects
data.Select(new SelectFilter(subjects, new Entity[] { p }, GetClosure(objects, subclasses), metas), sink);
// Process domains and ranges.
ResSet dom = new ResSet(), ran = new ResSet();
Hashtable domPropToType = new Hashtable();
Hashtable ranPropToType = new Hashtable();
foreach (Entity e in objects)
{
Entity[] dc = GetClosure((ResSet)domainof[e], subprops);
if (dc != null)
{
foreach (Entity c in dc)
{
dom.Add(c);
AddRelation(c, e, domPropToType, false);
}
}
dc = GetClosure((ResSet)rangeof[e], subprops);
if (dc != null)
{
foreach (Entity c in dc)
{
ran.Add(c);
AddRelation(c, e, ranPropToType, false);
}
}
}
// If it's in the domain of any of these properties,
// we know its type.
if (subjects != null)
{
if (dom.Count > 0)
{
data.Select(new SelectFilter(subjects, dom.ToEntityArray(), null, metas), new ExpandDomRan(0, domPropToType, sink));
}
if (ran.Count > 0)
{
data.Select(new SelectFilter(null, ran.ToEntityArray(), subjects, metas), new ExpandDomRan(1, ranPropToType, sink));
}
}
}
else if (subjects != null)
{
// What types do these subjects have?
// Expand the resulting types by the closure of their superclasses
data.Select(new SelectFilter(subjects, new Entity[] { p }, objects, metas), new Expand(superclasses, sink));
// Use domains and ranges to get type info
data.Select(new SelectFilter(subjects, null, null, metas), new Expand3(0, domains, superclasses, sink));
data.Select(new SelectFilter(null, null, subjects, metas), new Expand3(1, ranges, superclasses, sink));
}
else
{
// What has type what? We won't answer that question.
data.Select(filter, sink);
}
}
else if ((p == subClassOf || p == subPropertyOf) &&
(metas == null || metas[0] == Statement.DefaultMeta))
{
Hashtable supers = (p == subClassOf) ? superclasses : superprops;
Hashtable subs = (p == subClassOf) ? subclasses : subprops;
if (subjects != null && objects != null)
{
// Expand objects by the subs closure of the objects.
data.Select(new SelectFilter(subjects, new Entity[] { p }, GetClosure(objects, subs), metas), sink);
}
else if (subjects != null)
{
// get all of the supers of all of the subjects
foreach (Entity s in subjects)
{
foreach (Entity o in GetClosure(new Entity[] { s }, supers))
{
sink.Add(new Statement(s, p, o));
}
}
}
else if (objects != null)
{
// get all of the subs of all of the objects
foreach (Resource o in objects)
{
if (o is Literal)
{
continue;
}
foreach (Entity s in GetClosure(new Entity[] { (Entity)o }, subs))
{
sink.Add(new Statement(s, p, (Entity)o));
}
}
}
else
{
// What is a subclass/property of what? We won't answer that.
data.Select(filter, sink);
}
}
else
{
remainingPredicates.Add(p);
}
}
if (remainingPredicates.Count > 0)
{
// Also query the subproperties of any property
// being queried, but remember which subproperties
// came from which superproperties so we can map them
// back to the properties actually queried. The closures
// contain the queried properties themselves too.
ResSet qprops = new ResSet();
Hashtable propfrom = new Hashtable();
foreach (Entity p in remainingPredicates)
{
foreach (Entity sp in GetClosure(new Entity[] { p }, subprops))
{
AddRelation(sp, p, propfrom, false);
qprops.Add(sp);
}
}
//data.Select(subjects, qprops.ToEntityArray(), objects, metas, new LiteralDTMap(ranges, new PredMap(propfrom, sink)));
SelectFilter sf = new SelectFilter(subjects, qprops.ToEntityArray(), objects, metas);
sf.LiteralFilters = filter.LiteralFilters;
sf.Limit = filter.Limit;
data.Select(sf, new PredMap(propfrom, sink));
}
}
public override void Select(SelectFilter filter, StatementSink sink)
{
throw new NotImplementedException();
}
protected void LoadToSink(SourceDescriptor sourceDescr, IList <object> ids, IList <FieldDescriptor> predFlds, Resource[] vals, StatementSink sink, SelectFilter flt)
{
// todo: more effective impl using IDbDalc datareader
var ds = new DataSet();
var q = new Query(sourceDescr.SourceName);
var flds = predFlds ?? sourceDescr.Fields;
var loadType = flt.Predicates == null || flt.Predicates.Contains(NS.Rdf.typeEntity);
q.Fields = new string[flds.Count + 1];
q.Fields[0] = sourceDescr.IdFieldName;
for (int i = 0; i < flds.Count; i++)
{
q.Fields[i + 1] = flds[i].FieldName;
}
// compose query condition
var condition = new QueryGroupNode(GroupType.And);
if (ids != null)
{
condition.Nodes.Add(ComposeCondition(sourceDescr.IdFieldName, ids.ToArray()));
}
if (vals != null && !loadType)
{
var orGrp = new QueryGroupNode(GroupType.Or);
for (int i = 0; i < flds.Count; i++)
{
var valCnd = ComposeCondition(flds[i], vals);
if (valCnd != null)
{
orGrp.Nodes.Add(valCnd);
}
}
if (orGrp.Nodes.Count == 0)
{
return; //values are not for this source
}
condition.Nodes.Add(orGrp);
}
if (flt.LiteralFilters != null)
{
var literalFltCondition = ComposeLiteralFilter(flds, flt.LiteralFilters);
if (literalFltCondition != null)
{
condition.Nodes.Add(literalFltCondition);
}
}
q.Root = condition;
if (flt.Limit > 0)
{
q.RecordCount = flt.Limit;
}
// log
log.Write(LogEvent.Debug, q);
Console.WriteLine(q.ToString());
// query result handler
Action <IDataReader> loadToSinkAction = delegate(IDataReader dataReader) {
int recIndex = 0;
while (dataReader.Read() && (recIndex < q.RecordCount))
{
recIndex++;
var itemEntity = GetSourceItemEntity(sourceDescr, dataReader[sourceDescr.IdFieldName]);
for (int j = 0; j < flds.Count; j++)
{
var f = flds[j];
var obj = PrepareResource(f, dataReader[f.FieldName]);
if (vals == null || vals.Contains(obj))
{
// literals post-filter
if (flt.LiteralFilters != null && !LiteralFilter.MatchesFilters(obj, flt.LiteralFilters, this))
{
continue;
}
if (!sink.Add(new Statement(itemEntity, EntityFieldHash[f], obj)))
{
return;
}
}
}
// type predicate
if (loadType && flt.LiteralFilters == null)
{
if (vals == null || vals.Contains(EntitySourceHash[sourceDescr]))
{
if (!sink.Add(
new Statement(itemEntity, NS.Rdf.typeEntity, EntitySourceHash[sourceDescr])))
{
return;
}
}
}
}
};
// DB DALC datareader optimization
if (Dalc is IDbDalc)
{
var dbDalc = (IDbDalc)Dalc;
bool closeConn = false;
try {
if (dbDalc.Connection.State != ConnectionState.Open)
{
dbDalc.Connection.Open();
closeConn = true;
}
IDataReader rdr = dbDalc.LoadReader(q);
try {
loadToSinkAction(rdr);
} finally {
rdr.Close();
}
} finally {
if (closeConn)
{
dbDalc.Connection.Close();
}
}
}
else
{
Dalc.Load(ds, q);
var tblRdr = ds.Tables[q.SourceName].CreateDataReader();
try {
loadToSinkAction(tblRdr);
} finally {
tblRdr.Close();
}
}
}
public async Task <IList <FarmResult> > SelectAsync(SelectFilter filter, int page, int pageSize)
{
var famrs = await _farmRepository.SelectAsync(filter, page, pageSize);
return(famrs);
}
static async Task Main(string[] args)
{
Console.WriteLine("Resolving translations");
CultureInfo[] cultures = CultureInfo.GetCultures(CultureTypes.NeutralCultures);
string rootPath = AppDomain.CurrentDomain.BaseDirectory,
unresolvedTranslationPath = Path.Combine(rootPath, "UnResolvedTranslations"),
resolvedUploadPath = Path.Combine(rootPath, "Upload", "Internal", "Translations");
Dictionary <string, Dictionary <string, string> > fallbackTranslations = JsonConvert.DeserializeObject <Dictionary <string, Dictionary <string, string> > >(File.ReadAllText(Path.Combine(unresolvedTranslationPath, "en.json")));
List <string> sections = new List <string>();
foreach (string section in fallbackTranslations.Keys)
{
string resolvedSectionUploadPath = Path.Combine(resolvedUploadPath, section);
if (!Directory.Exists(resolvedSectionUploadPath))
{
Directory.CreateDirectory(resolvedSectionUploadPath);
}
}
foreach (CultureInfo currentCulture in cultures)
{
string locale = currentCulture.TwoLetterISOLanguageName,
unresolvedPath = Path.Combine(unresolvedTranslationPath, $"{locale}.json");
if (locale == "")
{
continue;
}
if (!File.Exists(unresolvedPath))
{
foreach (string section in fallbackTranslations.Keys)
{
File.WriteAllText(Path.Combine(resolvedUploadPath, section, $"{locale}.json"), JsonConvert.SerializeObject(fallbackTranslations[section]));
sections.Add($"{section}/{locale}.json");
}
}
else
{
Dictionary <string, Dictionary <string, string> > unresolvedTranslations = JsonConvert.DeserializeObject <Dictionary <string, Dictionary <string, string> > >(File.ReadAllText(unresolvedPath));
foreach (string section in fallbackTranslations.Keys)
{
if (!unresolvedTranslations.ContainsKey(section))
{
File.WriteAllText(Path.Combine(resolvedUploadPath, section, $"{locale}.json"), JsonConvert.SerializeObject(fallbackTranslations[section]));
}
else
{
Dictionary <string, string> fallbackTranslationsSection = fallbackTranslations[section],
unresolvedTranslationsSection = unresolvedTranslations[section];
foreach (KeyValuePair <string, string> translation in fallbackTranslationsSection)
{
if (!unresolvedTranslationsSection.ContainsKey(translation.Key))
{
unresolvedTranslationsSection.Add(translation.Key, translation.Value);
}
}
File.WriteAllText(Path.Combine(resolvedUploadPath, section, $"{locale}.json"), JsonConvert.SerializeObject(unresolvedTranslationsSection));
}
sections.Add($"{section}/{locale}.json");
}
}
}
Console.WriteLine("Resolving tile definitions");
string tileDefinitions = Path.Combine(rootPath, "Upload", "Public", "DataFiles", "V1"),
unresolvedTileDefinitions = Path.Combine(tileDefinitions, "TileDefinition.json"),
resolvedTileDefinitionsUploadPath = Path.Combine(tileDefinitions, "TileSearch.json");
IEnumerable <UnResolvedTag> tags = JsonConvert.DeserializeObject <TileConfiguration>(File.ReadAllText(unresolvedTileDefinitions)).tiles.SelectMany(t => t.tags);
List <SelectFilter> selectFilters = new List <SelectFilter>();
List <DateFilter> dateFilters = new List <DateFilter>();
List <BooleanFilter> booleanFilters = new List <BooleanFilter>();
foreach (UnResolvedTag tag in tags)
{
string titleTranslation = $"tag{tag.name}TitleTranslation";
if (!string.IsNullOrWhiteSpace(tag.selectValue))
{
SelectFilter currentSelectFilter = selectFilters.FirstOrDefault(t => t.name == tag.name);
SelectFilterOption currentOption = new SelectFilterOption
{
value = tag.selectValue,
translation = $"tag{tag.name}Value{tag.selectValue}Translation"
};
if (currentSelectFilter == null)
{
selectFilters.Add(new SelectFilter()
{
name = tag.name,
titleTranslation = titleTranslation,
options = new List <SelectFilterOption>
{
currentOption
},
optionTranslations = new Dictionary <string, string>
{
{
currentOption.value,
currentOption.translation
}
}
});
}
else if (!currentSelectFilter.options.Any(t => t.value == tag.selectValue))
{
currentSelectFilter.options.Add(currentOption);
currentSelectFilter.optionTranslations.Add(currentOption.value, currentOption.translation);
}
}
else if (tag.dateValue.HasValue)
{
DateFilter currentDateFilter = dateFilters.FirstOrDefault(t => t.name == tag.name);
DateTime currentDate = tag.dateValue.Value;
if (currentDateFilter == null)
{
dateFilters.Add(new DateFilter
{
name = tag.name,
titleTranslation = titleTranslation,
minimumValue = currentDate,
maximumValue = currentDate
});
}
else if (currentDate < currentDateFilter.minimumValue)
{
currentDateFilter.minimumValue = currentDate;
}
else if (currentDate > currentDateFilter.maximumValue)
{
currentDateFilter.maximumValue = currentDate;
}
}
else if (tag.booleanValue.HasValue && !booleanFilters.Any(t => t.name == tag.name))
{
booleanFilters.Add(new BooleanFilter
{
name = tag.name,
titleTranslation = titleTranslation
});
}
}
Dictionary <string, SelectFilter> selectFiltersByName = new Dictionary <string, SelectFilter>();
Dictionary <string, DateFilter> dateFiltersByName = new Dictionary <string, DateFilter>();
Dictionary <string, BooleanFilter> booleanFiltersByName = new Dictionary <string, BooleanFilter>();
File.WriteAllText(resolvedTileDefinitionsUploadPath, JsonConvert.SerializeObject(new
{
selectFilters,
selectFiltersByName = selectFilters.ToDictionary(sf => sf.name, sf => sf),
dateFilters,
dateFiltersByName = dateFilters.ToDictionary(sf => sf.name, sf => sf),
booleanFilters,
booleanFiltersByName = booleanFilters.ToDictionary(sf => sf.name, sf => sf)
}));
Console.WriteLine("Putting all the files into blob");
var provider = new ServiceCollection()
.AddMemoryCache()
.BuildServiceProvider();
var cache = provider.GetService <IMemoryCache>();
var keyVaultHelper = new KeyVaultHelper(
Options.Create(
new KeyVaultConfiguration()
{
BaseSecretURL = "https://cvvault.vault.azure.net/secrets/"
}
), cache);
var blobHelper = new BlobStoreHelper(
Options.Create(
new BlobStoreConfiguration()
{
InternalStoreSecretURL = "internalFilesStoreConnectionString",
PublicStoreSecretURL = "publicFilesStoreConnectionString"
}
),
keyVaultHelper
);
await blobHelper.UploadFolderToBlob(Path.Combine(rootPath, "Upload", "Internal"), BlobContainerType.Internal);
await blobHelper.UploadFolderToBlob(Path.Combine(rootPath, "Upload", "Public"), BlobContainerType.Public);
Console.WriteLine("Sending update message to subscribers");
await new ServiceBusHelper(
Options.Create(
new ServiceBusConfiguration()
{
ServiceBusSecretURL = "serviceBusConnectionString"
}
),
keyVaultHelper
).SendMessage(new ResetTranslationsCacheMessage(ServiceBusTopic.ResetTranslationsCache, sections));
}
public void Select(SelectFilter filter, StatementSink sink) {
Select(filter.Subjects, filter.Predicates, filter.Objects, filter.Metas, filter.LiteralFilters, filter.Limit, sink, false);
}
private bool WhereClause(SelectFilter filter, System.Text.StringBuilder cmd)
{
if (filter.Subjects == null && filter.Predicates == null && filter.Objects == null && filter.Metas == null)
return false;
cmd.Append(" WHERE ");
if (filter.Subjects != null)
WhereItem("subject", filter.Subjects, cmd, false);
if (filter.Predicates != null)
WhereItem("predicate", filter.Predicates, cmd, filter.Subjects != null);
if (filter.Objects != null)
WhereItem("object", filter.Objects, cmd, filter.Subjects != null || filter.Predicates != null);
if (filter.Metas != null)
WhereItem("meta", filter.Metas, cmd, filter.Subjects != null || filter.Predicates != null || filter.Objects != null);
return true;
}
private void Select2(SelectFilter filter, StatementSink result)
{
// Don't select on columns that we already know from the template.
SelectColumnFilter columns = new SelectColumnFilter();
columns.Subject = (filter.Subjects == null) || (filter.Subjects.Length > 1);
columns.Predicate = (filter.Predicates == null) || (filter.Predicates.Length > 1);
columns.Object = (filter.Objects == null) || (filter.Objects.Length > 1);
columns.Meta = (filter.Metas == null) || (filter.Metas.Length > 1);;
if (filter.Predicates != null | Array.IndexOf(filter.Predicates, rdfs_member) != 1)
columns.Predicate = true;
// Have to select something
if (!columns.Subject && !columns.Predicate && !columns.Object && !columns.Meta)
columns.Subject = true;
System.Text.StringBuilder cmd = new System.Text.StringBuilder("SELECT ");
if (!connection.AreStatementsUnique)
cmd.Append("DISTINCT ");
ArrayList cols = new ArrayList();
if (columns.Subject) { cols.Add("sinfo.type"); cols.Add("sinfo.value"); }
if (columns.Predicate) { cols.Add("pinfo.type"); cols.Add("pinfo.value"); }
if (columns.Object) { cols.Add("oinfo.type"); cols.Add("oinfo.value"); cols.Add("oinfo.language"); cols.Add("oinfo.datatype"); }
if (columns.Meta) { cols.Add("minfo.type"); cols.Add("minfo.value"); }
cmd.Append(String.Join(", ", (String[])cols.ToArray(typeof(String))));
cmd.Append(" FROM ");
cmd.Append(prefix);
cmd.Append("_statements AS q");
if (columns.Subject) {
cmd.Append(" LEFT JOIN ");
cmd.Append(prefix);
cmd.Append("_values AS sinfo ON q.subject = sinfo.id");
}
if (columns.Predicate) {
cmd.Append(" LEFT JOIN ");
cmd.Append(prefix);
cmd.Append("_values AS pinfo ON q.predicate = pinfo.id");
}
if (columns.Object) {
cmd.Append(" LEFT JOIN ");
cmd.Append(prefix);
cmd.Append("_values AS oinfo ON q.object = oinfo.id");
}
if (columns.Meta) {
cmd.Append(" LEFT JOIN ");
cmd.Append(prefix);
cmd.Append("_values AS minfo ON q.meta = minfo.id");
}
cmd.Append(' ');
bool wroteWhere = WhereClause(filter, cmd);
// Transform literal filters into SQL.
if (filter.LiteralFilters != null) {
foreach (LiteralFilter f in filter.LiteralFilters) {
string s = FilterToSQL(f, "oinfo.value");
if (s != null) {
if (!wroteWhere) { cmd.Append(" WHERE "); wroteWhere = true; }
else { cmd.Append(" AND "); }
cmd.Append(' ');
cmd.Append(s);
}
}
}
if (filter.Limit >= 1) {
cmd.Append(" LIMIT ");
cmd.Append(filter.Limit);
}
cmd.Append(';');
if (Debug) {
string cmd2 = cmd.ToString();
//if (cmd2.Length > 80) cmd2 = cmd2.Substring(0, 80);
Console.Error.WriteLine(cmd2);
}
using (IDataReader reader = connection.RunReader(cmd.ToString())) {
while (reader.Read()) {
Entity s = columns.Subject ? null : filter.Subjects[0];
Entity p = columns.Predicate ? null : filter.Predicates[0];
Resource o = columns.Object ? null : filter.Objects[0];
Entity m = columns.Meta ? null : filter.Metas[0];
int col = 0;
if (columns.Subject) { s = SelectEntity(reader.GetInt32(col++), reader.GetString(col++)); }
if (columns.Predicate) { p = SelectEntity(reader.GetInt32(col++), reader.GetString(col++)); }
if (columns.Object) { o = SelectResource(reader.GetInt32(col++), reader.GetString(col++), reader.GetString(col++), reader.GetString(col++)); }
if (columns.Meta) { m = SelectEntity(reader.GetInt32(col++), reader.GetString(col++)); }
if (filter.LiteralFilters != null && !LiteralFilter.MatchesFilters(o, filter.LiteralFilters, this))
continue;
bool ret = result.Add(new Statement(s, p, o, m));
if (!ret) break;
}
}
}
public void Select(SelectFilter filter, StatementSink result)
{
// We don't want to select on more than say 1000 resources
// at a time, so this breaks down the selection into
// a union of selections that each select on no more than
// 1000 resources.
if (result == null) throw new ArgumentNullException();
Init();
foreach (Entity[] s in SplitArray(filter.Subjects))
foreach (Entity[] p in SplitArray(filter.Predicates))
foreach (Resource[] o in SplitArray(filter.Objects))
foreach (Entity[] m in SplitArray(filter.Metas))
{
SelectFilter f = new SelectFilter(s, p, o, m);
f.LiteralFilters = filter.LiteralFilters;
f.Limit = filter.Limit; // TODO: Do the limit better since it should shrink on each iteration.
Select2(f, result);
}
}
public void Filter(string widget = "")
{
Options("Filter").Clicks();
ScpType.SelectDropdown("Number Of Period");
if (widget == "Sparkline" || widget == "Trend Chart")
{
ScpType.SelectDropdown("Year To Date");
}
if (widget == "")
{
Numberof.ClearText(); Numberof.EnterText("400");
AddFilter("Data").Clicks();
//SingleFilter.Clicks();
SelectFilter.WaitUntil();
SelectFilter.SelectDropdown("Parent Task Type");
ItemsCheck("Data").Clicks();
CheckItem("120").Clicks();
}
else if (widget == "Compliance Heatmap")
{
Numberof.ClearText(); Numberof.EnterText("400");
AddFilter("Data").Clicks();
SelectFilter.WaitUntil();
SelectFilter.SelectDropdown("User Compliances");
ItemsCheck("Data").Clicks();
CheckItem("On-Time").Clicks();
CheckItem("Unidentified").Clicks();
}
else if (widget == "Compliance Table")
{
ScpType.SelectDropdown("Month To Date");
Threshold.ClearText(); Threshold.EnterText("1");
}
else if (widget == "Heatmap")
{
ScpType.SelectDropdown("Month To Date");
}
else if (widget == "Trend Chart" || widget == "Heatmap")
{
AddFilter("Data").Clicks();
SelectFilter.WaitUntil();
SelectFilter.SelectDropdown("Parent Task Type");
ItemsCheck("Data").Clicks();
CheckItem("120").Clicks();
}
else if (widget == "Heatmap")
{
ScpType.SelectDropdown("Month To Date");
}
//CheckAll.Clicks();
}
public void Select(SelectFilter filter, StatementSink sink)
{
throw new NotImplementedException("Select");
// FIXME: not implemented yet, SelectFilter are a little more complex
// than Statements with wildcards
}
