public IJobLoader Parse(JobParser parser, ParseState state)
{
var connection = state.Require <string>("Database", true);
if (!connection.Contains("="))
{
var builder =
new SqlConnectionStringBuilder(
ConfigurationManager.ConnectionStrings["ExperienceExtractor.Sql"].ConnectionString);
builder.InitialCatalog = connection;
connection = builder.ConnectionString;
}
using (var conn = new SqlConnection(connection))
{
conn.Open();
if (DBNull.Value.Equals(new SqlCommand("SELECT OBJECT_ID('Sitecore.JobInfo', 'U')", conn).ExecuteScalar()))
{
throw new ParseException(state, "Database does not contain the job info table. Was it created with Experience Extractor?");
}
var json = (string)new SqlCommand("SELECT TOP 1 [Prototype] FROM Sitecore.JobInfo", conn).ExecuteScalar();
if (string.IsNullOrEmpty(json))
{
throw new ParseException(state, "Database does not contain a job prototype");
}
var job = new JsonJobParser(JObject.Parse(json));
return(new StaticJobLoader(job));
}
}
public ITableMapper Parse(JobParser parser, ParseState state)
{
var table = parser.ParseTableDefinition(state.Select("Table") ?? state.Select(null, true));
Type itemType;
var selector = Selectors.SelectFromName(GetSelector(parser, state), out itemType);
var expand = state.Select("Expand");
if (expand != null)
{
var expandField = parser.ParseFieldMapper(expand) as SimpleFieldMapper;
if (expandField == null)
{
throw ParseException.AttributeError(expand, "Invalid field specified");
}
var delimiter = state.TryGet("Delimiter", "|");
return(new SimpleTableMapper(selector,
new TableDefinition("").Map(
new SimpleTableMapper(
scope =>
expandField.Selector(scope)
.TryGet(v => "" + v, "")
.Split(new[] { delimiter }, StringSplitOptions.RemoveEmptyEntries), table))));
}
return(new SimpleTableMapper(selector, table));
}
/// <summary>
/// Look for all jobs that match a certian criteria.
/// </summary>
protected override void ProcessRecord()
{
Func <AtlasJob, bool> nameSelector = _ => true;
Func <AtlasJob, bool> versionSelector = _ => true;
if (!string.IsNullOrWhiteSpace(JobName))
{
var matcher = new Regex(JobName.Replace("*", ".*"));
nameSelector = j => matcher.Match(j.Name).Success;
}
if (!string.IsNullOrWhiteSpace(JobVersion))
{
int version = 0;
if (!int.TryParse(JobVersion, out version))
{
throw new ArgumentException(string.Format("JobVersion must be a valid integer, not '{0}'", JobVersion));
}
versionSelector = j => j.Version == version;
}
// Now we can actually go through and get all the jobs.
var jobs = JobParser.FindJobs(j => nameSelector(j) && versionSelector(j));
// And return what we found out.
foreach (var j in jobs.Select(fullSpec => new AtlasJobSpec()
{
JobName = fullSpec.Name, JobVersion = fullSpec.Version
}))
{
WriteObject(j);
}
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
return(new SimpleFieldMapper("AITraining",
scope => GetTrainingValue(scope.Current <IVisitAggregationContext>().Contact),
valueType: typeof(string),
fieldType: FieldType.Dimension));
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var tableName = state.Require <string>("Name");
var inline = state.TryGet("Inline", false);
var fieldState = state.Clone();
var affix = state.TryGet("AffixNames", true);
if (affix)
{
tableName = state.AffixName(tableName);
}
else
{
fieldState = fieldState.ClearAffix();
}
var fieldPrefix = state.TryGet("Prefix", "");
return(new FieldMapperSet(tableName, inline,
fieldState.SelectMany("Fields").Select(parser.ParseFieldMapper), state.AffixName,
fieldNamePrefix: fieldPrefix)
{
Key = state.TryGet("Key", false)
});
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var testIdString = state.Require <string>("TestId", true);
Guid testId;
if (!Guid.TryParse(testIdString, out testId))
{
throw state.AttributeError("Invalid test id specified ({0})", testIdString);
}
var testSet = TestManager.GetTestSet(testId);
var testMapper = new MvTestFieldMapper(testSet, state.AffixName(state.TryGet("Name", "MvTest")));
if (state.SelectMany("Fields").Any())
{
var splitter = new SplittingFieldMapper(new MvTestSplitter(testId),
postfix =>
state.Postfix(postfix).SelectMany("Fields").Select(parser.ParseFieldMapper).ToArray());
return(new FieldMapperSet(testMapper.Name, true, new IFieldMapper[] { testMapper, splitter }, state.AffixName));
}
return(testMapper);
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var name = state.AffixName(state.TryGet("Name", "Count", true));
var scopeOffset = Math.Abs(state.TryGet("Scope", 0));
return(new SimpleFieldMapper(name, ctx =>
{
if (scopeOffset == 0)
{
return 1;
}
var p = ctx;
for (var i = 0; i < scopeOffset && p != null; i++)
{
p = p.Parent;
}
if (p != null)
{
return ctx.OncePerScope(p, 1);
}
return 0;
}, typeof(int), FieldType.Fact));
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var source = state.TryGet("Select", "Pages", true);
var path = source.Split('.');
source = path[0];
var name = state.TryGet("Name", source);
Type itemType;
var items = Selectors.SelectFromName(source, out itemType);
var selector = string.Join(".", path.Skip(1));
if (string.IsNullOrEmpty(selector))
{
selector = Selectors.DefaultSelector(source);
}
var getterInfo = parser.CompileGetter(itemType, selector);
return(new SequenceFieldMapper(name,
scope => items(scope).Select(item => getterInfo.Getter(item, scope)),
(SequenceType)
Enum.Parse(typeof(SequenceType), state.TryGet("Type", "Path"), true)));
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var name = state.AffixName(state.TryGet("Name", "Index", true));
var scopeOffset = Math.Abs(state.TryGet("Scope", -1, true));
return(new SimpleFieldMapper(name, ctx =>
{
if (scopeOffset == 0)
{
return ctx.GlobalIndex;
}
var p = ctx;
for (var i = 0; i < scopeOffset && p != null; i++)
{
p = p.Parent;
}
if (p != null)
{
return ctx.Index(p);
}
return 0;
}, typeof(int), FieldType.Dimension));
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var selector = state.TryGet("Select", DefaultPath, true);
if (selector.StartsWith("/"))
{
selector = DefaultPath + selector;
}
var getter = parser.CompileGetter(typeof(T), selector);
var name = state.TryGet("Name", DefaultName ?? SuggestName(getter.SuggestedName));
var fieldType = (FieldType)Enum.Parse(typeof(FieldType), state.TryGet("Type", "Dimension"), true);
IFieldMapper mapper = new SimpleFieldMapper(state.AffixName(name),
scope => SelectItem(scope).TryGet(item => getter.Getter(item, scope), getter.DefaultValue),
getter.ValueType, fieldType);
var labelState = state.Select("Labels");
if (labelState != null)
{
var labels = labelState.Keys.ToDictionary(labelState.AffixName, key =>
(ILabelProvider)
new SitecoreFieldLabelProvider(labelState.TryGet <string>(key), parser.DefaultLanguage));
if (labels.Count > 0)
{
return(new LabeledFieldMapper(mapper, labels));
}
}
return(mapper);
}
public void JobParserTestMethod()
{
bool extraMargin = true;
bool taxExempt = false;
string description = "envelopes";
decimal price = 556.40m;
JobParser job1 = new JobParser("../../JobFiles/Job1.txt");
job1.LoadJobElements();
JobComponents temp = job1.getJob(0);
Assert.AreEqual(description, temp.getJobType);
Assert.AreEqual(extraMargin, temp.getIsExtraMargin);
Assert.AreEqual(price, temp.getPrice);
Assert.AreEqual(taxExempt, temp.getIsTaxExempt);
extraMargin = true;
taxExempt = true;
description = "letterhead";
price = 1983.37m;
temp = job1.getJob(1);
Assert.AreEqual(description, temp.getJobType);
Assert.AreEqual(extraMargin, temp.getIsExtraMargin);
Assert.AreEqual(price, temp.getPrice);
Assert.AreEqual(taxExempt, temp.getIsTaxExempt);
}
public ISplitter Parse(JobParser parser, ParseState state)
{
var pageIdString = state.TryGet <string>("PageId", mainParameter: true);
var includeSelf = state.TryGet("IncludePage", false);
var pageId = pageIdString != null?Guid.Parse(pageIdString) : (Guid?)null;
return(new PageConditionSplitter((context, page) =>
{
var thisPageId = pageId;
if (!thisPageId.HasValue && context != null)
{
var e = context.Current <PageData>();
if (e != null)
{
thisPageId = e.Item.Id;
}
else
{
var d = context.Current <DimensionData>();
if (d != null)
{
Guid result;
thisPageId = Guid.TryParse(d.DimensionKey, out result) ? result : (Guid?)null;
}
}
}
return thisPageId.HasValue && page.Item.Id == thisPageId;
}, includeMatchBefore: !includeSelf));
}
public static GetterInfo CompileGetter(this JobParser parser, Type type, string selector, Language language = null)
{
var slash = selector.IndexOf('/');
var path = (slash != -1 ? selector.Substring(0, slash) : selector).Split(new[] { "." }, StringSplitOptions.RemoveEmptyEntries);
var scField = slash != -1 ? selector.Substring(slash + 1) : null;
Type valueType;
var valueGetter = CompileGetter(type, path, out valueType);
Func <object, ProcessingScope, object> getter = (item, scope) => valueGetter(item);
if (!string.IsNullOrEmpty(scField))
{
valueType = typeof(string);
getter =
(item, scope) =>
valueGetter(item).TryGet(id => scope.FieldLookup.TryGet(lu => lu.Lookup(id, scField, language)));
}
return(new GetterInfo
{
Path = path,
SitecoreField = scField,
Type = type,
ValueType = valueType,
Getter = getter
});
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var splitter = parser.ParseSplitter(state.Select("Splitter") ?? state);
return(new SplittingFieldMapper(splitter, postfix =>
state.Postfix(postfix).SelectMany("Fields").Select(parser.ParseFieldMapper).ToArray()
));
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var factTypes = XaFactsFieldFactory.ParseFactTypes(state.TryGet("Types", new string[0], true));
var prefix = state.TryGet <string>("Prefix");
return(new FactsMapper(factTypes, state.Prefix(prefix).AffixName, state.TryGet("EntireVisit", false)));
}
public void JobParser_ValidJobWithParent_ShouldParse()
{
IJobParser parser = new JobParser();
var result = parser.Parse("A => B");
Assert.AreEqual(result.JobId, "A");
Assert.AreEqual(result.DependantJobId, "B");
}
/// <summary>
/// Do the lookup!
/// </summary>
protected override void ProcessRecord()
{
// Setup for verbosity if we need it.
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
PandaTask t = null;
bool needdatasets = OutputContainerNames.IsPresent || InputContainerNames.IsPresent;
if (ParameterSetName == "TaskID")
{
t = TaskID.FindPandaJobWithTaskName(needdatasets);
}
if (ParameterSetName == "TaskObject")
{
t = PandaTaskObject.ID.FindPandaJobWithTaskName(needdatasets);
}
if (ParameterSetName == "TaskName")
{
t = TaskName.FindPandaJobWithTaskName(needdatasets);
}
if (ParameterSetName == "DatasetGRIDJob")
{
// Get the job and resulting dataset name.
var job = JobParser.FindJob(JobName, JobVersion);
var ds = job.ResultingDataSetName(DatasetName.Trim(), JobIteration);
// Now, look up the job itself.
t = (ds + "/").FindPandaJobWithTaskName();
}
// We really can't deal with a bad task below. The returned objects are sufficiently complex.
if (t == null)
{
throw new ArgumentException("Unable to find the task in panda: was it ever submitted?");
}
// Dump the info to the output pipeline as requested.
if (JobStatus.IsPresent)
{
WriteObject(t.status);
}
if (InputContainerNames.IsPresent)
{
WriteObject(t.DataSetNamesIN());
}
if (OutputContainerNames.IsPresent)
{
WriteObject(t.DataSetNamesOUT());
}
} finally
{
Trace.Listeners.Remove(listener);
}
}
public IDataFilter Parse(JobParser parser, ParseState state)
{
var p = state.Require <double>("Percentage", mainParameter: true);
var offset = state.TryGet("Offset", 0d);
var filter = new MongoRandomSampleFilter(Math.Max(0, offset), Math.Min(1, offset + p));
return(filter);
}
/// <summary>
/// Process a single dataset and fetch the info about it.
/// </summary>
protected override void ProcessRecord()
{
var trimmedDSName = DatasetName.Trim();
if (_resultsCache.Value[trimmedDSName] is PSGRIDDatasetInfo cHit)
{
WriteObject(cHit);
}
else
{
// Setup for verbosity if we need it.
var listener = new PSListener(this);
Trace.Listeners.Add(listener);
try
{
// Where are we going to be doing query on - we need a machine.
var sm = JobParser.GetSubmissionMachine();
// Get the remove environment configured if it needs to be
if (_connection == null)
{
_connection = new SSHConnection(sm.MachineName, sm.UserName);
_connection
.Apply(() => DisplayStatus("Setting up ATLAS"))
.setupATLAS()
.Apply(() => DisplayStatus("Setting up Rucio"))
.setupRucio(_gridCredentials.Username)
.Apply(() => DisplayStatus("Acquiring GRID credentials"))
.VomsProxyInit("atlas", failNow: () => Stopping);
}
// Great - get the info on this dataset.
var fileInfo = _connection
.Apply(() => DisplayStatus($"Checking for info on {trimmedDSName}."))
.FileInfoFromGRID(trimmedDSName, failNow: () => Stopping);
// Next, build the resulting thingy.
var r = new PSGRIDDatasetInfo()
{
DatasetName = trimmedDSName,
nFiles = fileInfo.Count,
TotalSizeMB = (int)fileInfo.Sum(fi => fi.size),
FileInfo = fileInfo.ToArray()
};
_resultsCache.Value[trimmedDSName] = r;
using (var pp = listener.PauseListening())
{
WriteObject(r);
}
}
finally
{
Trace.Listeners.Remove(listener);
}
}
}
protected override IFieldMapper Parse(Func<ProcessingScope, DateTime?> selector, string defaultName,
JobParser parser, ParseState state)
{
return new TimeDimension(
state.AffixName(state.TryGet("Name", defaultName + "Time")),
selector,
inlineFields: state.TryGet<bool>("Inline"),
cultureInfo: parser.DefaultLanguage.CultureInfo,
detailLevel: state.TryGet("Resolution", TimeDetailLevel.Hour, true));
}
protected override IFieldMapper Parse(Func <ProcessingScope, DateTime?> selector, string defaultName,
JobParser parser, ParseState state)
{
return(new TimeDimension(
state.AffixName(state.TryGet("Name", defaultName + "Time")),
selector,
inlineFields: state.TryGet <bool>("Inline"),
cultureInfo: parser.DefaultLanguage.CultureInfo,
detailLevel: state.TryGet("Resolution", TimeDetailLevel.Hour, true)));
}
/// <summary>
/// Create a new instance of RepairInfo.
/// </summary>
/// <param name="nodes">Collection of HtmlNodes with relevant information.</param>
public RepairInfo(HtmlNodeCollection nodes)
{
var repair = nodes[2].LastChild.InnerText;
var jobString = repair.Substring(0, repair.IndexOf(" "));
var lvString = repair.Substring(repair.LastIndexOf(" ") + 1);
var grade = nodes[3].LastChild.InnerText[6].ToString();
Job = JobParser.Parse(jobString);
Level = int.Parse(lvString);
MaterialGrade = int.Parse(grade);
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
var dimension = state.ParseDimension();
var keyName = state.AffixName(state.TryGet("KeyName", () => dimension.GetType().Name + "Key"));
var labelName = state.AffixName(state.TryGet("LabelName", () => dimension.GetType().Name + "Label"));
return(new LabeledFieldMapper(new XaFieldMapper(dimension, false, keyName), labelName,
XaLabelProvider.FromDimension(dimension, parser.DefaultLanguage), friendlyName: XaFieldMapper.SuggestFriendlyLabelName(labelName)));
}
IDataFilter IParseFactory <IDataFilter> .Parse(JobParser parser, ParseState state)
{
var splitter = parser.ParseSplitter(state.Select("Splitter") ?? state);
var filter = splitter as IDataFilter;
if (filter == null)
{
throw new Exception(filter.GetType().Name + " does not implement IDataFilter");
}
return(filter);
}
public IDataFilter Parse(JobParser parser, ParseState state)
{
using (new SecurityDisabler())
{
var ruleContextItem = parser.Database.GetRootItem(parser.DefaultLanguage);
var json = state.TryGet <string>("Rule");
if (!string.IsNullOrEmpty(json))
{
var filter = RulesFilter.FromString(json);
filter.RuleContextItem = ruleContextItem;
return(filter);
}
var ruleItem = state.Require <string>("Item", true);
Item item;
ID id;
if (!ID.TryParse(ruleItem, out id))
{
if (!ruleItem.StartsWith("/"))
{
var rootItem =
state.Parser.Database.GetItem(
ExperienceExtractorApiContainer.ItemPaths.GetOrDefault("experienceAnalyticsFilters") ??
"/sitecore/system/Marketing Control Panel/Experience Analytics/Filters");
ruleItem = rootItem.Paths.FullPath + "/" + ruleItem;
}
item = parser.Database.GetItem(ruleItem, parser.DefaultLanguage);
}
else
{
item = parser.Database.GetItem(id, parser.DefaultLanguage);
}
if (item == null)
{
throw ParseException.AttributeError(state, string.Format("Rule item not found '{0}'", ruleItem));
}
var rules = RuleFactory.GetRules <RuleContext>(item.Fields["Rule"]);
return(new RulesFilter(rules)
{
RuleContextItem = ruleContextItem
});
}
}
public ISplitter Parse(JobParser parser, ParseState state)
{
var testIdString = state.Require <string>("TestId", true);
Guid testId;
if (!Guid.TryParse(testIdString, out testId))
{
throw state.AttributeError("Invalid test id specified ({0})", testIdString);
}
return(new MvTestSplitter(testId));
}
protected override IFieldMapper Parse(Func<ProcessingScope, DateTime?> selector, string defaultName,
JobParser parser, ParseState state)
{
return new DateDimension(
state.AffixName(state.TryGet("Name", defaultName)),
selector,
useDateForKey: state.TryGet("DateKey", true),
inlineFields: state.TryGet("Inline", false),
cultureInfo: parser.DefaultLanguage.CultureInfo,
key: state.TryGet("Key", false),
detailLevel:
state.TryGet("Resolution", DateDetailLevel.Date, true));
}
protected override IFieldMapper Parse(Func <ProcessingScope, DateTime?> selector, string defaultName,
JobParser parser, ParseState state)
{
return(new DateDimension(
state.AffixName(state.TryGet("Name", defaultName)),
selector,
useDateForKey: state.TryGet("DateKey", true),
inlineFields: state.TryGet("Inline", false),
cultureInfo: parser.DefaultLanguage.CultureInfo,
key: state.TryGet("Key", false),
detailLevel:
state.TryGet("Resolution", DateDetailLevel.Date, true)));
}
public IFieldMapper Parse(JobParser parser, ParseState state)
{
return(new SimpleFieldMapper(state.AffixName("ContactId"), scope => scope.Current <IVisitAggregationContext>().TryGet(ctx => ctx.Visit.ContactId), typeof(Guid),
valueKind: "ContactId",
calculatedFields: new[]
{
new CalculatedField {
Name = "Unique visitors",
DaxPattern = string.Format("DISTINCTCOUNT([{0}])", state.AffixName("ContactId")),
ChildDaxPattern = string.Format("CALCULATE(DISTINCTCOUNT(@Parent[{0}]), @TableName)", state.AffixName("ContactId")),
FormatString = CalculatedFieldFormat.Integer
}
}));
}
public IDataSource Parse(JobParser parser, ParseState state)
{
var connectionString = state.TryGet("connection", ExperienceExtractorWebApiConfig.XdbConnectionString, true);
if (!connectionString.StartsWith("mongodb://"))
{
var connection = ConfigurationManager.ConnectionStrings[connectionString];
if (connection == null)
{
throw state.AttributeError("Connection string '{0}' not found", connectionString);
}
connectionString = connection.ConnectionString;
}
var source = new MongoDbVisitAggregationContextSource(new MongoDbDriver(connectionString), ExperienceExtractorWebApiConfig.JobExecutionSettings.LoadThreads);
//source.SecondaryLookup = state.TryGet("SecondaryLookup", false);
var index = state.TryGet <string>("index");
if (!string.IsNullOrEmpty(index))
{
source.CursorInitializing += (sender, cursor) =>
{
if (index == "$natural")
{
cursor.SetSortOrder(SortBy.Ascending("$natural"));
}
else
{
cursor.SetHint(index);
}
};
}
var fields = state.TryGet("fields", new string[0]);
if (fields.Length > 0)
{
source.CursorInitializing += (sender, cursor) => cursor.SetFields(fields);
}
foreach (var filter in state.SelectMany("Filters"))
{
source.Filters.Add(parser.ParseDataFilter(filter));
}
return(source);
}
public IDataFilter Parse(JobParser parser, ParseState state)
{
var start = state.TryGet <DateTime?>("Start");
var end = state.TryGet <DateTime?>("End");
if (!start.HasValue && !end.HasValue)
{
throw ParseException.AttributeError(state, "Expected start and/or end");
}
var field = state.TryGet("Field", "StartDateTime");
return(new MongoDateRangeFilter(start, end, field));
}
static void ProcessJobs(int index)
{
Console.WriteLine("\nJob {0} Output\n", index);
JobParser job1 = new JobParser($"../../JobFiles/Job{index}.txt");
job1.LoadJobElements();
JobComponents temp = job1.getJob(0);
List <JobComponents> test = job1.getJobAllElements().ToList();
JobCalculator pCalc = new JobCalculator(temp, test);
pCalc.DumpOutput(index);
}
