public void Setup()
{
var accessToken = ConfigurationManager.AppSettings["accessToken"];
var deviceId = ConfigurationManager.AppSettings["deviceId"];
client = new SparkClient(accessToken, deviceId);
}
public async Task <IHttpActionResult> Post(string userToken, string botName, [FromBody] WebhookMessage message)
{
Trace.TraceInformation($"Get data from {message.id}");
if (message.data.personEmail.Contains("sparkbot.io"))
{
return(Ok());
}
ISparkClient sparkClient = new SparkClient(await _channelsService.GetBotToken(userToken, botName));
var text = await sparkClient.GetMessage(message.data.id);
var nameParts = message.name.Split(' ');
var coreAnswer = await Talk(userToken, botName, new InputMessage()
{
InterlocutorId = $"{message.data.roomId}-{message.data.personId}",
FirstName = nameParts.FirstOrDefault(),
LastName = nameParts.LastOrDefault(),
Text = text.text,
Username = text.personEmail
});
foreach (var item in coreAnswer.Messages)
{
await sparkClient.SendMessage(text.roomId, item.Text);
}
return(Ok());
}
/// <summary>
/// Gets the metric values.
/// </summary>
/// <param name="dataLakeClient">The data lake client.</param>
/// <param name="testContent">Content of the test.</param>
/// <param name="sparkClient">The spark client.</param>
/// <param name="sparkClientSettings">The spark client settings.</param>
/// <returns>MetricValues.</returns>
private static MetricValues GetMetricValues(
DataLakeClient dataLakeClient,
SparkMetricsTestContent testContent,
SparkClient sparkClient,
SparkClientSettings sparkClientSettings,
CancellationToken cancellationToken)
{
string currentPath = testContent.GetCurrentStreamPath();
string previousPath = testContent.GetPreviousStreamPath();
var current = GetMetricValue(
testContent,
sparkClient,
sparkClientSettings,
currentPath,
dataLakeClient,
cancellationToken);
var previous = GetMetricValue(
testContent,
sparkClient,
sparkClientSettings,
previousPath,
dataLakeClient,
cancellationToken);
var metricValues = new MetricValues
{
Baseline = previous,
Current = current
};
return(metricValues);
}
//[ExpectedException(typeof(SparkApiException), "Permission Denied")]
public void GivenInvalidDeviceIdExpectErrorPermissionDenied()
{
var badClient = new SparkClient(client.AccessToken, "badid");
try {
SparkFunctionResult result = badClient.ExecuteFunction("doesn't matter");
}
catch (SparkApiException spex)
{
Assert.AreEqual("Permission Denied", spex.Message);
}
}
public void GivenInvalidAccessTokenExpectErrorInvalidGrant()
{
var badClient = new SparkClient("badtoken", "");
try
{
SparkFunctionResult result = badClient.ExecuteFunction("doesn't matter");
}
catch (SparkApiException spex)
{
Assert.AreEqual("invalid_grant", spex.Message);
}
}
public async Task <IHttpActionResult> Get(string userToken, string botName, string roomId)
{
var sparkClient = new SparkClient(
await _channelsService.GetBotToken(userToken, botName));
var users = await sparkClient.GetUsersInRoom(roomId);
return(Ok(users.Where(i => !i.personDisplayName.Contains("(bot)")).Select(i => new
{
userId = i.personId,
userName = i.personDisplayName
})));
}
//[ExpectedException(typeof(SparkApiException), "invalid_grant")]
public void GivenInvalidAccessTokenExpectErrorInvalidGrant()
{
var badClient = new SparkClient("badtoken", "");
try
{
SparkFunctionResult result = badClient.ExecuteFunction("doesn't matter");
}
catch (SparkApiException spex)
{
Assert.AreEqual("invalid_grant", spex.Message);
}
}
public override void Setup()
{
if (!IsConfigValid())
{
throw new ArgumentNullException("", "Please enter your Spark Core Device Id and Access Token in Settings dialog.");
}
else
{
sparkClient = new SparkClient(Properties.Settings.Default.AccessToken, Properties.Settings.Default.DeviceId);
eventSource = sparkClient.GetEventStream();
eventSource.EventReceived += eventSource_EventReceived;
eventSource.Start(new CancellationToken());
}
}
public override void Setup()
{
if (!IsConfigValid())
{
throw new ArgumentNullException("", "Please enter your Spark Core Device Id and Access Token in Settings dialog.");
}
else
{
sparkClient = new SparkClient(Properties.Settings.Default.AccessToken, Properties.Settings.Default.DeviceId);
eventSource = sparkClient.GetEventStream();
eventSource.EventReceived += eventSource_EventReceived;
eventSource.Start(new CancellationToken());
}
}
public async Task <IHttpActionResult> Post(string userToken, string botName, string userId, [FromBody] TalkResponce message)
{
if (!userId.Contains("-"))
{
return(BadRequest("Invalid user id for Spark. Expected {roomId}-{peopleId}"));
}
var botToken = await _channelsService.GetBotToken(userToken, botName);
var client = new SparkClient(botToken);
var ids = userId.Split('-');
foreach (var item in message.Messages)
{
await client.SendMessage(ids[0], item.Text);
}
return(Ok());
}
public MainWindowModel(SparkClient spark, string nostalePath)
{
_spark = spark;
_nostalePath = nostalePath;
Accounts = new ObservableCollection <InGameAccount>();
StartGameCommand = new NTCommand {
Callback = OnStartButtonPressed
};
OptionsCommand = new NTCommand {
Callback = OnOptionsButtonPressed
};
QuitCommand = new NTCommand {
Callback = OnQuitButtonPressed
};
ClearAccountsCommand = new NTCommand {
Callback = () => Accounts.Clear()
};
StayOnTopCommand = new NTCommand <MainWindow> {
Callback = (w) => w.Topmost = !w.Topmost
};
ConnectAccountsCommand = new NTCommand <PasswordBox> {
Callback = OnConnectButtonPressed
};
DeleteNTCommand = new NTCommand {
Callback = OnDeletePressedListBoxItem
};
QuitNTCommand = new NTCommand {
Callback = OnQPressedListBoxItem
};
FrontNTCommand = new NTCommand {
Callback = OnEnterPressedListBoxItem
};
}
/// <summary>
/// Processes the metrics correctness test.
/// </summary>
/// <param name="dataLakeClient">The data lake client.</param>
/// <param name="testContent">Content of the test.</param>
/// <param name="sparkClient">The spark client.</param>
/// <param name="sparkClientSettings">The spark client settings.</param>
/// <returns>SparkMetricsTestResult.</returns>
/// <exception cref="NotSupportedException">ComparisonType {testContent.ComparisonType}</exception>
private static List <MetricsTestResult> ProcessMetricsCorrectnessTest(
DataLakeClient dataLakeClient,
SparkMetricsTestContent testContent,
SparkClient sparkClient,
SparkClientSettings sparkClientSettings,
CancellationToken cancellationToken)
{
MetricValues metricValues = null;
switch (testContent.ComparisonType)
{
case ComparisonType.DayOverDay:
case ComparisonType.WeekOverWeek:
case ComparisonType.MonthOverMonth:
case ComparisonType.YearOverYear:
metricValues = GetMetricValues(
dataLakeClient,
testContent,
sparkClient,
sparkClientSettings,
cancellationToken);
break;
case ComparisonType.VarianceToTarget:
var metricValue = GetMetricValue(
testContent,
sparkClient,
sparkClientSettings,
testContent.GetCurrentStreamPath(),
dataLakeClient,
cancellationToken);
testContent.NotebookParameters["cmdText"] = testContent.NotebookParameters["targetCmdText"];
var targetValue = GetMetricValue(
testContent,
sparkClient,
sparkClientSettings,
testContent.TargetStreamPath != null ? testContent.GetCurrentTargetStreamPath() : testContent.GetCurrentStreamPath(),
dataLakeClient,
cancellationToken);
metricValues = new MetricValues
{
Baseline = targetValue,
Current = metricValue
};
break;
default:
throw new NotSupportedException($"ComparisonType {testContent.ComparisonType} not supported for SparkWorker");
}
var results = new List <MetricsTestResult>();
foreach (var threshold in testContent.Thresholds)
{
var result = new MetricsTestResult
{
ComparisonType = testContent.ComparisonType,
Date = testContent.Date,
BaselineMetricValue = metricValues.Baseline[threshold.Name],
MetricValue = metricValues.Current[threshold.Name],
LowerBoundThreshold = threshold.LowerBound,
UpperBoundThreshold = threshold.UpperBound,
PercentDiff = MetricValues.ComputePercentDiff(metricValues, threshold.Name),
PreviousDate = testContent.GetPreviousDate(),
TestName = testRunName,
TestRunId = testRunId,
MetricName = threshold.Name
};
results.Add(result);
}
return(results);
}
/// <summary>
/// Gets the metric value.
/// </summary>
/// <param name="testContent">Content of the test.</param>
/// <param name="sparkClient">The spark client.</param>
/// <param name="sparkClientSettings">The spark client settings.</param>
/// <param name="stream">The stream.</param>
/// <param name="dataLakeClient">The data lake client.</param>
/// <returns>System.Nullable<System.Double>.</returns>
/// <exception cref="InvalidOperationException">Stream does not exist : {stream}</exception>
private static IDictionary <string, double> GetMetricValue(
SparkMetricsTestContent testContent,
SparkClient sparkClient,
SparkClientSettings sparkClientSettings,
string stream,
DataLakeClient dataLakeClient,
CancellationToken cancellationToken)
{
if (!dataLakeClient.CheckExists(testContent.GetDatalakeStore(), stream))
{
throw new FileNotFoundException($"Stream does not exist : {stream}");
}
var sparkRequest = new SparkClientRequest
{
NodeType = sparkClientSettings.NodeType,
NumWorkersMin = sparkClientSettings.NumWorkersMin,
NumWorkersMax = sparkClientSettings.NumWorkersMax,
CostPerNode = GetCostPerNode(sparkClientSettings.NodeTypes, sparkClientSettings.NodeType),
Libraries = sparkClientSettings.Libraries,
NotebookPath = testContent.NotebookPath,
NotebookParameters = testContent.NotebookParameters ?? new Dictionary <string, string>(),
TestRunId = testRunId,
TimeoutSeconds = sparkClientSettings.TimeoutSeconds
};
Console.WriteLine($"Running notebook={testContent.NotebookPath} for DataLakeStore={testContent.GetDatalakeStore()}, Path={stream}");
var mountPoint = GetMountPoint(testContent.GetDatalakeStore(), stream);
Console.WriteLine($"Running notebook={testContent.NotebookPath} for mountPoint={mountPoint}");
string streamPath = mountPoint;
// Disable this function
//if (testContent.ConvertToParquet)
//{
// var parquetFile = messageTag.MountPointToParquetFile[mountPoint];
// Console.WriteLine($"Running notebook={testContent.NotebookPath} using parquetFile={parquetFile}");
// streamPath = parquetFile;
//}
//else
//{
// streamPath = mountPoint;
//}
sparkRequest.NotebookParameters["streamPath"] = streamPath;
Console.WriteLine($"Notebook parameters : {string.Join(", ", sparkRequest.NotebookParameters.Select(t => t.Key + "=" + t.Value))}");
// Log request to OMS
var response = sparkClient.RunNotebook(sparkRequest, cancellationToken);
response.TestRunId = testRunId;
if (response.IsRunSuccess())
{
// For format reference see:
// https://pandas.pydata.org/pandas-docs/version/0.24.2/reference/api/pandas.DataFrame.to_json.html
var resultDataFrame = JsonConvert.DeserializeObject <Dictionary <string, Dictionary <string, double> > >(response.RunOutput);
var resultDictionary = new Dictionary <string, double>();
foreach (var pair in resultDataFrame)
{
// pair.Value is the column name
if (pair.Value == null || pair.Value.Count == 0)
{
throw new InvalidOperationException("Result does not contain any rows");
}
// We take the first row only
resultDictionary.Add(pair.Key, pair.Value.First().Value);
}
return(resultDictionary);
}
else
{
Console.WriteLine("Error getting metric.");
Console.WriteLine(JObject.Parse(JsonConvert.SerializeObject(response)));
throw new Exception($"Error getting metric. TestRun = {testRunId}, Spark job {response?.Run?.RunId} failed");
}
}
public void GivenInvalidDeviceIdExpectErrorPermissionDenied()
{
var badClient = new SparkClient(client.AccessToken, "badid");
try {
SparkFunctionResult result = badClient.ExecuteFunction("doesn't matter");
}
catch (SparkApiException spex)
{
Assert.AreEqual("Permission Denied", spex.Message);
}
}
public void Setup()
{
var accessToken = ConfigurationManager.AppSettings["accessToken"];
var deviceId = ConfigurationManager.AppSettings["deviceId"];
client = new SparkClient(accessToken, deviceId);
}
