public void Loaded()
{
KeyAssignCore.RegisterOperation("SookIkemen", () =>
KeyAssignHelper.ExecuteTabAction(tab =>
{
try
{
tab.TabProperty.LinkAccountInfos.ForEach(a =>
PostOffice.UpdateTweet(a, "スークイケメンﾅｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰ #sook_ikemen"));
}
catch (Exception e)
{
ExceptionStorage.Register(e, ExceptionCategory.PluginError, "スークイケメンﾅｰｰｰｰｰｰｰｰｰｰｰｰｰｰに失敗しました: " + e.Message);
}
}));
KeyAssignCore.RegisterOperation("SenselessRetweet", () =>
KeyAssignHelper.ExecuteTVMAction(tvm =>
{
var ts = tvm.Tweet.Status as TwitterStatus;
if (ts == null)
{
return;
}
KernelService.MainWindowViewModel.InputBlockViewModel.SetOpenText(true, true);
KernelService.MainWindowViewModel.InputBlockViewModel.SetText(BuildSenseless(ts));
}));
KeyAssignCore.RegisterOperation("SenselessRetweetFast", () =>
KeyAssignHelper.ExecuteTVMAction(tvm =>
{
try
{
var ts = tvm.Tweet.Status as TwitterStatus;
if (ts == null)
{
return;
}
tvm.Parent.TabProperty.LinkAccountInfos.ForEach(
ai => PostOffice.UpdateTweet(ai, BuildSenseless(ts)));
}
catch (Exception e)
{
ExceptionStorage.Register(e, ExceptionCategory.PluginError, "非常識RTに失敗しました: " + e.Message);
}
}));
KernelService.AddMenu("ｽｰｸｲｹﾒﾝﾅｰ", () =>
KeyAssignHelper.ExecuteTabAction(tab =>
{
try
{
tab.TabProperty.LinkAccountInfos.ForEach(a =>
PostOffice.UpdateTweet(a, "スークイケメンﾅｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰｰ #sook_ikemen"));
}
catch (Exception e)
{
ExceptionStorage.Register(e, ExceptionCategory.PluginError, "スークイケメンﾅｰｰｰｰｰｰｰｰｰｰｰｰｰｰに失敗しました: " + e.Message);
}
}));
}
public KernelServiceWrapper(System.Windows.Forms.WebBrowser browserControl, MainForm parentForm)
{
this.parentForm = parentForm;
this.browserControl = browserControl;
this.kernel = new KernelService();
this.printService = new PrintService();
this.serviceMethods = typeof(KernelService).GetMethods(BindingFlags.Public | BindingFlags.Instance);
this.asyncResponseDelegate = new AsyncResponse(SendResponse);
this.asyncErrorResponseDelegate = new AsyncResponse(SendErrorResponse);
}
private void Restart()
{
try
{
Setting.Instance.Save();
Process.Start(Process.GetCurrentProcess().MainModule.FileName);
KernelService.AppShutdown();
}
catch { }
}
public void TestMountCode()
{
string expectedValue = "dbutils.fs.mount(source = \"wasbs://[email protected]/\", mountPoint = \"/mnt/livequery//mycontainer\", extraConfigs = Map(\"fs.azure.account.key.teststorageaccount.blob.core.windows.net\"->dbutils.secrets.get(scope = \"testkeyvault\", key = \"datax-sa-teststorageaccount\")))";
//Test unnested file path
string dbfsPath = "dbfs:/mnt/livequery/mycontainer/testFile.json";
string opsStorageAccountName = "teststorageaccount";
string sparkKeyVaultName = "testkeyvault";
string actualValue = KernelService.CreateMountCode(dbfsPath, opsStorageAccountName, sparkKeyVaultName);
Assert.AreEqual(expectedValue, actualValue, "Mount code is incorrect");
//Test nested file path
dbfsPath = "dbfs:/mnt/livequery/mycontainer/folder1/folder2/testFile.json";
actualValue = KernelService.CreateMountCode(dbfsPath, opsStorageAccountName, sparkKeyVaultName);
Assert.AreEqual(expectedValue, actualValue, "Mount code is incorrect");
}
/// <summary>
/// Provides one of the services on which the current service depends.
/// </summary>
/// <param name="dependency">The dependency service.</param>
public override void ConnectTo(KernelService dependency)
{
var core = dependency as CoreProxy;
if (core != null)
{
m_Core = core;
m_Core.OnStartupComplete += OnStartupComplete;
}
var projects = dependency as ProjectService;
if (projects != null)
{
m_Projects = projects;
}
}
public void TestConvertToJson()
{
string result = @"org.apache.spark.sql.catalyst.parser.ParseException:\r\nmismatched input '<EOF>' expecting { '(', 'SELECT'} (line 1, pos 146)";
var json = KernelService.ConvertToJson(10, result);
Assert.IsTrue(json.Error.HasValue && json.Error.Value);
Assert.IsNotNull(json.Message);
string source = @"{'prop1':6, 'prop2': 'evnetName1', 'status':0}
{'prop1':6, 'prop2': 'evnetName2', 'status':1}
Input: org.apache.spark.sql.DataFrame = [prop1: bigint, prop2: string... 1 more field]";
json = KernelService.ConvertToJson(10, source.ToString());
Assert.IsTrue(!json.Error.HasValue);
Assert.AreEqual(2, JArray.Parse(json.Result.ToString()).Count);
source = @"{'prop1':6, 'prop2': 'evnetName1', 'status':0}
{'prop1':6, 'prop2': 'evnetName2', 'status':1}";
json = KernelService.ConvertToJson(10, source.ToString());
Assert.IsTrue(!json.Error.HasValue);
Assert.AreEqual(2, JArray.Parse(json.Result.ToString()).Count);
}
public void Loaded()
{
KernelService.AddMenu("Import JSON", () =>
{
var dialog = new OpenFileDialog();
dialog.Filter = "JSON Format|*.json|すべてのファイル|*.*";
dialog.Multiselect = true;
var result = dialog.ShowDialog();
if (result.HasValue && result.Value)
{
Task.Factory.StartNew(() =>
{
dialog.FileNames.AsParallel().ForAll(file =>
{
var fileName = Path.GetFileName(file);
using (var notify = NotifyStorage.NotifyManually(fileName + " を読み込んでいます"))
{
try
{
using (var reader = JsonReaderWriterFactory.CreateJsonReader(new FileStream(file, FileMode.Open, FileAccess.Read), XmlDictionaryReaderQuotas.Max))
{
XElement.Load(reader).Elements()
.Select(TwitterStatus.FromNode)
.ForEach(s => TweetStorage.Register(s));
}
}
catch (Exception ex)
{
ExceptionStorage.Register(ex, ExceptionCategory.UserError, fileName + " を読み込めませんでした");
}
}
});
});
}
});
}
public void Loaded()
{
Task.Factory.StartNew(() =>
{
NotifyStorage.Notify("YACQ スクリプトの読み込みを開始しました");
try
{
Directory.EnumerateFiles(
Path.Combine(
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location),
"yacq_lib"
),
"*.yacq",
SearchOption.TopDirectoryOnly
)
.Where(file => !file.EndsWith("\\rc.yacq"))
.ForEach(this.RunCode);
}
catch { }
NotifyStorage.Notify("YACQ スクリプトの読み込みが完了しました");
});
KernelService.AddMenu("YACQ コンソール", () => new ReplWindow().Show());
}
public void TestCreateLoadFunctionCode()
{
string realPath = "wasbs://[email protected]/sample/udfsample.jar";
string functionId = "myFunction";
// Test CreateLoadFunctionCode for UDF on HDInsight
string sparkType = "hdinsight";
string functionType = "UDF";
string actualValue = KernelService.CreateLoadFunctionCode(
realPath,
sparkType,
functionType,
functionId,
new Common.Models.PropertiesUD {
ClassName = "SampleClass", Libs = new System.Collections.Generic.List <string>(), Path = "keyvault://testkeyvault/sample"
});
string expectedValue = "val jarPath = \"wasbs://[email protected]/sample/udfsample.jar\"\nval mainClass = \"SampleClass\"\nval jarFileUrl = datax.host.SparkJarLoader.addJarOnDriver(spark, jarPath, 0, false)\nspark.sparkContext.addJar(jarFileUrl)\ndatax.host.SparkJarLoader.registerJavaUDF(spark.udf, \"myFunction\", mainClass, null)\nprintln(\"done\")";
Assert.AreEqual(expectedValue, actualValue, "Load UDF function code for HDInsight is incorrect");
// Test CreateLoadFunctionCode for UDF on Databricks
sparkType = "databricks";
functionType = "UDF";
actualValue = KernelService.CreateLoadFunctionCode(
realPath,
sparkType,
functionType,
functionId,
new Common.Models.PropertiesUD {
ClassName = "SampleClass", Libs = new System.Collections.Generic.List <string>(), Path = "keyvault://testkeyvault/sample"
});
expectedValue = "val jarPath = \"wasbs://[email protected]/sample/udfsample.jar\"\nval mainClass = \"SampleClass\"\nval jarFileUrl = datax.host.SparkJarLoader.addJarOnDriver(spark, jarPath, 0, true)\nspark.sparkContext.addJar(jarFileUrl)\ndatax.host.SparkJarLoader.registerJavaUDF(spark.udf, \"myFunction\", mainClass, null)\nprintln(\"done\")";
Assert.AreEqual(expectedValue, actualValue, "Load UDF function code for Databricks is incorrect");
// Test CreateLoadFunctionCode for UDAF on HDInsight
sparkType = "hdinsight";
functionType = "UDAF";
actualValue = KernelService.CreateLoadFunctionCode(
realPath,
sparkType,
functionType,
functionId,
new Common.Models.PropertiesUD {
ClassName = "SampleClass", Libs = new System.Collections.Generic.List <string>(), Path = "keyvault://testkeyvault/sample"
});
expectedValue = "val jarPath = \"wasbs://[email protected]/sample/udfsample.jar\"\nval mainClass = \"SampleClass\"\nval jarFileUrl = datax.host.SparkJarLoader.addJarOnDriver(spark, jarPath, 0, false)\nspark.sparkContext.addJar(jarFileUrl)\ndatax.host.SparkJarLoader.registerJavaUDAF(spark.udf, \"myFunction\", mainClass)\nprintln(\"done\")";
Assert.AreEqual(expectedValue, actualValue, "Load UDAF function code for HDInsight is incorrect");
// Test CreateLoadFunctionCode for UDAF on Databricks
sparkType = "databricks";
functionType = "UDAF";
actualValue = KernelService.CreateLoadFunctionCode(
realPath,
sparkType,
functionType,
functionId,
new Common.Models.PropertiesUD {
ClassName = "SampleClass", Libs = new System.Collections.Generic.List <string>(), Path = "keyvault://testkeyvault/sample"
});
expectedValue = "val jarPath = \"wasbs://[email protected]/sample/udfsample.jar\"\nval mainClass = \"SampleClass\"\nval jarFileUrl = datax.host.SparkJarLoader.addJarOnDriver(spark, jarPath, 0, true)\nspark.sparkContext.addJar(jarFileUrl)\ndatax.host.SparkJarLoader.registerJavaUDAF(spark.udf, \"myFunction\", mainClass)\nprintln(\"done\")";
Assert.AreEqual(expectedValue, actualValue, "Load UDAF function code for Databricks is incorrect");
}
private void Exit()
{
KernelService.AppShutdown();
}
private void RunApplication(string appToOpen, string[] args)
{
byte[] application;
string pathToOpen = _myLib.CurrentDirectory + appToOpen + ".mye";
if (!_currentDrive.Exists(FileSystemPath.Parse(pathToOpen)))
{
if (_currentDrive.Exists(FileSystemPath.Parse("/system/" + appToOpen + ".mye")))
{
pathToOpen = "/system/" + appToOpen + ".mye";
}
else
{
Console.WriteLine("Application not found");
return;
}
}
using (Stream applicationStream = _currentDrive.OpenFile(FileSystemPath.Parse(pathToOpen), FileAccess.Read))
{
application = applicationStream.ReadAllBytes();
}
try
{
Assembly assembly = Assembly.Load(application);
IEnumerable <Type> osType = from t in assembly.GetTypes() where t.Name == "Application" select t;
if (osType.Count() > 1)
{
throw new Exception("Too many applications");
}
IProcess applicationItem = (IProcess)Activator.CreateInstance(osType.First());
switch (applicationItem.ProcessType)
{
case ProcessType.KernelApplication:
KernelProcess kernelProcess = (KernelProcess)applicationItem;
kernelProcess.Initialize(_kernel, _processManager, _myLib);
_processManager.Register(kernelProcess);
kernelProcess.Run(args);
break;
case ProcessType.UserApplication:
UserProcess userProcess = (UserProcess)applicationItem;
userProcess.Initialize(_processManager, _myLib);
_processManager.Register(userProcess);
userProcess.Run(args);
break;
case ProcessType.KernelService:
KernelService kernelService = (KernelService)applicationItem;
kernelService.Initialize(_kernel, _processManager, _myLib);
_processManager.Register(kernelService);
kernelService.Run(args);
break;
default:
Console.WriteLine("Don't know how to start {0}", applicationItem.ProcessType);
break;
}
}
catch (ReflectionTypeLoadException error)
{
Console.WriteLine("Can't load {0}:", appToOpen);
error.LoaderExceptions.ToList().ForEach(x => Console.WriteLine(" {0}", x.Message));
}
catch (Exception e)
{
Console.WriteLine("FATAL ERROR: {0}\n{1}", e.Message, e.StackTrace);
}
}
/// <summary>
/// Provides one of the services on which the current service depends.
/// </summary>
/// <param name="dependency">The dependency service.</param>
public override void ConnectTo(KernelService dependency)
{
var core = dependency as CoreProxy;
if (core != null)
{
m_Core = core;
m_Core.OnStartupComplete += OnStartupComplete;
}
var projects = dependency as ProjectService;
if (projects != null)
{
m_Projects = projects;
}
}
