public static void Main(string[] args)
{
// Inizialize the kernel
// prompt the shell and execute the action request
//
// trovare un meccanismo per caricare la classe in modo dinamico per il momento
// carico l'implementazione di default
// Deinisce la versione di default del kernel in base
// ai parametri
MonoBossKernel.setInstance(new DefaultMonoBossKernel());
// Fa partire ed inizializzare il logger, per questa prima versione
// usiamo solamente ed unicamente NLog, per come è stato implementato
// possiamo usare un nuovo logger
LoggerServiceFactory.setInstance(new NLoggerServiceFactory());
// Fa partire la shell
MonoBossShell shell = new MonoBossShell();
// processa la linea di comando
shell.processCommandLine(args);
// fa partire l'ambiente
// in questa fase vengono letti i parametri di instanza
// del server
shell.startEnviroment();
}
public static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(PiExample));
var sparkContext = new SparkContext(new SparkConf());
try
{
const int slices = 3;
var numberOfItems = (int)Math.Min(100000L * slices, int.MaxValue);
var values = new List <int>(numberOfItems);
for (var i = 0; i <= numberOfItems; i++)
{
values.Add(i);
}
var rdd = sparkContext.Parallelize(values, slices);
CalculatePiUsingAnonymousMethod(numberOfItems, rdd);
CalculatePiUsingSerializedClassApproach(numberOfItems, rdd);
Logger.LogInfo("Completed calculating the value of Pi");
}
catch (Exception ex)
{
Logger.LogError("Error calculating Pi");
Logger.LogException(ex);
}
sparkContext.Stop();
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
var logger = LoggerServiceFactory.GetLogger(typeof(HiveDataFrameExample));
var sparkConf = new SparkConf();
var sparkContext = new SparkContext(sparkConf);
var hiveContext = new HiveContext(sparkContext);
var peopleDataFrame = hiveContext.Read().Json(Path.Combine(Environment.CurrentDirectory, @"data\people.json"));
const string dbName = "SampleHiveDataBaseForMobius";
const string tableName = "people";
hiveContext.Sql(string.Format("CREATE DATABASE IF NOT EXISTS {0}", dbName)); // create database if not exists
hiveContext.Sql(string.Format("USE {0}", dbName));
hiveContext.Sql(string.Format("DROP TABLE {0}", tableName)); // drop table if exists
peopleDataFrame.Write().Mode(SaveMode.Overwrite).SaveAsTable(tableName); // create table
var tablesDataFrame = hiveContext.Tables(dbName); // get all tables in database
logger.LogInfo(string.Format("table count in database {0}: {1}", dbName, tablesDataFrame.Count()));
tablesDataFrame.Show();
hiveContext.Sql(string.Format("SELECT * FROM {0}", tableName)).Show(); // select from table
}
internal void Run(ISocketWrapper listener)
{
try
{
listener.Listen();
// Communicate the server port back to the Spark using standard output.
Stream outputStream = Console.OpenStandardOutput();
var serverPort = ((IPEndPoint)listener.LocalEndPoint).Port;
SerDe.Write(outputStream, serverPort);
// Now the logger can be initialized after standard output's usage is done.
s_logger = LoggerServiceFactory.GetLogger(typeof(DaemonWorker));
s_logger.LogInfo($"Started .NET DaemonServer with port {serverPort}.");
// Start accepting connections from JVM.
new Thread(() => { StartServer(listener); }).Start();
WaitForSignal();
}
catch (Exception e)
{
s_logger.LogError($".NET DaemonWorker is exiting with exception: {e}.");
Environment.Exit(-1);
}
finally
{
_waitingTaskRunners.Dispose();
}
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
Configuration = CommandlineArgumentProcessor.ProcessArugments(args);
PrintLogLocation();
bool status = true;
if (Configuration.IsDryrun)
{
status = SamplesRunner.RunSamples();
}
else
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
status = SamplesRunner.RunSamples();
PrintLogLocation();
ConsoleWriteLine("Completed running samples. Calling SparkContext.Stop() to tear down ...");
//following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333
ConsoleWriteLine("If this program (SparkCLRSamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!");
//TODO - add instructions to terminate java process
SparkContext.Stop();
}
if (Configuration.IsValidationEnabled && !status)
{
Environment.Exit(1);
}
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
var logger = LoggerServiceFactory.GetLogger(typeof(SparkXmlExample));
var inputXmlFilePath = args[0];
var outputXmlFilePath = args[1];
var sparkConf = new SparkConf();
sparkConf.SetAppName("myapp");
var sparkContext = new SparkContext(sparkConf);
var sqlContext = new SqlContext(sparkContext);
var df = sqlContext.Read()
.Format("com.databricks.spark.xml")
.Option("rowTag", "book")
.Load(inputXmlFilePath); //"D:\temp\books.xml", "file:/D:/temp/books.xml" or "hdfs://temp/books.xml"
df.ShowSchema();
var rowCount = df.Count();
logger.LogInfo("Row count is " + rowCount);
var selectedData = df.Select("author", "@id");
selectedData.Write()
.Format("com.databricks.spark.xml")
.Option("rootTag", "books")
.Option("rowTag", "book")
.Save(outputXmlFilePath); //"D:\temp\booksUpdated.xml", "file:/D:/temp/booksUpdated.xml" or "hdfs://temp/booksUpdated.xml"
sparkContext.Stop();
}
/// <summary>
/// Main method which is to be the starting point
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
logger = LoggerServiceFactory.GetLogger(typeof(HiveDataFrameExample));
try
{
string studioHome = Directory.GetParent("..\\..\\..\\..\\..\\..\\..\\..\\..\\").ToString();
string sparkHome = studioHome + @"\BigDataSDK\SDK\Spark\bin";
string appConfigFile = System.IO.Path.Combine(Directory.GetParent("..\\..\\").ToString(), "App.config");
cSharpRunner = new CSharpRunner(sparkHome, appConfigFile);
cSharpRunner.UpdateConfigFile(configFile);
// Starting CSharpRunner is essential to execute a C# Spark samples
StartCSharpRunner();
if (cSharpRunner.IsCSharpRunnerStarted)
{
logger.LogInfo("CSharpRunner Started.................");
HiveDataFrame();
DisplayEndInfo();
session.SparkContext.Stop();
cSharpRunner.process.Kill();
}
else
{
DisplayEndInfo();
}
}
catch (Exception ex)
{
logger.LogError(ex.Message);
}
}
public static int Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(WordCountExample));
if (args.Length != 1)
{
Console.Error.WriteLine("Usage: WordCount <file>");
return(1);
}
var sparkContext = new SparkContext(new SparkConf().SetAppName("MobiusWordCount"));
try
{
var lines = sparkContext.TextFile(args[0]);
var counts = lines
.FlatMap(x => x.Split(' '))
.Map(w => new KeyValuePair <string, int>(w, 1))
.ReduceByKey((x, y) => x + y);
foreach (var wordcount in counts.Collect())
{
Console.WriteLine("{0}: {1}", wordcount.Key, wordcount.Value);
}
}
catch (Exception ex)
{
Logger.LogError("Error performing Word Count");
Logger.LogException(ex);
}
sparkContext.Stop();
return(0);
}
private static void InitializeLogger()
{
if (logger == null)
{
Worker.InitializeLogger();
logger = LoggerServiceFactory.GetLogger(typeof(MultiThreadWorker));
}
}
public void LogServiceTest()
{
//LocatorCreator.Execute();
var container = new WindsorContainer();
container.Register(Component.For <ILoggerService>().ImplementedBy <DbLoggerService>().LifeStyle.BoundTo <IService>());
//container.Register(Component.For<ILogRepository>().ImplementedBy<LogRepository>().LifeStyle.Transient);
container.Register(Component.For <IConnectionProvider>()
.UsingFactoryMethod(c => new ConnectionProvider(() =>
{
var s = System.Configuration.ConfigurationManager.
ConnectionStrings["PMSDBConnection"].ConnectionString;
var res = new SqlConnection(s);
res.Open();
return(res);
})).LifestyleBoundTo <IService>());
DataAccessConfigHelper.ConfigureContainer <UserRepository>(container,
() =>
{
var session = PMSSecuritySession.GetSession(container.Resolve <IConnectionProvider>().GetConnection());
return(session);
}, "PMSSecurity");
var locator = new WindsorServiceLocator(container);
ServiceLocator.SetLocatorProvider(() => locator);
//var userRep = ServiceLocator.Current.GetInstance<UserRepository>();
var uows = new MITD.Domain.Repository.UnitOfWorkScope(
new Data.NH.NHUnitOfWorkFactory(() => PMSSecurity.Persistence.NH.PMSSecuritySession.GetSession()));
using (var uow = new NHUnitOfWork(PMSSession.GetSession()))
using (var uow2 = uows.CurrentUnitOfWork)
{
var logFactory = new LoggerServiceFactory();
var logManager = new LogManagerService(logFactory);
var logService = new LogService(logManager);
var gid = Guid.NewGuid();
Log log = new EventLog(new LogId(gid), "diddd", LogLevel.Information, null, "clll", "mett", "ttttt", "mmmmmmm");
logService.AddEventLog(log.Code, log.LogLevel, null, log.ClassName, log.MethodName, log.Title, log.Messages);
}
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
var logger = LoggerServiceFactory.GetLogger(typeof(HiveDataFrameExample));
var jsonFilePath = args[0];
const string dbName = "SampleHiveDataBaseForMobius";
const string tableName = "people";
var builder = SparkSession.Builder().EnableHiveSupport();
// The following setting is required to use Spark 2.0 in Windows
// It may be provided in command line when running Mobius app
//builder = builder.Config("spark.sql.warehouse.dir", "<hdfs or local path>");
var session = builder.GetOrCreate();
var peopleDataFrame = session.Read().Json(jsonFilePath);
session.Sql(string.Format("CREATE DATABASE IF NOT EXISTS {0}", dbName)); // create database if not exists
session.Sql(string.Format("USE {0}", dbName));
//hiveContext.Sql(string.Format("DROP TABLE {0}", tableName)); // drop table if exists
peopleDataFrame.Write().Mode(SaveMode.Overwrite).SaveAsTable(tableName); // create table
var tablesDataFrame = session.Table(tableName); // get all tables in database
logger.LogInfo(string.Format("table count in database {0}: {1}", dbName, tablesDataFrame.Count()));
tablesDataFrame.Show();
session.Sql(string.Format("SELECT * FROM {0}", tableName)).Show(); // select from table
// Following example is for the deprecated API
/*
* var sparkConf = new SparkConf();
* // The following setting is required to use Spark 2.0 in Windows
* // It may be provided in command line when running Mobius app
* //sparkConf.Set("spark.sql.warehouse.dir", @"<hdfs or local path>");
* var sparkContext = new SparkContext(sparkConf);
* var hiveContext = new HiveContext(sparkContext);
* var peopleDataFrame = hiveContext.Read().Json(jsonFilePath);
*
* hiveContext.Sql(string.Format("CREATE DATABASE IF NOT EXISTS {0}", dbName)); // create database if not exists
* hiveContext.Sql(string.Format("USE {0}", dbName));
* //hiveContext.Sql(string.Format("DROP TABLE {0}", tableName)); // drop table if exists
*
* peopleDataFrame.Write().Mode(SaveMode.Overwrite).SaveAsTable(tableName); // create table
* var tablesDataFrame = hiveContext.Tables(dbName); // get all tables in database
* logger.LogInfo(string.Format("table count in database {0}: {1}", dbName, tablesDataFrame.Count()));
* tablesDataFrame.Show();
*
* hiveContext.Sql(string.Format("SELECT * FROM {0}", tableName)).Show(); // select from table
*/
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
var logger = LoggerServiceFactory.GetLogger(typeof(JdbcDataFrameExample));
var sparkConf = new SparkConf();
var sparkContext = new SparkContext(sparkConf);
var sqlContext = new SqlContext(sparkContext);
var df = sqlContext.Read()
.Jdbc("jdbc:sqlserver://localhost:1433;databaseName=Temp;;integratedSecurity=true;", "xyz",
new Dictionary <string, string>());
df.ShowSchema();
var rowCount = df.Count();
logger.LogInfo("Row count is " + rowCount);
}
public static void InitializeLogger()
{
try
{
// if there exists exe.config file, then use log4net
if (File.Exists(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile))
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance);
}
logger = LoggerServiceFactory.GetLogger(typeof(Worker));
}
catch (Exception e)
{
Console.WriteLine("InitializeLogger exception {0}, will exit", e);
Environment.Exit(-1);
}
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
ProcessArugments(args);
if (Configuration.IsDryrun)
{
RunSamples();
}
else
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
RunSamples();
SparkContext.Stop();
}
System.Console.ReadLine();
}
/// <summary>
/// Runs the DaemonWorker server.
/// </summary>
internal void Run()
{
// TODO: Note that daemon server is stopped from Spark, it is done with
// Process.destroy(). It should send SIGTERM and the signal should be handled
// by the following:
// AppDomain.CurrentDomain.ProcessExit += (s, e) => {};,
// but the above handler is not invoked. This can be investigated if more
// graceful exit is required.
try
{
ISocketWrapper listener = SocketFactory.CreateSocket();
listener.Listen();
// Communicate the server port back to the Spark using standard output.
Stream outputStream = Console.OpenStandardOutput();
var serverPort = ((IPEndPoint)listener.LocalEndPoint).Port;
SerDe.Write(outputStream, serverPort);
// Now the logger can be initialized after standard output's usage is done.
s_logger = LoggerServiceFactory.GetLogger(typeof(DaemonWorker));
s_logger.LogInfo($"Started .NET DaemonServer with port {serverPort}.");
// Start accepting connections from JVM.
new Thread(() => { StartServer(listener); }).Start();
WaitForSignal();
}
catch (Exception e)
{
s_logger.LogError($".NET DaemonWorker is exiting with exception: {e}.");
Environment.Exit(-1);
}
finally
{
_waitingTaskRunners.Dispose();
}
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
ProcessArugments(args);
PrintLogLocation();
if (Configuration.IsDryrun)
{
RunSamples();
}
else
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
RunSamples();
PrintLogLocation();
ConsoleWriteLine("Main", "Completed RunSamples. Calling SparkContext.Stop() to tear down ...");
ConsoleWriteLine("Main", "If the program does not terminate in 10 seconds, please manually terminate java process !!!");
SparkContext.Stop();
}
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
var logger = LoggerServiceFactory.GetLogger(typeof(JdbcDataFrameExample));
//For SQL Server use the connection string formats below
//"jdbc:sqlserver://localhost:1433;databaseName=Temp;integratedSecurity=true;" or
//"jdbc:sqlserver://localhost;databaseName=Temp;user=MyUserName;password=myPassword;"
var connectionString = args[0];
var tableName = args[1];
var sparkConf = new SparkConf();
var sparkContext = new SparkContext(sparkConf);
var sqlContext = new SqlContext(sparkContext);
var df = sqlContext
.Read()
.Jdbc(connectionString, tableName, new Dictionary <string, string>());
df.ShowSchema();
var rowCount = df.Count();
logger.LogInfo("Row count is " + rowCount);
sparkContext.Stop();
}
public JvmThreadPoolGCTests(SparkFixture fixture)
{
_loggerService = LoggerServiceFactory.GetLogger(typeof(JvmThreadPoolGCTests));
_spark = fixture.Spark;
_jvmBridge = ((IJvmObjectReferenceProvider)_spark).Reference.Jvm;
}
static void Main(string[] args)
{
// if there exists exe.config file, then use log4net
if (File.Exists(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile))
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance);
}
logger = LoggerServiceFactory.GetLogger(typeof(Worker));
Socket sock = null;
try
{
PrintFiles();
int javaPort = int.Parse(Console.ReadLine());
logger.LogInfo("java_port: " + javaPort);
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sock.Connect(IPAddress.Parse("127.0.0.1"), javaPort);
}
catch (Exception e)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(e);
Environment.Exit(-1);
}
using (NetworkStream s = new NetworkStream(sock))
{
try
{
DateTime bootTime = DateTime.UtcNow;
int splitIndex = SerDe.ReadInt(s);
logger.LogInfo("split_index: " + splitIndex);
if (splitIndex == -1)
{
Environment.Exit(-1);
}
string ver = SerDe.ReadString(s);
logger.LogInfo("ver: " + ver);
//// initialize global state
//shuffle.MemoryBytesSpilled = 0
//shuffle.DiskBytesSpilled = 0
// fetch name of workdir
string sparkFilesDir = SerDe.ReadString(s);
logger.LogInfo("spark_files_dir: " + sparkFilesDir);
//SparkFiles._root_directory = sparkFilesDir
//SparkFiles._is_running_on_worker = True
// fetch names of includes - not used //TODO - complete the impl
int numberOfIncludesItems = SerDe.ReadInt(s);
logger.LogInfo("num_includes: " + numberOfIncludesItems);
if (numberOfIncludesItems > 0)
{
for (int i = 0; i < numberOfIncludesItems; i++)
{
string filename = SerDe.ReadString(s);
}
}
// fetch names and values of broadcast variables
int numBroadcastVariables = SerDe.ReadInt(s);
logger.LogInfo("num_broadcast_variables: " + numBroadcastVariables);
if (numBroadcastVariables > 0)
{
for (int i = 0; i < numBroadcastVariables; i++)
{
long bid = SerDe.ReadLong(s);
if (bid >= 0)
{
string path = SerDe.ReadString(s);
Broadcast.broadcastRegistry[bid] = new Broadcast(path);
}
else
{
bid = -bid - 1;
Broadcast.broadcastRegistry.Remove(bid);
}
}
}
Accumulator.accumulatorRegistry.Clear();
int lengthOCommandByteArray = SerDe.ReadInt(s);
logger.LogInfo("command_len: " + lengthOCommandByteArray);
IFormatter formatter = new BinaryFormatter();
if (lengthOCommandByteArray > 0)
{
string deserializerMode = SerDe.ReadString(s);
logger.LogInfo("Deserializer mode: " + deserializerMode);
string serializerMode = SerDe.ReadString(s);
logger.LogInfo("Serializer mode: " + serializerMode);
byte[] command = SerDe.ReadBytes(s);
logger.LogInfo("command bytes read: " + command.Length);
var stream = new MemoryStream(command);
var func = (Func <int, IEnumerable <dynamic>, IEnumerable <dynamic> >)formatter.Deserialize(stream);
DateTime initTime = DateTime.UtcNow;
int count = 0;
foreach (var message in func(splitIndex, GetIterator(s, deserializerMode)))
{
byte[] buffer;
if (serializerMode == "None")
{
buffer = message as byte[];
}
else if (serializerMode == "String")
{
buffer = SerDe.ToBytes(message as string);
}
else if (serializerMode == "Row")
{
Pickler pickler = new Pickler();
buffer = pickler.dumps(new ArrayList {
message
});
}
else
{
try
{
var ms = new MemoryStream();
formatter.Serialize(ms, message);
buffer = ms.ToArray();
}
catch (Exception)
{
logger.LogError(string.Format("{0} : {1}", message.GetType().Name, message.GetType().FullName));
throw;
}
}
count++;
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
//TODO - complete the impl
logger.LogInfo("Count: " + count);
//if profiler:
// profiler.profile(process)
//else:
// process()
DateTime finish_time = DateTime.UtcNow;
const string format = "MM/dd/yyyy hh:mm:ss.fff tt";
logger.LogInfo(string.Format("bootTime: {0}, initTime: {1}, finish_time: {2}",
bootTime.ToString(format), initTime.ToString(format), finish_time.ToString(format)));
SerDe.Write(s, (int)SpecialLengths.TIMING_DATA);
SerDe.Write(s, ToUnixTime(bootTime));
SerDe.Write(s, ToUnixTime(initTime));
SerDe.Write(s, ToUnixTime(finish_time));
SerDe.Write(s, 0L); //shuffle.MemoryBytesSpilled
SerDe.Write(s, 0L); //shuffle.DiskBytesSpilled
}
else
{
logger.LogWarn("Nothing to execute :-(");
}
// Mark the beginning of the accumulators section of the output
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, Accumulator.accumulatorRegistry.Count);
foreach (var item in Accumulator.accumulatorRegistry)
{
var ms = new MemoryStream();
var value = item.Value.GetType().GetField("value", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(item.Value);
logger.LogInfo(string.Format("({0}, {1})", item.Key, value));
formatter.Serialize(ms, new KeyValuePair <int, dynamic>(item.Key, value));
byte[] buffer = ms.ToArray();
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
int end = SerDe.ReadInt(s);
// check end of stream
if (end == (int)SpecialLengths.END_OF_DATA_SECTION || end == (int)SpecialLengths.END_OF_STREAM)
{
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
logger.LogInfo("END_OF_STREAM: " + (int)SpecialLengths.END_OF_STREAM);
}
else
{
// write a different value to tell JVM to not reuse this worker
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
Environment.Exit(-1);
}
s.Flush();
// wait for server to complete, otherwise server gets 'connection reset' exception
// TODO: need to detect java side has closed socket properly
System.Threading.Thread.Sleep(1000);
}
catch (Exception e)
{
logger.LogError(e.ToString());
try
{
SerDe.Write(s, e.ToString());
}
catch (IOException)
{
// JVM close the socket
}
catch (Exception ex)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(ex);
}
Environment.Exit(-1);
}
}
sock.Close();
}
public static int Run(Socket sock = null)
{
// if there exists exe.config file, then use log4net
if (File.Exists(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile))
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance);
}
logger = LoggerServiceFactory.GetLogger(typeof(Worker));
if (sock == null)
{
try
{
PrintFiles();
int javaPort = int.Parse(Console.ReadLine());
logger.LogDebug("java_port: " + javaPort);
sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sock.Connect(IPAddress.Loopback, javaPort);
}
catch (Exception e)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(e);
Environment.Exit(-1);
}
}
using (NetworkStream s = new NetworkStream(sock))
{
try
{
DateTime bootTime = DateTime.UtcNow;
int splitIndex = SerDe.ReadInt(s);
logger.LogDebug("split_index: " + splitIndex);
if (splitIndex == -1)
{
Environment.Exit(-1);
}
string ver = SerDe.ReadString(s);
logger.LogDebug("ver: " + ver);
//// initialize global state
//shuffle.MemoryBytesSpilled = 0
//shuffle.DiskBytesSpilled = 0
// fetch name of workdir
string sparkFilesDir = SerDe.ReadString(s);
logger.LogDebug("spark_files_dir: " + sparkFilesDir);
//SparkFiles._root_directory = sparkFilesDir
//SparkFiles._is_running_on_worker = True
// fetch names of includes - not used //TODO - complete the impl
int numberOfIncludesItems = SerDe.ReadInt(s);
logger.LogDebug("num_includes: " + numberOfIncludesItems);
if (numberOfIncludesItems > 0)
{
for (int i = 0; i < numberOfIncludesItems; i++)
{
string filename = SerDe.ReadString(s);
}
}
// fetch names and values of broadcast variables
int numBroadcastVariables = SerDe.ReadInt(s);
logger.LogDebug("num_broadcast_variables: " + numBroadcastVariables);
if (numBroadcastVariables > 0)
{
for (int i = 0; i < numBroadcastVariables; i++)
{
long bid = SerDe.ReadLong(s);
if (bid >= 0)
{
string path = SerDe.ReadString(s);
Broadcast.broadcastRegistry[bid] = new Broadcast(path);
}
else
{
bid = -bid - 1;
Broadcast.broadcastRegistry.Remove(bid);
}
}
}
Accumulator.accumulatorRegistry.Clear();
int lengthOfCommandByteArray = SerDe.ReadInt(s);
logger.LogDebug("command length: " + lengthOfCommandByteArray);
IFormatter formatter = new BinaryFormatter();
if (lengthOfCommandByteArray > 0)
{
Stopwatch commandProcessWatch = new Stopwatch();
Stopwatch funcProcessWatch = new Stopwatch();
commandProcessWatch.Start();
int rddId = SerDe.ReadInt(s);
int stageId = SerDe.ReadInt(s);
int partitionId = SerDe.ReadInt(s);
logger.LogInfo(string.Format("rddInfo: rddId {0}, stageId {1}, partitionId {2}", rddId, stageId, partitionId));
string deserializerMode = SerDe.ReadString(s);
logger.LogDebug("Deserializer mode: " + deserializerMode);
string serializerMode = SerDe.ReadString(s);
logger.LogDebug("Serializer mode: " + serializerMode);
byte[] command = SerDe.ReadBytes(s);
logger.LogDebug("command bytes read: " + command.Length);
var stream = new MemoryStream(command);
var workerFunc = (CSharpWorkerFunc)formatter.Deserialize(stream);
var func = workerFunc.Func;
//logger.LogDebug("------------------------ Printing stack trace of workerFunc for ** debugging ** ------------------------------");
//logger.LogDebug(workerFunc.StackTrace);
//logger.LogDebug("--------------------------------------------------------------------------------------------------------------");
DateTime initTime = DateTime.UtcNow;
int count = 0;
// here we use low level API because we need to get perf metrics
WorkerInputEnumerator inputEnumerator = new WorkerInputEnumerator(s, deserializerMode);
IEnumerable <dynamic> inputEnumerable = Enumerable.Cast <dynamic>(inputEnumerator);
funcProcessWatch.Start();
IEnumerable <dynamic> outputEnumerable = func(splitIndex, inputEnumerable);
var outputEnumerator = outputEnumerable.GetEnumerator();
funcProcessWatch.Stop();
while (true)
{
funcProcessWatch.Start();
bool hasNext = outputEnumerator.MoveNext();
funcProcessWatch.Stop();
if (!hasNext)
{
break;
}
funcProcessWatch.Start();
var message = outputEnumerator.Current;
funcProcessWatch.Stop();
if (object.ReferenceEquals(null, message))
{
continue;
}
byte[] buffer;
switch ((SerializedMode)Enum.Parse(typeof(SerializedMode), serializerMode))
{
case SerializedMode.None:
buffer = message as byte[];
break;
case SerializedMode.String:
buffer = SerDe.ToBytes(message as string);
break;
case SerializedMode.Row:
Pickler pickler = new Pickler();
buffer = pickler.dumps(new ArrayList {
message
});
break;
default:
try
{
var ms = new MemoryStream();
formatter.Serialize(ms, message);
buffer = ms.ToArray();
}
catch (Exception)
{
logger.LogError("Exception serializing output");
logger.LogError("{0} : {1}", message.GetType().Name, message.GetType().FullName);
throw;
}
break;
}
count++;
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
//TODO - complete the impl
logger.LogDebug("Output entries count: " + count);
//if profiler:
// profiler.profile(process)
//else:
// process()
DateTime finish_time = DateTime.UtcNow;
const string format = "MM/dd/yyyy hh:mm:ss.fff tt";
logger.LogDebug(string.Format("bootTime: {0}, initTime: {1}, finish_time: {2}",
bootTime.ToString(format), initTime.ToString(format), finish_time.ToString(format)));
SerDe.Write(s, (int)SpecialLengths.TIMING_DATA);
SerDe.Write(s, ToUnixTime(bootTime));
SerDe.Write(s, ToUnixTime(initTime));
SerDe.Write(s, ToUnixTime(finish_time));
SerDe.Write(s, 0L); //shuffle.MemoryBytesSpilled
SerDe.Write(s, 0L); //shuffle.DiskBytesSpilled
commandProcessWatch.Stop();
// log statistics
inputEnumerator.LogStatistic();
logger.LogInfo(string.Format("func process time: {0}", funcProcessWatch.ElapsedMilliseconds));
logger.LogInfo(string.Format("command process time: {0}", commandProcessWatch.ElapsedMilliseconds));
}
else
{
logger.LogWarn("lengthOfCommandByteArray = 0. Nothing to execute :-(");
}
// Mark the beginning of the accumulators section of the output
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
SerDe.Write(s, Accumulator.accumulatorRegistry.Count);
foreach (var item in Accumulator.accumulatorRegistry)
{
var ms = new MemoryStream();
var value = item.Value.GetType().GetField("value", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(item.Value);
logger.LogDebug(string.Format("({0}, {1})", item.Key, value));
formatter.Serialize(ms, new KeyValuePair <int, dynamic>(item.Key, value));
byte[] buffer = ms.ToArray();
SerDe.Write(s, buffer.Length);
SerDe.Write(s, buffer);
}
int end = SerDe.ReadInt(s);
// check end of stream
if (end == (int)SpecialLengths.END_OF_STREAM)
{
SerDe.Write(s, (int)SpecialLengths.END_OF_STREAM);
logger.LogDebug("END_OF_STREAM: " + (int)SpecialLengths.END_OF_STREAM);
}
else
{
// write a different value to tell JVM to not reuse this worker
SerDe.Write(s, (int)SpecialLengths.END_OF_DATA_SECTION);
Environment.Exit(-1);
}
s.Flush();
// log bytes read and write
logger.LogDebug(string.Format("total read bytes: {0}", SerDe.totalReadNum));
logger.LogDebug(string.Format("total write bytes: {0}", SerDe.totalWriteNum));
}
catch (Exception e)
{
logger.LogError(e.ToString());
try
{
SerDe.Write(s, e.ToString());
}
catch (IOException)
{
// JVM close the socket
}
catch (Exception ex)
{
logger.LogError("CSharpWorker failed with exception:");
logger.LogException(ex);
}
Environment.Exit(-1);
return(-1);
}
return(0);
}
}
public JvmThreadPoolGCTests(SparkFixture fixture)
{
_loggerService = LoggerServiceFactory.GetLogger(typeof(JvmThreadPoolGCTests));
_spark = fixture.Spark;
_jvmBridge = _spark.Reference.Jvm;
}
