public void TestSignaturesV2_3_X()
{
SparkContext sc = SparkContext.GetOrCreate(new SparkConf());
Assert.IsType <SparkConf>(sc.GetConf());
Assert.IsType <int>(sc.DefaultParallelism);
sc.SetJobDescription("job description");
sc.SetLogLevel("ALL");
sc.SetLogLevel("debug");
Assert.Throws <Exception>(() => sc.SetLogLevel("INVALID"));
sc.SetJobGroup("group id", "description");
sc.SetJobGroup("group id", "description", true);
sc.ClearJobGroup();
string filePath = $"{TestEnvironment.ResourceDirectory}people.txt";
sc.AddFile(filePath);
sc.AddFile(filePath, true);
using var tempDir = new TemporaryDirectory();
sc.SetCheckpointDir(TestEnvironment.ResourceDirectory);
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
Configuration = CommandlineArgumentProcessor.ProcessArugments(args);
PrintLogLocation();
bool status = true;
if (Configuration.IsDryrun)
{
status = SamplesRunner.RunSamples();
}
else
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
status = SamplesRunner.RunSamples();
PrintLogLocation();
ConsoleWriteLine("Completed running samples. Calling SparkContext.Stop() to tear down ...");
//following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333
ConsoleWriteLine("If this program (SparkCLRSamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!");
//TODO - add instructions to terminate java process
SparkContext.Stop();
}
if (Configuration.IsValidationEnabled && !status)
{
Environment.Exit(1);
}
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
Configuration = CommandlineArgumentProcessor.ProcessArugments(args);
PrintLogLocation();
bool status = true;
if (Configuration.IsDryrun)
{
status = SamplesRunner.RunSamples();
}
else
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
status = SamplesRunner.RunSamples();
PrintLogLocation();
ConsoleWriteLine("Completed running samples. Calling SparkContext.Stop() to tear down ...");
//following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333
ConsoleWriteLine("If this program (SparkCLRSamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!");
//TODO - add instructions to terminate java process
SparkContext.Stop();
}
if (Configuration.IsValidationEnabled && !status)
{
Environment.Exit(1);
}
}
public void TestSparkContextProxy()
{
var sparkContext = new SparkContext("masterUrl", "appName");
sparkContext.AddFile(null);
sparkContext.BinaryFiles(null, null);
sparkContext.CancelAllJobs();
sparkContext.CancelJobGroup(null);
sparkContext.EmptyRDD <string>();
sparkContext.GetLocalProperty(null);
sparkContext.HadoopFile(null, null, null, null);
sparkContext.HadoopRDD(null, null, null);
sparkContext.NewAPIHadoopFile(null, null, null, null);
sparkContext.NewAPIHadoopRDD(null, null, null);
sparkContext.Parallelize <int>(new int[] { 1, 2, 3, 4, 5 });
sparkContext.SequenceFile(null, null, null, null, null, null);
sparkContext.SetCheckpointDir(null);
sparkContext.SetJobGroup(null, null);
sparkContext.SetLocalProperty(null, null);
sparkContext.SetLogLevel(null);
sparkContext.TextFile(null);
sparkContext.WholeTextFiles(null);
sparkContext.Stop();
sparkContext.Union <string>(null);
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
ProcessArugments(args);
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
RunSamples();
SparkContext.Stop();
}
public void TestSetCheckpointDir()
{
// Arrange
const string directory = @"D:\tmp";
Mock <ISparkContextProxy> sparkContextProxy = new Mock <ISparkContextProxy>();
sparkContextProxy.Setup(m => m.SetCheckpointDir(It.IsAny <string>()));
SparkContext sc = new SparkContext(sparkContextProxy.Object, null);
// Act
sc.SetCheckpointDir(directory);
// Assert
sparkContextProxy.Verify(m => m.SetCheckpointDir(directory), Times.Once);
}
internal static void DStreamTextFileSamples()
{
SparkContext sc = SparkCLRSamples.SparkContext;
string directory = SparkCLRSamples.Configuration.SampleDataLocation;
sc.SetCheckpointDir(directory);
StreamingContext ssc = new StreamingContext(sc, 2000);
var lines = ssc.TextFileStream(Path.Combine(directory, "test"));
var words = lines.FlatMap(l => l.Split(' '));
var pairs = words.Map(w => new KeyValuePair <string, int>(w, 1));
var wordCounts = pairs.ReduceByKey((x, y) => x + y);
var join = wordCounts.Join(wordCounts, 2);
var state = join.UpdateStateByKey <string, Tuple <int, int>, int>((vs, s) => vs.Sum(x => x.Item1 + x.Item2) + s);
state.ForeachRDD((time, rdd) =>
{
// there's chance rdd.Take conflicts with ssc.Stop
if (stopFileServer)
{
return;
}
object[] taken = rdd.Take(10);
Console.WriteLine("-------------------------------------------");
Console.WriteLine("Time: {0}", time);
Console.WriteLine("-------------------------------------------");
foreach (object record in taken)
{
Console.WriteLine(record);
}
Console.WriteLine();
stopFileServer = count++ > 3;
});
ssc.Start();
StartFileServer(directory, "words.txt", 100);
ssc.AwaitTermination();
ssc.Stop();
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
ProcessArugments(args);
if (Configuration.IsDryrun)
{
RunSamples();
}
else
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
RunSamples();
SparkContext.Stop();
}
System.Console.ReadLine();
}
public void TestSignaturesV2_3_X()
{
SparkContext sc = SparkContext.GetOrCreate(new SparkConf());
_ = sc.GetConf();
_ = sc.DefaultParallelism;
sc.SetJobDescription("job description");
sc.SetJobGroup("group id", "description");
sc.SetJobGroup("group id", "description", true);
sc.ClearJobGroup();
string filePath = TestEnvironment.ResourceDirectory + "people.txt";
sc.AddFile(filePath);
sc.AddFile(filePath, true);
sc.SetCheckpointDir(TestEnvironment.ResourceDirectory);
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
ProcessArugments(args);
PrintLogLocation();
if (Configuration.IsDryrun)
{
RunSamples();
}
else
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
RunSamples();
PrintLogLocation();
ConsoleWriteLine("Main", "Completed RunSamples. Calling SparkContext.Stop() to tear down ...");
ConsoleWriteLine("Main", "If the program does not terminate in 10 seconds, please manually terminate java process !!!");
SparkContext.Stop();
}
}
static void Main(string[] args)
{
LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set
Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples));
ProcessArugments(args);
PrintLogLocation();
if (Configuration.IsDryrun)
{
RunSamples();
}
else
{
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
RunSamples();
PrintLogLocation();
ConsoleWriteLine("Main", "Completed RunSamples. Calling SparkContext.Stop() to tear down ...");
ConsoleWriteLine("Main", "If the program does not terminate in 10 seconds, please manually terminate java process !!!");
SparkContext.Stop();
}
}
private static void Main(string[] args)
{
var success = true;
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
var stopWatch = Stopwatch.StartNew();
var clockStart = stopWatch.Elapsed;
try
{
Logger.Info("----- Running Pi example -----");
Pi();
var duration = stopWatch.Elapsed - clockStart;
Logger.InfoFormat("----- Successfully finished running Pi example (duration={0}) -----", duration);
}
catch (Exception ex)
{
success = false;
var duration = stopWatch.Elapsed - clockStart;
Logger.InfoFormat("----- Error running Pi example (duration={0}) -----{1}{2}", duration, Environment.NewLine, ex);
}
Logger.Info("Completed running examples. Calling SparkContext.Stop() to tear down ...");
// following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333
Logger.Info("If this program (SparkCLRExamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!");
SparkContext.Stop();
if (!success)
{
Environment.Exit(1);
}
}
public void TestSparkContextProxy()
{
var sparkContext = new SparkContext("masterUrl", "appName");
sparkContext.AddFile(null);
sparkContext.BinaryFiles(null, null);
sparkContext.CancelAllJobs();
sparkContext.CancelJobGroup(null);
sparkContext.EmptyRDD<string>();
sparkContext.GetLocalProperty(null);
sparkContext.HadoopFile(null, null, null, null);
sparkContext.HadoopRDD(null, null, null);
sparkContext.NewAPIHadoopFile(null, null, null, null);
sparkContext.NewAPIHadoopRDD(null, null, null);
sparkContext.Parallelize<int>(new int[] { 1, 2, 3, 4, 5 });
sparkContext.SequenceFile(null, null, null, null, null, null);
sparkContext.SetCheckpointDir(null);
sparkContext.SetJobGroup(null, null);
sparkContext.SetLocalProperty(null, null);
sparkContext.SetLogLevel(null);
sparkContext.TextFile(null);
sparkContext.WholeTextFiles(null);
sparkContext.Stop();
sparkContext.Union<string>(null);
}
private static void Main(string[] args)
{
var success = true;
SparkContext = CreateSparkContext();
SparkContext.SetCheckpointDir(Path.GetTempPath());
var stopWatch = Stopwatch.StartNew();
var clockStart = stopWatch.Elapsed;
try
{
Logger.Info("----- Running Pi example -----");
Pi();
var duration = stopWatch.Elapsed - clockStart;
Logger.InfoFormat("----- Successfully finished running Pi example (duration={0}) -----", duration);
}
catch (Exception ex)
{
success = false;
var duration = stopWatch.Elapsed - clockStart;
Logger.InfoFormat("----- Error running Pi example (duration={0}) -----{1}{2}", duration, Environment.NewLine, ex);
}
Logger.Info("Completed running examples. Calling SparkContext.Stop() to tear down ...");
// following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333
Logger.Info("If this program (SparkCLRExamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!");
SparkContext.Stop();
if (!success)
{
Environment.Exit(1);
}
}
