public void TestSignaturesV2_3_X() { SparkContext sc = SparkContext.GetOrCreate(new SparkConf()); Assert.IsType <SparkConf>(sc.GetConf()); Assert.IsType <int>(sc.DefaultParallelism); sc.SetJobDescription("job description"); sc.SetLogLevel("ALL"); sc.SetLogLevel("debug"); Assert.Throws <Exception>(() => sc.SetLogLevel("INVALID")); sc.SetJobGroup("group id", "description"); sc.SetJobGroup("group id", "description", true); sc.ClearJobGroup(); string filePath = $"{TestEnvironment.ResourceDirectory}people.txt"; sc.AddFile(filePath); sc.AddFile(filePath, true); using var tempDir = new TemporaryDirectory(); sc.SetCheckpointDir(TestEnvironment.ResourceDirectory); }
static void Main(string[] args) { LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples)); Configuration = CommandlineArgumentProcessor.ProcessArugments(args); PrintLogLocation(); bool status = true; if (Configuration.IsDryrun) { status = SamplesRunner.RunSamples(); } else { SparkContext = CreateSparkContext(); SparkContext.SetCheckpointDir(Path.GetTempPath()); status = SamplesRunner.RunSamples(); PrintLogLocation(); ConsoleWriteLine("Completed running samples. Calling SparkContext.Stop() to tear down ..."); //following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333 ConsoleWriteLine("If this program (SparkCLRSamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!"); //TODO - add instructions to terminate java process SparkContext.Stop(); } if (Configuration.IsValidationEnabled && !status) { Environment.Exit(1); } }
public void TestSparkContextProxy() { var sparkContext = new SparkContext("masterUrl", "appName"); sparkContext.AddFile(null); sparkContext.BinaryFiles(null, null); sparkContext.CancelAllJobs(); sparkContext.CancelJobGroup(null); sparkContext.EmptyRDD <string>(); sparkContext.GetLocalProperty(null); sparkContext.HadoopFile(null, null, null, null); sparkContext.HadoopRDD(null, null, null); sparkContext.NewAPIHadoopFile(null, null, null, null); sparkContext.NewAPIHadoopRDD(null, null, null); sparkContext.Parallelize <int>(new int[] { 1, 2, 3, 4, 5 }); sparkContext.SequenceFile(null, null, null, null, null, null); sparkContext.SetCheckpointDir(null); sparkContext.SetJobGroup(null, null); sparkContext.SetLocalProperty(null, null); sparkContext.SetLogLevel(null); sparkContext.TextFile(null); sparkContext.WholeTextFiles(null); sparkContext.Stop(); sparkContext.Union <string>(null); }
static void Main(string[] args) { LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples)); ProcessArugments(args); SparkContext = CreateSparkContext(); SparkContext.SetCheckpointDir(Path.GetTempPath()); RunSamples(); SparkContext.Stop(); }
public void TestSetCheckpointDir() { // Arrange const string directory = @"D:\tmp"; Mock <ISparkContextProxy> sparkContextProxy = new Mock <ISparkContextProxy>(); sparkContextProxy.Setup(m => m.SetCheckpointDir(It.IsAny <string>())); SparkContext sc = new SparkContext(sparkContextProxy.Object, null); // Act sc.SetCheckpointDir(directory); // Assert sparkContextProxy.Verify(m => m.SetCheckpointDir(directory), Times.Once); }
internal static void DStreamTextFileSamples() { SparkContext sc = SparkCLRSamples.SparkContext; string directory = SparkCLRSamples.Configuration.SampleDataLocation; sc.SetCheckpointDir(directory); StreamingContext ssc = new StreamingContext(sc, 2000); var lines = ssc.TextFileStream(Path.Combine(directory, "test")); var words = lines.FlatMap(l => l.Split(' ')); var pairs = words.Map(w => new KeyValuePair <string, int>(w, 1)); var wordCounts = pairs.ReduceByKey((x, y) => x + y); var join = wordCounts.Join(wordCounts, 2); var state = join.UpdateStateByKey <string, Tuple <int, int>, int>((vs, s) => vs.Sum(x => x.Item1 + x.Item2) + s); state.ForeachRDD((time, rdd) => { // there's chance rdd.Take conflicts with ssc.Stop if (stopFileServer) { return; } object[] taken = rdd.Take(10); Console.WriteLine("-------------------------------------------"); Console.WriteLine("Time: {0}", time); Console.WriteLine("-------------------------------------------"); foreach (object record in taken) { Console.WriteLine(record); } Console.WriteLine(); stopFileServer = count++ > 3; }); ssc.Start(); StartFileServer(directory, "words.txt", 100); ssc.AwaitTermination(); ssc.Stop(); }
static void Main(string[] args) { LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples)); ProcessArugments(args); if (Configuration.IsDryrun) { RunSamples(); } else { SparkContext = CreateSparkContext(); SparkContext.SetCheckpointDir(Path.GetTempPath()); RunSamples(); SparkContext.Stop(); } System.Console.ReadLine(); }
public void TestSignaturesV2_3_X() { SparkContext sc = SparkContext.GetOrCreate(new SparkConf()); _ = sc.GetConf(); _ = sc.DefaultParallelism; sc.SetJobDescription("job description"); sc.SetJobGroup("group id", "description"); sc.SetJobGroup("group id", "description", true); sc.ClearJobGroup(); string filePath = TestEnvironment.ResourceDirectory + "people.txt"; sc.AddFile(filePath); sc.AddFile(filePath, true); sc.SetCheckpointDir(TestEnvironment.ResourceDirectory); }
static void Main(string[] args) { LoggerServiceFactory.SetLoggerService(Log4NetLoggerService.Instance); //this is optional - DefaultLoggerService will be used if not set Logger = LoggerServiceFactory.GetLogger(typeof(SparkCLRSamples)); ProcessArugments(args); PrintLogLocation(); if (Configuration.IsDryrun) { RunSamples(); } else { SparkContext = CreateSparkContext(); SparkContext.SetCheckpointDir(Path.GetTempPath()); RunSamples(); PrintLogLocation(); ConsoleWriteLine("Main", "Completed RunSamples. Calling SparkContext.Stop() to tear down ..."); ConsoleWriteLine("Main", "If the program does not terminate in 10 seconds, please manually terminate java process !!!"); SparkContext.Stop(); } }
private static void Main(string[] args) { var success = true; SparkContext = CreateSparkContext(); SparkContext.SetCheckpointDir(Path.GetTempPath()); var stopWatch = Stopwatch.StartNew(); var clockStart = stopWatch.Elapsed; try { Logger.Info("----- Running Pi example -----"); Pi(); var duration = stopWatch.Elapsed - clockStart; Logger.InfoFormat("----- Successfully finished running Pi example (duration={0}) -----", duration); } catch (Exception ex) { success = false; var duration = stopWatch.Elapsed - clockStart; Logger.InfoFormat("----- Error running Pi example (duration={0}) -----{1}{2}", duration, Environment.NewLine, ex); } Logger.Info("Completed running examples. Calling SparkContext.Stop() to tear down ..."); // following comment is necessary due to known issue in Spark. See https://issues.apache.org/jira/browse/SPARK-8333 Logger.Info("If this program (SparkCLRExamples.exe) does not terminate in 10 seconds, please manually terminate java process launched by this program!!!"); SparkContext.Stop(); if (!success) { Environment.Exit(1); } }
public void TestSparkContextProxy() { var sparkContext = new SparkContext("masterUrl", "appName"); sparkContext.AddFile(null); sparkContext.BinaryFiles(null, null); sparkContext.CancelAllJobs(); sparkContext.CancelJobGroup(null); sparkContext.EmptyRDD<string>(); sparkContext.GetLocalProperty(null); sparkContext.HadoopFile(null, null, null, null); sparkContext.HadoopRDD(null, null, null); sparkContext.NewAPIHadoopFile(null, null, null, null); sparkContext.NewAPIHadoopRDD(null, null, null); sparkContext.Parallelize<int>(new int[] { 1, 2, 3, 4, 5 }); sparkContext.SequenceFile(null, null, null, null, null, null); sparkContext.SetCheckpointDir(null); sparkContext.SetJobGroup(null, null); sparkContext.SetLocalProperty(null, null); sparkContext.SetLogLevel(null); sparkContext.TextFile(null); sparkContext.WholeTextFiles(null); sparkContext.Stop(); sparkContext.Union<string>(null); }