static void Main(string[] args) { if (args.Length != 2) { Console.Error.WriteLine( "Usage: Remember to include input and output path as arguments"); Environment.Exit(1); } var sparkConf = SparkConfUtils.GetSparkConfigurationForFilePath(args); var spark = SparkSession .Builder() .AppName("Batch Job example using Apache Spark .Net") .GetOrCreate(); if (sparkConf != null) { sparkConf.ToList().ForEach(kv => { spark.Conf().Set(kv.Key, kv.Value); }); } var df = spark .Read() .Schema(GetSchema()) .Option("header", true) .Csv(args[0]); var processedDF = ProcessDataset(df); ShowDatasetInfo(processedDF); WriteData(processedDF, args[1]); Console.WriteLine("Finished .Net Spark Job!!"); }
static void Main(string[] args) { if (args.Length != 2) { Console.Error.WriteLine( "Usage: Remember to include input and output path as arguments"); Environment.Exit(1); } var sparkConf = SparkConfUtils.GetSparkConfigurationForFilePath(args); SparkSession spark = SparkSession .Builder() .AppName("Streaming example using Spark.NET") .GetOrCreate(); if (sparkConf != null) { sparkConf.ToList().ForEach(kv => { spark.Conf().Set(kv.Key, kv.Value); }); } var events = spark .ReadStream() .Format("eventhubs") .Options(EventHubConnection.GetEventHubConnectionSettings(eventHubPartitionCount: 2)) .Load(); var processedEvents = events .Select( FromJson(Col("body").Cast("string"), "temperature String, humidity String").As("Raw"), Col("properties"), Col("enqueuedTime") ) .WithColumn("Raw.temperature", Col("Raw.temperature").Cast("double")) .WithColumn("Raw.humidity", Col("Raw.humidity").Cast("double")) .WithColumnRenamed("Raw.temperature", "temperature") .WithColumnRenamed("Raw.humidity", "humidity") .WithColumn("temperatureAlert", Col("temperature") >= 40) .SelectExpr("temperature", "humidity", "properties", "enqueuedTime", "temperatureAlert"); processedEvents.PrintSchema(); var streamingQuery = processedEvents .WriteStream() .OutputMode(OutputMode.Append) .Format("console") .Option("path", args[0]) .Option("checkpointLocation", args[1]) .Start(); streamingQuery.AwaitTermination(); }