public S3LowLevelTestController(
IAWSSettings awsSettings
, ILogger <S3LowLevelTestController> logger)
{
this._logger = logger;
this._awsSettings = awsSettings ??
throw new ArgumentNullException(nameof(awsSettings));
//RegionEndpoint endpoint = RegionEndpoint.GetBySystemName(this._awsSettings.Region); //don't work
this._s3Client = new AmazonS3Client(this._awsSettings.AccessKey, this._awsSettings.SecretKey, RegionEndpoint.USEast1);
}
public string SparkTest([FromServices] IAWSSettings awsSettings)
{
string result = "ok";
try
{
SparkSession spark = SparkSession
.Builder()
.AppName("itur")
.GetOrCreate();
var mySchema = new Microsoft.Spark.Sql.Types.StructType(new[]
{
new StructField("IturCode", new Microsoft.Spark.Sql.Types.StringType()),
new StructField("IturERP", new Microsoft.Spark.Sql.Types.StringType()),
new StructField("QuantityEdit", new Microsoft.Spark.Sql.Types.StringType()),
new StructField("PartialQuantity", new Microsoft.Spark.Sql.Types.StringType())
});
string assemblyLocation = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string iturInputPath = Path.Combine(assemblyLocation, "data", "itur.csv");
DataFrame df = spark.Read()
.Format("csv")
.Schema(mySchema)
.Option("delimiter", ",")
.Option("header", true)
//.Option("dateFormat", "dd/MM/yyyy")
.Load(iturInputPath);
string dt = DateTime.Now.ToString("MMddhhmmss");
string outputfile = Path.Combine(assemblyLocation, "outputData", $"itur_out{dt}.json");
df.Write().Json(outputfile);
//string toPath = $"s3n://{awsSettings.AccessKey}:{awsSettings.SecretKey}@{_bucketName}/{path}";
//spark.Range(100).Repartition(5).Write().Mode("overwrite").Text(toPath) ;
spark.Stop();
}
catch (Exception ex)
{
result = ex.Message;
}
return(result);
}
public string DeltaTest([FromServices] IAWSSettings awsSettings)
{
string result = String.Empty;
try
{
SparkSession spark = SparkSession
.Builder()
.AppName("DeltaTest")
.GetOrCreate();
string tempDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string dt = DateTime.Now.ToString("MMddhhmmss");
string path = Path.Combine(tempDirectory, $"delta-table{dt}");
// Write data to a Delta table.
DataFrame data = spark.Range(0, 5);
result += "Write data to a Delta table >> spark.Range(0, 5)" + " ";
foreach (var row in data.ToDF().Collect())
{
result += row.Values[0];
result += " | ";
}
result += " ";
data.Write().Format("delta").Save(path);
// Create a second iteration of the table.
data = spark.Range(5, 10);
result += "Create a second iteration of the table >> spark.Range(0, 5)" + " ";
foreach (var row in data.ToDF().Collect())
{
result += row.Values[0];
result += " | ";
}
result += " ";
data.Write().Format("delta").Mode("overwrite").Save(path);
// Load the data into a DeltaTable object.
DeltaTable deltaTable = DeltaTable.ForPath(path);
result += "Load the data into a DeltaTable object >> DeltaTable.ForPath" + " ";
foreach (var row in deltaTable.ToDF().Collect())
{
result += row.Values[0];
result += " | ";
}
result += " ";
// Update every even value by adding 100 to it.
deltaTable.Update(
condition: Functions.Expr("id % 2 == 0"),
set: new Dictionary <string, Column>()
{
{ "id", Functions.Expr("id + 100") }
});
result += "Update every even value by adding 100 to it." + " ";
foreach (var row in deltaTable.ToDF().Collect())
{
result += row.Values[0];
result += " | ";
}
result += " ";
// Delete every even value.
deltaTable.Delete(condition: Functions.Expr("id % 2 == 0"));
result += "Delete every even value id % 2 == 0" + " ";
foreach (var row in deltaTable.ToDF().Collect())
{
result += row.Values[0];
result += " | ";
}
result += " ";
// Upsert (merge) new data.
DataFrame newData = spark.Range(0, 20).As("newData").ToDF();
result += "Upsert (merge) new data" + Environment.NewLine;
foreach (var row in newData.ToDF().Collect())
{
result += row.Values[0];
result += " | ";
}
result += " ";
deltaTable.As("oldData")
.Merge(newData, "oldData.id = newData.id")
.WhenMatched()
.Update(
new Dictionary <string, Column>()
{
{ "id", Functions.Col("newData.id") }
})
.WhenNotMatched()
.InsertExpr(new Dictionary <string, string>()
{
{ "id", "newData.id" }
})
.Execute();
spark.Stop();
}
catch (Exception ex)
{
result = ex.Message;
}
return(result);
}
