public static async Task Main(string[] args)
{
// Use a specific memory pool from which arrays will be allocated (optional)
var memoryAllocator = new NativeMemoryAllocator(alignment: 64);
// Build a record batch using the Fluent API
var recordBatch = new RecordBatch.Builder(memoryAllocator)
.Append("Column A", false, col => col.Int32(array => array.AppendRange(Enumerable.Range(0, 10))))
.Append("Column B", false, col => col.Float(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => Convert.ToSingle(x * 2)))))
.Append("Column C", false, col => col.String(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => $"Item {x+1}"))))
.Append("Column D", false, col => col.Boolean(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => x % 2 == 0))))
.Build();
// Print memory allocation statistics
Console.WriteLine("Allocations: {0}", memoryAllocator.Statistics.Allocations);
Console.WriteLine("Allocated: {0} byte(s)", memoryAllocator.Statistics.BytesAllocated);
// Write record batch to a file
using (var stream = File.OpenWrite("test.arrow"))
using (var writer = new ArrowFileWriter(stream, recordBatch.Schema))
{
await writer.WriteRecordBatchAsync(recordBatch);
await writer.WriteFooterAsync();
}
Console.WriteLine("Done");
Console.ReadKey();
}
public ASAHost(string sql)
{
//Console.WriteLine("The sql query is: " + sql);
// Console.WriteLine(inputSchema);
this.outputs = SqlQueryRunner.Query(
sql,
new CompilerConfig()
{
SqlCompatibility = new SqlCompatibility()
},
ClrFramework.NetStandard20,
QueryHelper.BinLocations[ClrFramework.NetStandard20],
new Dictionary <string, Subject <IRecord> >()
{
{ "input", this.input }
});
if (this.outputs.Count != 1)
{
throw new ArgumentException("Query: '" + sql + "' returned 0 or more than 1 output: " + this.outputs.Count);
}
this.outputs.First().Value.Subscribe(r => this.outputRecords.Enqueue(r));
this.memoryAllocator = new NativeMemoryAllocator(alignment: 64);
}
public void Process(int entities, int iterations, IAction[] actions)
{
var allocator = new NativeMemoryAllocator();
var batch = Generator.GenerateBatch(allocator, entities);
ExecuteActions(allocator, batch, actions.ToArray(), iterations);
}
public void BenchmarkSetup()
{
// load csv file into list of objects
string landRegistryDataPath = string.Concat(Enumerable.Repeat("..\\", 9)) + "data\\pp-monthly-update-new-version.csv";
Console.WriteLine("Loading land registry data");
var dataLoadTime = Stopwatch.StartNew();
using (var reader = new StreamReader(landRegistryDataPath))
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
// land registry data file does not have headers
csv.Configuration.HasHeaderRecord = false;
csv.Configuration.RegisterClassMap <LandRegistryRecordMap>();
this.landRegistryRecords = csv.GetRecords <LandRegistryRecord>().ToList();
}
dataLoadTime.Stop();
Console.WriteLine("Loading CSV data took {0}ms", dataLoadTime.ElapsedMilliseconds);
Console.WriteLine("Loaded {0} records", landRegistryRecords.Count);
Console.WriteLine("----------------");
// load csv file into apache arrow arrays / table
Console.WriteLine("Loading land registry records in arrow arrays");
var stringEncoding = Encoding.ASCII;
var arrowLoadTime = Stopwatch.StartNew();
var memoryAllocator = new NativeMemoryAllocator(alignment: 64);
this.recordBatch = new RecordBatch.Builder(memoryAllocator)
.Append("Date", false, col => col.Date32(array => array.AppendRange(landRegistryRecords.Select(r => r.Date))))
.Append("Price", false, col => col.Float(array => array.AppendRange(landRegistryRecords.Select(r => r.Price))))
.Append("PropertyType", false, col => col.String(array => array.AppendRange(landRegistryRecords.Select(r => r.PropertyType), stringEncoding)))
.Append("Tenure", false, col => col.String(array => array.AppendRange(landRegistryRecords.Select(r => r.Tenure), stringEncoding)))
.Build();
arrowLoadTime.Stop();
Console.WriteLine("Loaded {0} arrays with length {1} in {2}ms", recordBatch.ColumnCount, recordBatch.Length, arrowLoadTime.ElapsedMilliseconds);
Console.WriteLine("----------------");
}
