public static async Task Main(string[] args) { // Use a specific memory pool from which arrays will be allocated (optional) var memoryAllocator = new NativeMemoryAllocator(alignment: 64); // Build a record batch using the Fluent API var recordBatch = new RecordBatch.Builder(memoryAllocator) .Append("Column A", false, col => col.Int32(array => array.AppendRange(Enumerable.Range(0, 10)))) .Append("Column B", false, col => col.Float(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => Convert.ToSingle(x * 2))))) .Append("Column C", false, col => col.String(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => $"Item {x+1}")))) .Append("Column D", false, col => col.Boolean(array => array.AppendRange(Enumerable.Range(0, 10).Select(x => x % 2 == 0)))) .Build(); // Print memory allocation statistics Console.WriteLine("Allocations: {0}", memoryAllocator.Statistics.Allocations); Console.WriteLine("Allocated: {0} byte(s)", memoryAllocator.Statistics.BytesAllocated); // Write record batch to a file using (var stream = File.OpenWrite("test.arrow")) using (var writer = new ArrowFileWriter(stream, recordBatch.Schema)) { await writer.WriteRecordBatchAsync(recordBatch); await writer.WriteFooterAsync(); } Console.WriteLine("Done"); Console.ReadKey(); }
public ASAHost(string sql) { //Console.WriteLine("The sql query is: " + sql); // Console.WriteLine(inputSchema); this.outputs = SqlQueryRunner.Query( sql, new CompilerConfig() { SqlCompatibility = new SqlCompatibility() }, ClrFramework.NetStandard20, QueryHelper.BinLocations[ClrFramework.NetStandard20], new Dictionary <string, Subject <IRecord> >() { { "input", this.input } }); if (this.outputs.Count != 1) { throw new ArgumentException("Query: '" + sql + "' returned 0 or more than 1 output: " + this.outputs.Count); } this.outputs.First().Value.Subscribe(r => this.outputRecords.Enqueue(r)); this.memoryAllocator = new NativeMemoryAllocator(alignment: 64); }
public void Process(int entities, int iterations, IAction[] actions) { var allocator = new NativeMemoryAllocator(); var batch = Generator.GenerateBatch(allocator, entities); ExecuteActions(allocator, batch, actions.ToArray(), iterations); }
public void BenchmarkSetup() { // load csv file into list of objects string landRegistryDataPath = string.Concat(Enumerable.Repeat("..\\", 9)) + "data\\pp-monthly-update-new-version.csv"; Console.WriteLine("Loading land registry data"); var dataLoadTime = Stopwatch.StartNew(); using (var reader = new StreamReader(landRegistryDataPath)) using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture)) { // land registry data file does not have headers csv.Configuration.HasHeaderRecord = false; csv.Configuration.RegisterClassMap <LandRegistryRecordMap>(); this.landRegistryRecords = csv.GetRecords <LandRegistryRecord>().ToList(); } dataLoadTime.Stop(); Console.WriteLine("Loading CSV data took {0}ms", dataLoadTime.ElapsedMilliseconds); Console.WriteLine("Loaded {0} records", landRegistryRecords.Count); Console.WriteLine("----------------"); // load csv file into apache arrow arrays / table Console.WriteLine("Loading land registry records in arrow arrays"); var stringEncoding = Encoding.ASCII; var arrowLoadTime = Stopwatch.StartNew(); var memoryAllocator = new NativeMemoryAllocator(alignment: 64); this.recordBatch = new RecordBatch.Builder(memoryAllocator) .Append("Date", false, col => col.Date32(array => array.AppendRange(landRegistryRecords.Select(r => r.Date)))) .Append("Price", false, col => col.Float(array => array.AppendRange(landRegistryRecords.Select(r => r.Price)))) .Append("PropertyType", false, col => col.String(array => array.AppendRange(landRegistryRecords.Select(r => r.PropertyType), stringEncoding))) .Append("Tenure", false, col => col.String(array => array.AppendRange(landRegistryRecords.Select(r => r.Tenure), stringEncoding))) .Build(); arrowLoadTime.Stop(); Console.WriteLine("Loaded {0} arrays with length {1} in {2}ms", recordBatch.ColumnCount, recordBatch.Length, arrowLoadTime.ElapsedMilliseconds); Console.WriteLine("----------------"); }