} // end AvroEncode
/// <summary>
/// Make a copy of an object as an Avro GenericRecord
/// </summary>
/// <param name="obj">Original object</param>
/// <param name="ktype">An associated KineticaType object that
/// describes the original object.</param>
/// <returns>GenericRecord object which is a copy of the specified object</returns>
private Avro.Generic.GenericRecord MakeGenericRecord(object obj, KineticaType ktype)
{
// Get the schema
var schema = KineticaData.SchemaFromType(obj.GetType(), ktype);
// Create a new GenericRecord for this schema
var recordToSend = new Avro.Generic.GenericRecord(schema);
// Copy each field from obj to recordToSend
foreach (var field in schema.Fields)
{
var property = obj.GetType()
.GetProperties()
.FirstOrDefault(prop => prop.Name.ToLowerInvariant() == field.Name.ToLowerInvariant());
if (property == null)
{
continue;
}
recordToSend.Add(field.Name, property.GetValue(obj, null));
}
// Return the newly created object
return(recordToSend);
}
private T Parse <T>(string fieldName, Avro.Generic.GenericRecord record)
{
try
{
object data;
record.TryGetValue(fieldName, out data);
return((T)data);
}
catch
{
return(default(T));
}
}
public EventHubAvroData(Avro.Generic.GenericRecord record)
{
this.SequenceNumber = Parse <long>("SequenceNumber", record);
this.Offset = Parse <string>("Offset", record);
string dateString = Parse <string>("EnqueuedTimeUtc", record);
DateTime enq;
if (DateTime.TryParse(dateString, out enq) == true)
{
EnqueuedTimeUtc = enq;
}
this.Body = Parse <byte[]>("Body", record);
}
/// <summary>
/// Decode binary Avro data into an object.
/// </summary>
/// <typeparam name="T">Type of expected object</typeparam>
/// <param name="bytes">Binary Avro data</param>
/// <param name="ktype">An optional KineticaType object to help in decoding the object.</param>
/// <returns>New object</returns>
private T AvroDecode <T>(byte[] bytes, KineticaType ktype = null) where T : new()
{
// Get the schema
var schema = KineticaData.SchemaFromType(typeof(T), ktype);
// Create a stream to read the binary data
using (var ms = new MemoryStream(bytes))
{
// Create a new object to return
T obj = new T();
if (obj is Avro.Specific.ISpecificRecord)
{
var reader = new Avro.Specific.SpecificDefaultReader(schema, schema);
reader.Read(obj, new BinaryDecoder(ms));
}
else
{
// Not ISpecificRecord, so first read into a new GenericRecord
var reader = new Avro.Generic.DefaultReader(schema, schema);
Avro.Generic.GenericRecord recordToReceive = new Avro.Generic.GenericRecord(schema);
reader.Read(recordToReceive, new BinaryDecoder(ms));
// Now, copy all the fields from the GenericRecord to obj
foreach (var field in schema.Fields)
{
var property = obj.GetType()
.GetProperties()
.FirstOrDefault(prop => prop.Name.ToLowerInvariant() == field.Name.ToLowerInvariant());
if (property == null)
{
continue;
}
object val;
// Try to get the property
if (recordToReceive.TryGetValue(field.Name, out val))
{
// If successful, write the property to obj
property.SetValue(obj, val);
}
} // end foreach
} // end if-else
// Return the new object
return(obj);
} // end using
} // end AvroDecode<T>
} // end ContentsToString
/// <summary>
/// Decodes binary encoded data of a dynamically created table returned by the server.
/// </summary>
/// <param name="dynamic_table_schema_string">The schema string for the dynamically created table.</param>
/// <param name="encoded_data">The binary encoded data.</param>
/// <returns>A list of KineticaRecord objects with the decoded data.</returns>
public static IList <KineticaRecord> DecodeDynamicTableRecords(string dynamic_table_schema_string, byte[] encoded_data)
{
// Get a record schema from the schema string
Schema dynamic_table_schema;
try
{
dynamic_table_schema = Avro.Schema.Parse(dynamic_table_schema_string);
}
catch (Exception ex)
{
throw new KineticaException(ex.ToString());
}
// The container for the decoded data (put into distinct records)
IList <KineticaRecord> records = new List <KineticaRecord>();
// Decode the dynamic table schema to extract the column names and types.
// Then, decode each individual record.
using (var ms = new MemoryStream(encoded_data))
{
// Read the table schema into a new GenericRecord
// ----------------------------------------------
var reader = new Avro.Generic.DefaultReader(dynamic_table_schema, dynamic_table_schema);
BinaryDecoder decoder = new BinaryDecoder(ms);
Avro.Generic.GenericRecord obj = (Avro.Generic.GenericRecord)reader.Read(null, dynamic_table_schema, dynamic_table_schema, decoder);
// Extract the column names from the encoded data
object column_headers_0 = new object();
Object[] column_headers = null;
if (obj.TryGetValue("column_headers", out column_headers_0)) // try to get the data out
{
column_headers = ( Object[] )column_headers_0;
}
// Extract the column types from the encoded data
object column_types_0 = new object();
Object[] column_types = null;
if (obj.TryGetValue("column_datatypes", out column_types_0)) // try to get the data out
{
column_types = ( Object[] )column_types_0;
}
// Find out how many columns are returned
int num_columns = column_headers.Length;
// Extract the column data from the encoded data (ignore the headers and types)
// and create a list with only the record data
Object[][] encoded_column_data = new Object[num_columns][];
for (int i = 0; i < num_columns; ++i)
{
// Get the column name (e.g. the first column is titled "column_1")
string column_name = $"column_{i+1}";
// Get the column data out
object column_data_0 = new object();
Object[] column_data = null;
if (obj.TryGetValue(column_name, out column_data_0)) // try to get the data out
{
column_data = ( Object[] )column_data_0;
}
// Save this column's data in the 2D array declared above
encoded_column_data[i] = column_data;
} // done separating the column data from the headers and type
// Find out how many values per column are returned
int num_records = encoded_column_data[0].Length;
// Make sure that all the column data are of the same length
foreach (Object[] l in encoded_column_data)
{
if (l.Length != num_records)
{
throw new KineticaException("Dynamic table has uneven column data lengths");
}
}
// Based on the column headers and types, create a KineticaType
KineticaType dynamic_record_type = KineticaType.fromDynamicSchema(dynamic_table_schema_string, column_headers, column_types);
// Using the dynamic record type, create a RecordSchema object
// ( off which we'll create the records)
Avro.RecordSchema record_schema = (Avro.RecordSchema)dynamic_record_type.getSchema();
// Create the records by decoding the binary encoded data
// (column-major data into row-major data)
for (int record_idx = 0; record_idx < num_records; ++record_idx)
{
// Create a GenericRecord object based on the KineticaType
KineticaRecord record = new KineticaRecord(record_schema);
// Go through each column, decode the next value and put it into the record
for (int column_idx = 0; column_idx < num_columns; ++column_idx)
{
// Get the value to be put
var val = encoded_column_data[column_idx][record_idx];
// Get the property of the record into which the value need to saved
var field = record_schema.Fields[column_idx];
// Set the value
record.Add(field.Name, val);
} // end inner for loop
// Save the record
records.Add(record);
} // end outer for loop
} // end decoding block
return(records);
} // end DecodeDynamicTableRecords
public GenericAvroRecord(byte[] schemaVersion, Avro.Schema schema, IList <Field> fields, Avro.Generic.GenericRecord record) : base(schemaVersion, fields)
{
_schema = schema;
_record = record;
}
