public void CompareTo_Same()
{
var schema1 = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Protobuf, new List <SchemaReference>());
var schema2 = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Protobuf, new List <SchemaReference>());
Assert.Equal(0, schema1.CompareTo(schema2));
}
public void CompareToTests()
{
// Note: the id and schema strings are not valid, however checking this
// is considered outside the scope of the CompareTo function.
var schema1 = new RegisteredSchema("test-a", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
var schema2 = new RegisteredSchema("test-b", 2, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
Assert.True(schema1.CompareTo(schema2) < 0);
Assert.True(schema2.CompareTo(schema1) > 0);
var schema3 = new RegisteredSchema("test-c", 4, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
var schema4 = new RegisteredSchema("test-d", 3, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
Assert.True(schema3.CompareTo(schema4) < 0);
Assert.True(schema4.CompareTo(schema3) > 0);
var schema5 = new RegisteredSchema("test-b", 2, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
Assert.Equal(0, schema2.CompareTo(schema5));
var schema6 = new RegisteredSchema("test-a", 2, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
Assert.True(schema1.CompareTo(schema6) < 0);
Assert.True(schema6.CompareTo(schema1) > 0);
var schema7 = new RegisteredSchema("test-b", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
Assert.True(schema1.CompareTo(schema7) < 0);
Assert.True(schema7.CompareTo(schema1) > 0);
}
public void ToStringTest()
{
var schema = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
Assert.Contains("test-key", schema.ToString());
Assert.Contains("1", schema.ToString());
Assert.Contains("42", schema.ToString());
}
public void Construct()
{
var schema = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
Assert.Equal("test-key", schema.Subject);
Assert.Equal(1, schema.Version);
Assert.Equal(42, schema.Id);
Assert.Equal("test-schema-string", schema.SchemaString);
Assert.Empty(schema.References);
Assert.Equal(SchemaType.Avro, schema.SchemaType);
}
public void GetHashCodeTest()
{
var schema1 = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Json, new List <SchemaReference> {
new SchemaReference("a", "b", 3)
});
var schema2 = new RegisteredSchema("test-value", 1, 42, "test-schema-string", SchemaType.Json, new List <SchemaReference> {
new SchemaReference("a", "b", 3)
});
// (very unlikely)
Assert.NotEqual(schema1.GetHashCode(), schema2.GetHashCode());
}
public void Construct2()
{
var schema = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Protobuf, new List <SchemaReference> {
new SchemaReference("schema-name", "subj", 4)
});
Assert.Equal("test-key", schema.Subject);
Assert.Equal(1, schema.Version);
Assert.Equal(42, schema.Id);
Assert.Equal("test-schema-string", schema.SchemaString);
Assert.Equal(SchemaType.Protobuf, schema.SchemaType);
Assert.Single(schema.References);
}
public void EqualsTests()
{
var schema1 = new RegisteredSchema("test-a", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
var schema2 = new RegisteredSchema("test-b", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
var schema3 = new RegisteredSchema("test-a", 2, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
var schema4 = new RegisteredSchema("test-a", 1, 44, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
var schema5 = new RegisteredSchema("test-a", 1, 42, "test-schema-string2", SchemaType.Avro, new List <SchemaReference>());
var schema6 = new RegisteredSchema("test-a", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>());
Assert.NotEqual(schema1, schema2);
Assert.NotEqual(schema1, schema3);
Assert.NotEqual(schema1, schema4);
Assert.NotEqual(schema1, schema5);
Assert.Equal(schema1, schema6);
Assert.False(schema1.Equals(schema2));
Assert.False(schema1.Equals(schema3));
Assert.False(schema1.Equals(schema4));
Assert.False(schema1.Equals(schema5));
Assert.True(schema1.Equals(schema6));
}
public static SData ReadSchema(RegisteredSchema registeredSchema)
{
return new SParser().ReadSchemaFile(GetResourceStream(registeredSchema.ToString()));
}
/// <summary>
/// Serialize GenericRecord instance to a byte array in Avro format. The serialized
/// data is preceded by a "magic byte" (1 byte) and the id of the schema as registered
/// in Confluent's Schema Registry (4 bytes, network byte order). This call may block or throw
/// on first use for a particular topic during schema registration.
/// </summary>
/// <param name="topic">
/// The topic associated with the data.
/// </param>
/// <param name="data">
/// The object to serialize.
/// </param>
/// <param name="isKey">
/// whether or not the data represents a message key.
/// </param>
/// <returns>
/// <paramref name="data" /> serialized as a byte array.
/// </returns>
public async Task <byte[]> Serialize(string topic, GenericRecord data, bool isKey)
{
try
{
int schemaId;
global::Avro.Schema writerSchema;
await serializeMutex.WaitAsync().ConfigureAwait(continueOnCapturedContext: false);
try
{
// TODO: If any of these caches fills up, this is probably an
// indication of misuse of the serializer. Ideally we would do
// something more sophisticated than the below + not allow
// the misuse to keep happening without warning.
if (knownSchemas.Count > schemaRegistryClient.MaxCachedSchemas ||
registeredSchemas.Count > schemaRegistryClient.MaxCachedSchemas ||
schemaIds.Count > schemaRegistryClient.MaxCachedSchemas)
{
knownSchemas.Clear();
registeredSchemas.Clear();
schemaIds.Clear();
}
// Determine a schema string corresponding to the schema object.
// TODO: It would be more efficient to use a hash function based
// on the instance reference, not the implementation provided by
// Schema.
writerSchema = data.Schema;
string writerSchemaString = null;
if (knownSchemas.ContainsKey(writerSchema))
{
writerSchemaString = knownSchemas[writerSchema];
}
else
{
writerSchemaString = writerSchema.ToString();
knownSchemas.Add(writerSchema, writerSchemaString);
}
// Verify schema compatibility (& register as required) + get the
// id corresponding to the schema.
// TODO: Again, the hash functions in use below are potentially
// slow since writerSchemaString is potentially long. It would be
// better to use hash functions based on the writerSchemaString
// object reference, not value.
string subject = this.subjectNameStrategy != null
// use the subject name strategy specified in the serializer config if available.
? this.subjectNameStrategy(new SerializationContext(isKey ? MessageComponentType.Key : MessageComponentType.Value, topic), data.Schema.Fullname)
// else fall back to the deprecated config from (or default as currently supplied by) SchemaRegistry.
: isKey
? schemaRegistryClient.ConstructKeySubjectName(topic, data.Schema.Fullname)
: schemaRegistryClient.ConstructValueSubjectName(topic, data.Schema.Fullname);
var subjectSchemaPair = new KeyValuePair <string, string>(subject, writerSchemaString);
if (!registeredSchemas.Contains(subjectSchemaPair))
{
int newSchemaId;
// first usage: register/get schema to check compatibility
if (autoRegisterSchema)
{
newSchemaId = await schemaRegistryClient.RegisterSchemaAsync(subject, writerSchemaString).ConfigureAwait(continueOnCapturedContext: false);
}
// https://www.confluent.io/blog/multiple-event-types-in-the-same-kafka-topic/
else if (useLatestSchema)
{
RegisteredSchema regSchema = await schemaRegistryClient.GetLatestSchemaAsync(subject)
.ConfigureAwait(continueOnCapturedContext: false);
//Do we have an Avro union with schema references
if (regSchema.References.Any() && IsUnion(regSchema.SchemaString))
{
RegisteredSchema registeredRefSchema = null;
StringBuilder schemaBuilder = new StringBuilder();
schemaBuilder.Append("[");
//We need to loop the schema references and perform a schema registry lookup
// in order to check compability with referencced schema
foreach (var refSchemaString in regSchema.References)
{
registeredRefSchema = await schemaRegistryClient.GetRegisteredSchemaAsync(refSchemaString.Subject,
refSchemaString.Version)
.ConfigureAwait(continueOnCapturedContext: false);
Avro.Schema refSchema = Avro.Schema.Parse(registeredRefSchema.SchemaString);
if (refSchema.Tag != Avro.Schema.Type.Record)
{
throw new NotSupportedException("Only union schemas containing references to a record are supported for now");
}
schemaBuilder.Append($"{registeredRefSchema.SchemaString}");
if (regSchema.References.Last() != refSchemaString)
{
schemaBuilder.Append(",");
}
}
schemaBuilder.Append("]");
unionSchemas[writerSchema] = global::Avro.Schema.Parse(schemaBuilder.ToString());
newSchemaId = regSchema.Id;
// subjectSchemaPair = new KeyValuePair<string, string>(subject, writerSchema.ToString());
}
else
{
newSchemaId = await schemaRegistryClient.GetSchemaIdAsync(subject, writerSchemaString)
.ConfigureAwait(continueOnCapturedContext: false);
}
}
else
{
newSchemaId = await schemaRegistryClient.GetSchemaIdAsync(subject, writerSchemaString).ConfigureAwait(continueOnCapturedContext: false);
}
if (!schemaIds.ContainsKey(writerSchemaString))
{
schemaIds.Add(writerSchemaString, newSchemaId);
}
else if (schemaIds[writerSchemaString] != newSchemaId)
{
schemaIds.Clear();
registeredSchemas.Clear();
throw new KafkaException(new Error(isKey ? ErrorCode.Local_KeySerialization : ErrorCode.Local_ValueSerialization, $"Duplicate schema registration encountered: Schema ids {schemaIds[writerSchemaString]} and {newSchemaId} are associated with the same schema."));
}
registeredSchemas.Add(subjectSchemaPair);
}
schemaId = schemaIds[writerSchemaString];
}
finally
{
serializeMutex.Release();
}
Avro.Schema unionSchema;
if (unionSchemas.TryGetValue(writerSchema, out unionSchema))
{
writerSchema = unionSchema;
}
using (var stream = new MemoryStream(initialBufferSize))
using (var writer = new BinaryWriter(stream))
{
stream.WriteByte(Constants.MagicByte);
writer.Write(IPAddress.HostToNetworkOrder(schemaId));
new GenericWriter <GenericRecord>(writerSchema)
.Write(data, new BinaryEncoder(stream));
return(stream.ToArray());
}
}
catch (AggregateException e)
{
throw e.InnerException;
}
}
public static SData ReadSchema(RegisteredSchema registeredSchema)
{
return(new SParser().ReadSchemaFile(GetResourceStream(registeredSchema.ToString())));
}
public static bool Read(RegisteredSchema schemaName, string p21FileName)
{
try
{
var p21 = new InstanceDB(p21FileName);
var sData = SParser.ReadSchema(schemaName);
Instance.AssignEntityDefToInstance(sData, p21);
Instance.ConvertToEarlyBinding(p21);
return true;
}
catch (Exception)
{
MessageBox.Show(string.Format("There was an error in parsing {0}.", Path.GetFileName(p21FileName)));
return false;
}
}
