public void CompareTo_Same() { var schema1 = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Protobuf, new List <SchemaReference>()); var schema2 = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Protobuf, new List <SchemaReference>()); Assert.Equal(0, schema1.CompareTo(schema2)); }
public void CompareToTests() { // Note: the id and schema strings are not valid, however checking this // is considered outside the scope of the CompareTo function. var schema1 = new RegisteredSchema("test-a", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); var schema2 = new RegisteredSchema("test-b", 2, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); Assert.True(schema1.CompareTo(schema2) < 0); Assert.True(schema2.CompareTo(schema1) > 0); var schema3 = new RegisteredSchema("test-c", 4, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); var schema4 = new RegisteredSchema("test-d", 3, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); Assert.True(schema3.CompareTo(schema4) < 0); Assert.True(schema4.CompareTo(schema3) > 0); var schema5 = new RegisteredSchema("test-b", 2, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); Assert.Equal(0, schema2.CompareTo(schema5)); var schema6 = new RegisteredSchema("test-a", 2, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); Assert.True(schema1.CompareTo(schema6) < 0); Assert.True(schema6.CompareTo(schema1) > 0); var schema7 = new RegisteredSchema("test-b", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); Assert.True(schema1.CompareTo(schema7) < 0); Assert.True(schema7.CompareTo(schema1) > 0); }
public void ToStringTest() { var schema = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); Assert.Contains("test-key", schema.ToString()); Assert.Contains("1", schema.ToString()); Assert.Contains("42", schema.ToString()); }
public void Construct() { var schema = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); Assert.Equal("test-key", schema.Subject); Assert.Equal(1, schema.Version); Assert.Equal(42, schema.Id); Assert.Equal("test-schema-string", schema.SchemaString); Assert.Empty(schema.References); Assert.Equal(SchemaType.Avro, schema.SchemaType); }
public void GetHashCodeTest() { var schema1 = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Json, new List <SchemaReference> { new SchemaReference("a", "b", 3) }); var schema2 = new RegisteredSchema("test-value", 1, 42, "test-schema-string", SchemaType.Json, new List <SchemaReference> { new SchemaReference("a", "b", 3) }); // (very unlikely) Assert.NotEqual(schema1.GetHashCode(), schema2.GetHashCode()); }
public void Construct2() { var schema = new RegisteredSchema("test-key", 1, 42, "test-schema-string", SchemaType.Protobuf, new List <SchemaReference> { new SchemaReference("schema-name", "subj", 4) }); Assert.Equal("test-key", schema.Subject); Assert.Equal(1, schema.Version); Assert.Equal(42, schema.Id); Assert.Equal("test-schema-string", schema.SchemaString); Assert.Equal(SchemaType.Protobuf, schema.SchemaType); Assert.Single(schema.References); }
public void EqualsTests() { var schema1 = new RegisteredSchema("test-a", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); var schema2 = new RegisteredSchema("test-b", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); var schema3 = new RegisteredSchema("test-a", 2, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); var schema4 = new RegisteredSchema("test-a", 1, 44, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); var schema5 = new RegisteredSchema("test-a", 1, 42, "test-schema-string2", SchemaType.Avro, new List <SchemaReference>()); var schema6 = new RegisteredSchema("test-a", 1, 42, "test-schema-string", SchemaType.Avro, new List <SchemaReference>()); Assert.NotEqual(schema1, schema2); Assert.NotEqual(schema1, schema3); Assert.NotEqual(schema1, schema4); Assert.NotEqual(schema1, schema5); Assert.Equal(schema1, schema6); Assert.False(schema1.Equals(schema2)); Assert.False(schema1.Equals(schema3)); Assert.False(schema1.Equals(schema4)); Assert.False(schema1.Equals(schema5)); Assert.True(schema1.Equals(schema6)); }
public static SData ReadSchema(RegisteredSchema registeredSchema) { return new SParser().ReadSchemaFile(GetResourceStream(registeredSchema.ToString())); }
/// <summary> /// Serialize GenericRecord instance to a byte array in Avro format. The serialized /// data is preceded by a "magic byte" (1 byte) and the id of the schema as registered /// in Confluent's Schema Registry (4 bytes, network byte order). This call may block or throw /// on first use for a particular topic during schema registration. /// </summary> /// <param name="topic"> /// The topic associated with the data. /// </param> /// <param name="data"> /// The object to serialize. /// </param> /// <param name="isKey"> /// whether or not the data represents a message key. /// </param> /// <returns> /// <paramref name="data" /> serialized as a byte array. /// </returns> public async Task <byte[]> Serialize(string topic, GenericRecord data, bool isKey) { try { int schemaId; global::Avro.Schema writerSchema; await serializeMutex.WaitAsync().ConfigureAwait(continueOnCapturedContext: false); try { // TODO: If any of these caches fills up, this is probably an // indication of misuse of the serializer. Ideally we would do // something more sophisticated than the below + not allow // the misuse to keep happening without warning. if (knownSchemas.Count > schemaRegistryClient.MaxCachedSchemas || registeredSchemas.Count > schemaRegistryClient.MaxCachedSchemas || schemaIds.Count > schemaRegistryClient.MaxCachedSchemas) { knownSchemas.Clear(); registeredSchemas.Clear(); schemaIds.Clear(); } // Determine a schema string corresponding to the schema object. // TODO: It would be more efficient to use a hash function based // on the instance reference, not the implementation provided by // Schema. writerSchema = data.Schema; string writerSchemaString = null; if (knownSchemas.ContainsKey(writerSchema)) { writerSchemaString = knownSchemas[writerSchema]; } else { writerSchemaString = writerSchema.ToString(); knownSchemas.Add(writerSchema, writerSchemaString); } // Verify schema compatibility (& register as required) + get the // id corresponding to the schema. // TODO: Again, the hash functions in use below are potentially // slow since writerSchemaString is potentially long. It would be // better to use hash functions based on the writerSchemaString // object reference, not value. string subject = this.subjectNameStrategy != null // use the subject name strategy specified in the serializer config if available. ? this.subjectNameStrategy(new SerializationContext(isKey ? MessageComponentType.Key : MessageComponentType.Value, topic), data.Schema.Fullname) // else fall back to the deprecated config from (or default as currently supplied by) SchemaRegistry. : isKey ? schemaRegistryClient.ConstructKeySubjectName(topic, data.Schema.Fullname) : schemaRegistryClient.ConstructValueSubjectName(topic, data.Schema.Fullname); var subjectSchemaPair = new KeyValuePair <string, string>(subject, writerSchemaString); if (!registeredSchemas.Contains(subjectSchemaPair)) { int newSchemaId; // first usage: register/get schema to check compatibility if (autoRegisterSchema) { newSchemaId = await schemaRegistryClient.RegisterSchemaAsync(subject, writerSchemaString).ConfigureAwait(continueOnCapturedContext: false); } // https://www.confluent.io/blog/multiple-event-types-in-the-same-kafka-topic/ else if (useLatestSchema) { RegisteredSchema regSchema = await schemaRegistryClient.GetLatestSchemaAsync(subject) .ConfigureAwait(continueOnCapturedContext: false); //Do we have an Avro union with schema references if (regSchema.References.Any() && IsUnion(regSchema.SchemaString)) { RegisteredSchema registeredRefSchema = null; StringBuilder schemaBuilder = new StringBuilder(); schemaBuilder.Append("["); //We need to loop the schema references and perform a schema registry lookup // in order to check compability with referencced schema foreach (var refSchemaString in regSchema.References) { registeredRefSchema = await schemaRegistryClient.GetRegisteredSchemaAsync(refSchemaString.Subject, refSchemaString.Version) .ConfigureAwait(continueOnCapturedContext: false); Avro.Schema refSchema = Avro.Schema.Parse(registeredRefSchema.SchemaString); if (refSchema.Tag != Avro.Schema.Type.Record) { throw new NotSupportedException("Only union schemas containing references to a record are supported for now"); } schemaBuilder.Append($"{registeredRefSchema.SchemaString}"); if (regSchema.References.Last() != refSchemaString) { schemaBuilder.Append(","); } } schemaBuilder.Append("]"); unionSchemas[writerSchema] = global::Avro.Schema.Parse(schemaBuilder.ToString()); newSchemaId = regSchema.Id; // subjectSchemaPair = new KeyValuePair<string, string>(subject, writerSchema.ToString()); } else { newSchemaId = await schemaRegistryClient.GetSchemaIdAsync(subject, writerSchemaString) .ConfigureAwait(continueOnCapturedContext: false); } } else { newSchemaId = await schemaRegistryClient.GetSchemaIdAsync(subject, writerSchemaString).ConfigureAwait(continueOnCapturedContext: false); } if (!schemaIds.ContainsKey(writerSchemaString)) { schemaIds.Add(writerSchemaString, newSchemaId); } else if (schemaIds[writerSchemaString] != newSchemaId) { schemaIds.Clear(); registeredSchemas.Clear(); throw new KafkaException(new Error(isKey ? ErrorCode.Local_KeySerialization : ErrorCode.Local_ValueSerialization, $"Duplicate schema registration encountered: Schema ids {schemaIds[writerSchemaString]} and {newSchemaId} are associated with the same schema.")); } registeredSchemas.Add(subjectSchemaPair); } schemaId = schemaIds[writerSchemaString]; } finally { serializeMutex.Release(); } Avro.Schema unionSchema; if (unionSchemas.TryGetValue(writerSchema, out unionSchema)) { writerSchema = unionSchema; } using (var stream = new MemoryStream(initialBufferSize)) using (var writer = new BinaryWriter(stream)) { stream.WriteByte(Constants.MagicByte); writer.Write(IPAddress.HostToNetworkOrder(schemaId)); new GenericWriter <GenericRecord>(writerSchema) .Write(data, new BinaryEncoder(stream)); return(stream.ToArray()); } } catch (AggregateException e) { throw e.InnerException; } }
public static SData ReadSchema(RegisteredSchema registeredSchema) { return(new SParser().ReadSchemaFile(GetResourceStream(registeredSchema.ToString()))); }
public static bool Read(RegisteredSchema schemaName, string p21FileName) { try { var p21 = new InstanceDB(p21FileName); var sData = SParser.ReadSchema(schemaName); Instance.AssignEntityDefToInstance(sData, p21); Instance.ConvertToEarlyBinding(p21); return true; } catch (Exception) { MessageBox.Show(string.Format("There was an error in parsing {0}.", Path.GetFileName(p21FileName))); return false; } }