/**
* Create a User Record.
*
* @param aggregated whether the record is aggregated
* @param record Kinesis record
* @param subSequenceNumber subsequence number
* @param explicitHashKey explicit hash key
*/
protected UserRecord(bool aggregated, Amazon.Kinesis.Model.Record record, long subSequenceNumber, string explicitHashKey)
{
if (subSequenceNumber < 0)
{
throw new Amazon.Kinesis.Model.InvalidArgumentException("Cannot have an invalid, negative subsequence number");
}
this.aggregated = aggregated;
this.subSequenceNumber = subSequenceNumber;
this.explicitHashKey = explicitHashKey;
this.SequenceNumber = record.SequenceNumber;
this.Data = record.Data;
this.PartitionKey = record.PartitionKey;
this.ApproximateArrivalTimestamp = record.ApproximateArrivalTimestamp;
}
/**
* Create a User Record from a Kinesis Record.
*
* @param record Kinesis record
*/
public UserRecord(Amazon.Kinesis.Model.Record record)
: this(false, record, 0, null)
{
}
/**
* This method deaggregates the given list of Amazon Kinesis records into a
* list of KPL user records. This method will then return the resulting list
* of KPL user records.
*
* @param records A list of Amazon Kinesis records, each possibly aggregated.
* @return A resulting list of deaggregated KPL user records.
*/
public static List <UserRecord> deaggregate(List <Amazon.Kinesis.Model.Record> records, out string errorMesaage)
{
errorMesaage = "";
List <UserRecord> result = new List <UserRecord>();
byte[] kMagic = null;
unchecked
{
kMagic = new byte[] { (byte)-13, (byte)-119, (byte)-102, (byte)-62 };
}
foreach (var record in records)
{
bool aggregated = true;
long subSeqNum = 0;
MemoryStream bb = record.Data;
byte[] bytes = new byte[bb.Length];
bb.Read(bytes, 0, bytes.Length);
record.Data = new MemoryStream(bytes);//setting data in case of aggregated = false;
if (bytes.Length < kMagic.Length)
{
aggregated = false;
}
else if (!Utils.AreArrayEqual(kMagic, bytes.Take(kMagic.Length).ToArray()))
{
aggregated = false;
}
if (bytes.Length < DIGEST_SIZE + kMagic.Length)
{
aggregated = false;
}
if (aggregated)
{
byte[] serializedData = bytes.Skip(kMagic.Length).Take(bytes.Length - DIGEST_SIZE - kMagic.Length).ToArray();
byte[] md5Hash = bytes.Skip(bytes.Length - DIGEST_SIZE).Take(DIGEST_SIZE).ToArray();
byte[] calcMD5 = Utils.GetMD5(serializedData);
if (!Utils.AreArrayEqual(md5Hash, calcMD5))
{
aggregated = false;
}
else
{
ByteString bytestr = ByteString.CopyFrom(serializedData);
try
{
Aws.Kinesis.Protobuf.AggregatedRecord ar = Aws.Kinesis.Protobuf.AggregatedRecord.Parser.ParseFrom(bytestr);
aggregated = true;
var pks = ar.PartitionKeyTable;
var ehks = ar.ExplicitHashKeyTable;
var aat = record.ApproximateArrivalTimestamp;
try
{
int recordsInCurrRecord = 0;
for (int i = 0; i < ar.Records.Count; i++)
{
Aws.Kinesis.Protobuf.Record mr = ar.Records[i];
string partitionKey = pks[(int)mr.PartitionKeyIndex];
string explicitHashKey = ehks[(int)mr.ExplicitHashKeyIndex];
//long effectiveHashKey = explicitHashKey != "-1" ? int.Parse(explicitHashKey) : new BigInteger(1, md5(partitionKey.getBytes("UTF-8")));
//if (effectiveHashKey < startingHashKey || effectiveHashKey > endingHashKey )
//{
// for (int toRemove = 0; toRemove < recordsInCurrRecord; ++toRemove)
// result.RemoveAt(result.Count - 1);
// break;
//}
++recordsInCurrRecord;
Amazon.Kinesis.Model.Record recd = new Amazon.Kinesis.Model.Record()
{
Data = new MemoryStream(mr.Data.ToByteArray()),
PartitionKey = partitionKey,
SequenceNumber = record.SequenceNumber,
ApproximateArrivalTimestamp = aat
};
result.Add(new UserRecord(true, recd, subSeqNum++, explicitHashKey));
}
}
catch (Exception e)
{
StringBuilder sb = new StringBuilder();
sb.Append("Unexpected exception during deaggregation, record was:\n");
sb.Append("PKS:\n");
foreach (string s in pks)
{
sb.Append(s).Append("\n");
}
sb.Append("EHKS: \n");
foreach (string s in ehks)
{
sb.Append(s).Append("\n");
}
foreach (Aws.Kinesis.Protobuf.Record mr in ar.Records)
{
sb.Append("Record: [hasEhk=").Append(mr.ExplicitHashKeyIndex >= 0 ? "0" : "1").Append(", ")
.Append("ehkIdx=").Append(mr.ExplicitHashKeyIndex).Append(", ")
.Append("pkIdx=").Append(mr.PartitionKeyIndex).Append(", ")
.Append("dataLen=").Append(mr.Data.ToByteArray().Length).Append("]\n");
}
sb.Append("Sequence number: ").Append(record.SequenceNumber).Append("\n")
.Append("Raw data: ")
.Append(serializedData).Append("\n");
errorMesaage += ("\n" + new Exception(sb.ToString(), e).ToString());
}
}
catch (InvalidProtocolBufferException e)
{
aggregated = false;
}
}
}
if (!aggregated)
{
result.Add(new UserRecord(record));
}
}
return(result);
}
