public virtual async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient,
DateTime?lastReadTime = null, TaskCompletionSource <DateTime>?tcs = null, bool isDiscoverRead = false)
{
long pageNumber = 1;
long maxPageNumber;
DateTime tcsDateTime;
do
{
var response = await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}?PageNumber={pageNumber}{(lastReadTime.HasValue ? $"&Since={lastReadTime.Value.ToUniversalTime():O}" : "")}");
Logger.Debug($"Date Header value: {response.Headers.Date}");
tcsDateTime = response.Headers.Date?.UtcDateTime ?? DateTime.UtcNow;
var recordsList =
JsonConvert.DeserializeObject <DataWrapper>(await response.Content.ReadAsStringAsync());
maxPageNumber = recordsList.TotalPages;
if (recordsList.Items == null)
{
yield break;
}
foreach (var recordMap in recordsList.Items)
{
var normalizedRecordMap = new Dictionary <string, object?>();
foreach (var kv in recordMap)
{
if (
!string.IsNullOrWhiteSpace(DetailPath) &&
!string.IsNullOrWhiteSpace(DetailPropertyId) &&
kv.Key.Equals(DetailPropertyId) && kv.Value != null)
{
var detailResponse =
await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{DetailPath.TrimStart('/')}/{kv.Value}");
var detailsRecord =
JsonConvert.DeserializeObject <Dictionary <string, object> >(
await detailResponse.Content.ReadAsStringAsync());
foreach (var detailKv in detailsRecord)
{
if (detailKv.Key.Equals(EndpointHelper.CustomFieldsId) && detailKv.Value != null)
{
var customFields =
JsonConvert.DeserializeObject <List <CustomField> >(
JsonConvert.SerializeObject(detailKv.Value));
foreach (var cf in customFields)
{
normalizedRecordMap.TryAdd(cf.FieldName, cf.Value);
}
continue;
}
if (detailKv.Key.Equals(EndpointHelper.LinksPropertyId))
{
continue;
}
normalizedRecordMap.TryAdd(detailKv.Key, detailKv.Value);
}
continue;
}
if (kv.Key.Equals(EndpointHelper.CustomFieldsId) && kv.Value != null)
{
var customFields =
JsonConvert.DeserializeObject <List <CustomField> >(
JsonConvert.SerializeObject(kv.Value));
foreach (var cf in customFields)
{
normalizedRecordMap.TryAdd(cf.FieldName, cf.Value);
}
continue;
}
if (kv.Key.Equals(EndpointHelper.LinksPropertyId))
{
continue;
}
normalizedRecordMap.TryAdd(kv.Key, kv.Value);
}
yield return(new Record
{
Action = Record.Types.Action.Upsert,
DataJson = JsonConvert.SerializeObject(normalizedRecordMap)
});
}
pageNumber++;
} while (pageNumber <= maxPageNumber);
if (tcs != null)
{
Logger.Debug($"Setting tcs with value {tcsDateTime.ToUniversalTime():O}");
tcs.SetResult(tcsDateTime);
}
}
public override async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient,
DateTime?lastReadTime = null, TaskCompletionSource <DateTime>?tcs = null, bool isDiscoverRead = false)
{
var response = await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}");
var recordsList =
JsonConvert.DeserializeObject <FiltersResponse>(await response.Content.ReadAsStringAsync());
if (recordsList.Filters == null)
{
yield break;
}
foreach (var recordMap in recordsList.Filters)
{
var normalizedRecordMap = new Dictionary <string, object?>();
foreach (var kv in recordMap)
{
if (
!string.IsNullOrWhiteSpace(DetailPath) &&
!string.IsNullOrWhiteSpace(DetailPropertyId) &&
kv.Key.Equals(DetailPropertyId) && kv.Value != null)
{
var detailResponse =
await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{DetailPath.TrimStart('/')}/{kv.Value}");
var detailsRecord =
JsonConvert.DeserializeObject <Dictionary <string, object> >(
await detailResponse.Content.ReadAsStringAsync());
foreach (var detailKv in detailsRecord)
{
if (detailKv.Key.Equals(EndpointHelper.LinksPropertyId))
{
continue;
}
normalizedRecordMap.TryAdd(detailKv.Key, detailKv.Value);
}
continue;
}
if (kv.Key.Equals(EndpointHelper.LinksPropertyId))
{
continue;
}
normalizedRecordMap.TryAdd(kv.Key, kv.Value);
}
yield return(new Record
{
Action = Record.Types.Action.Upsert,
DataJson = JsonConvert.SerializeObject(normalizedRecordMap)
});
}
}
public override async Task <string> WriteRecordAsync(IApiClient apiClient, Schema schema, Record record,
IServerStreamWriter <RecordAck> responseStream)
{
var recordMap = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson);
foreach (var requiredPropertyId in RequiredWritePropertyIds)
{
if (!recordMap.ContainsKey(requiredPropertyId))
{
var errorMessage = $"Record did not contain required property {requiredPropertyId}";
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
if (recordMap.ContainsKey(requiredPropertyId) && recordMap[requiredPropertyId] == null)
{
var errorMessage = $"Required property {requiredPropertyId} was NULL";
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
}
// get live object
Dictionary <string, object> liveRecord = new Dictionary <string, object>();
var getResponse = await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{recordMap[DetailPropertyId]}/{DetailPath.TrimStart('/')}");
if (getResponse.IsSuccessStatusCode)
{
liveRecord =
JsonConvert.DeserializeObject <Dictionary <string, object> >(await getResponse.Content.ReadAsStringAsync());
}
var putObject = new Dictionary <string, object>();
var customFieldObject = new List <CustomField>();
foreach (var property in schema.Properties)
{
if (property.TypeAtSource == Constants.CustomProperty)
{
var customField = new CustomField
{
FieldName = property.Id,
Value = null
};
if (recordMap.ContainsKey(property.Id))
{
customField.Value = recordMap[property.Id];
}
customFieldObject.Add(customField);
}
else
{
object value = null;
if (recordMap.ContainsKey(property.Id))
{
value = recordMap[property.Id];
if (property.Type == PropertyType.Json)
{
if (value is string s)
{
value = JsonConvert.DeserializeObject(s) ?? null;
}
}
}
putObject.Add(property.Id, value);
}
}
putObject.Add("CustomFields", customFieldObject);
if (putObject.ContainsKey("Lists") && putObject["Lists"] != null)
{
JArray j = (JArray)putObject["Lists"];
putObject["Lists"] = j.Select(x => x["ListID"]).ToList();
}
else
{
JArray j = (JArray)liveRecord["Lists"];
putObject["Lists"] = j.Select(x => x["ListID"]).ToList();
}
if (putObject.ContainsKey("Publications") && putObject["Publications"] != null)
{
JArray j = (JArray)putObject["Publications"];
putObject["Publications"] = j.Select(x => x["PublicationID"]).ToList();
}
else
{
JArray j = (JArray)liveRecord["Publications"];
putObject["Publications"] = j.Select(x => x["PublicationID"]).ToList();
}
if (putObject.ContainsKey("Source") && putObject["Source"] != null)
{
JObject j = (JObject)putObject["Source"];
if (j.ContainsKey("SourceID"))
{
putObject["SourceID"] = j["SourceID"];
}
putObject.Remove("Source");
}
else
{
JObject j = (JObject)liveRecord["Source"];
if (j.ContainsKey("SourceID"))
{
putObject["SourceID"] = j["SourceID"];
}
putObject.Remove("Source");
}
var json = new StringContent(
JsonConvert.SerializeObject(putObject),
Encoding.UTF8,
"application/json"
);
var response =
await apiClient.PutAsync($"{BasePath.TrimEnd('/')}/{recordMap[WritePathPropertyId]}", json);
if (response.StatusCode == HttpStatusCode.NotFound)
{
response =
await apiClient.PostAsync($"{BasePath.TrimEnd('/')}", json);
}
if (!response.IsSuccessStatusCode)
{
var errorMessage = await response.Content.ReadAsStringAsync();
var errorAck = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = errorMessage
};
await responseStream.WriteAsync(errorAck);
return(errorMessage);
}
var ack = new RecordAck
{
CorrelationId = record.CorrelationId,
Error = ""
};
await responseStream.WriteAsync(ack);
return("");
}
