public override async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient,
DateTime?lastReadTime = null, TaskCompletionSource <DateTime>?tcs = null, bool isDiscoverRead = false)
{
var response = await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}");
var recordsList =
JsonConvert.DeserializeObject <SuppressionListsResponse>(await response.Content.ReadAsStringAsync());
if (recordsList.SuppressionLists == null)
{
yield break;
}
foreach (var recordMap in recordsList.SuppressionLists)
{
var normalizedRecordMap = new Dictionary <string, object?>();
foreach (var kv in recordMap)
{
if (kv.Key.Equals(EndpointHelper.LinksPropertyId))
{
continue;
}
normalizedRecordMap.TryAdd(kv.Key, kv.Value);
}
yield return(new Record
{
Action = Record.Types.Action.Upsert,
DataJson = JsonConvert.SerializeObject(normalizedRecordMap)
});
}
}
public virtual async Task <Count> GetCountOfRecords(IApiClient apiClient)
{
var response = await apiClient.GetAsync($"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}");
var recordsList = JsonConvert.DeserializeObject <DataWrapper>(await response.Content.ReadAsStringAsync());
return(new Count
{
Kind = Count.Types.Kind.Exact,
Value = (int)recordsList.TotalRecords
});
}
private async Task PreLoadLookup(IApiClient apiClient)
{
var readResponse = await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}");
var recordsList =
JsonConvert.DeserializeObject <ListsResponse>(await readResponse.Content.ReadAsStringAsync());
foreach (var list in recordsList.Lists)
{
NameToListIdDictionary.TryAdd(list["Name"].ToString() ?? "UNKNOWN_LIST", (long)list["ListID"]);
}
}
public virtual async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient, Schema schema, bool isDiscoverRead = false)
{
var after = "";
var hasMore = false;
do
{
var response = await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}?limit=100&properties={string.Join(",",schema.Properties.Select(p => p.Id))}{(string.IsNullOrWhiteSpace(after) ? "" : $"&after={after}")}");
response.EnsureSuccessStatusCode();
var objectResponseWrapper =
JsonConvert.DeserializeObject <ObjectResponseWrapper>(await response.Content.ReadAsStringAsync());
after = objectResponseWrapper?.Paging?.Next?.After ?? "";
hasMore = !string.IsNullOrWhiteSpace(after);
if (objectResponseWrapper?.Results.Count == 0)
{
yield break;
}
foreach (var objectResponse in objectResponseWrapper?.Results)
{
var recordMap = new Dictionary <string, object>();
foreach (var objectProperty in objectResponse.Properties)
{
try
{
recordMap[objectProperty.Key] = objectProperty.Value.ToString() ?? "";
}
catch
{
recordMap[objectProperty.Key] = "";
}
}
yield return(new Record
{
Action = Record.Types.Action.Upsert,
DataJson = JsonConvert.SerializeObject(recordMap)
});
}
} while (hasMore);
public virtual async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient,
DateTime?lastReadTime = null, TaskCompletionSource <DateTime>?tcs = null, bool isDiscoverRead = false)
{
long pageNumber = 1;
long maxPageNumber;
DateTime tcsDateTime;
do
{
var response = await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}?PageNumber={pageNumber}{(lastReadTime.HasValue ? $"&Since={lastReadTime.Value.ToUniversalTime():O}" : "")}");
Logger.Debug($"Date Header value: {response.Headers.Date}");
tcsDateTime = response.Headers.Date?.UtcDateTime ?? DateTime.UtcNow;
var recordsList =
JsonConvert.DeserializeObject <DataWrapper>(await response.Content.ReadAsStringAsync());
maxPageNumber = recordsList.TotalPages;
if (recordsList.Items == null)
{
yield break;
}
foreach (var recordMap in recordsList.Items)
{
var normalizedRecordMap = new Dictionary <string, object?>();
foreach (var kv in recordMap)
{
if (
!string.IsNullOrWhiteSpace(DetailPath) &&
!string.IsNullOrWhiteSpace(DetailPropertyId) &&
kv.Key.Equals(DetailPropertyId) && kv.Value != null)
{
var detailResponse =
await apiClient.GetAsync(
$"{BasePath.TrimEnd('/')}/{DetailPath.TrimStart('/')}/{kv.Value}");
var detailsRecord =
JsonConvert.DeserializeObject <Dictionary <string, object> >(
await detailResponse.Content.ReadAsStringAsync());
foreach (var detailKv in detailsRecord)
{
if (detailKv.Key.Equals(EndpointHelper.CustomFieldsId) && detailKv.Value != null)
{
var customFields =
JsonConvert.DeserializeObject <List <CustomField> >(
JsonConvert.SerializeObject(detailKv.Value));
foreach (var cf in customFields)
{
normalizedRecordMap.TryAdd(cf.FieldName, cf.Value);
}
continue;
}
if (detailKv.Key.Equals(EndpointHelper.LinksPropertyId))
{
continue;
}
normalizedRecordMap.TryAdd(detailKv.Key, detailKv.Value);
}
continue;
}
if (kv.Key.Equals(EndpointHelper.CustomFieldsId) && kv.Value != null)
{
var customFields =
JsonConvert.DeserializeObject <List <CustomField> >(
JsonConvert.SerializeObject(kv.Value));
foreach (var cf in customFields)
{
normalizedRecordMap.TryAdd(cf.FieldName, cf.Value);
}
continue;
}
if (kv.Key.Equals(EndpointHelper.LinksPropertyId))
{
continue;
}
normalizedRecordMap.TryAdd(kv.Key, kv.Value);
}
yield return(new Record
{
Action = Record.Types.Action.Upsert,
DataJson = JsonConvert.SerializeObject(normalizedRecordMap)
});
}
pageNumber++;
} while (pageNumber <= maxPageNumber);
if (tcs != null)
{
Logger.Debug($"Setting tcs with value {tcsDateTime.ToUniversalTime():O}");
tcs.SetResult(tcsDateTime);
}
}
