public override async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient, DateTime?lastReadTime = null, TaskCompletionSource <DateTime>?tcs = null, bool isDiscoverRead = false) { var response = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}"); var recordsList = JsonConvert.DeserializeObject <SuppressionListsResponse>(await response.Content.ReadAsStringAsync()); if (recordsList.SuppressionLists == null) { yield break; } foreach (var recordMap in recordsList.SuppressionLists) { var normalizedRecordMap = new Dictionary <string, object?>(); foreach (var kv in recordMap) { if (kv.Key.Equals(EndpointHelper.LinksPropertyId)) { continue; } normalizedRecordMap.TryAdd(kv.Key, kv.Value); } yield return(new Record { Action = Record.Types.Action.Upsert, DataJson = JsonConvert.SerializeObject(normalizedRecordMap) }); } }
public virtual async Task <Count> GetCountOfRecords(IApiClient apiClient) { var response = await apiClient.GetAsync($"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}"); var recordsList = JsonConvert.DeserializeObject <DataWrapper>(await response.Content.ReadAsStringAsync()); return(new Count { Kind = Count.Types.Kind.Exact, Value = (int)recordsList.TotalRecords }); }
private async Task PreLoadLookup(IApiClient apiClient) { var readResponse = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}"); var recordsList = JsonConvert.DeserializeObject <ListsResponse>(await readResponse.Content.ReadAsStringAsync()); foreach (var list in recordsList.Lists) { NameToListIdDictionary.TryAdd(list["Name"].ToString() ?? "UNKNOWN_LIST", (long)list["ListID"]); } }
public virtual async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient, Schema schema, bool isDiscoverRead = false) { var after = ""; var hasMore = false; do { var response = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}?limit=100&properties={string.Join(",",schema.Properties.Select(p => p.Id))}{(string.IsNullOrWhiteSpace(after) ? "" : $"&after={after}")}"); response.EnsureSuccessStatusCode(); var objectResponseWrapper = JsonConvert.DeserializeObject <ObjectResponseWrapper>(await response.Content.ReadAsStringAsync()); after = objectResponseWrapper?.Paging?.Next?.After ?? ""; hasMore = !string.IsNullOrWhiteSpace(after); if (objectResponseWrapper?.Results.Count == 0) { yield break; } foreach (var objectResponse in objectResponseWrapper?.Results) { var recordMap = new Dictionary <string, object>(); foreach (var objectProperty in objectResponse.Properties) { try { recordMap[objectProperty.Key] = objectProperty.Value.ToString() ?? ""; } catch { recordMap[objectProperty.Key] = ""; } } yield return(new Record { Action = Record.Types.Action.Upsert, DataJson = JsonConvert.SerializeObject(recordMap) }); } } while (hasMore);
public virtual async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient, DateTime?lastReadTime = null, TaskCompletionSource <DateTime>?tcs = null, bool isDiscoverRead = false) { long pageNumber = 1; long maxPageNumber; DateTime tcsDateTime; do { var response = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}?PageNumber={pageNumber}{(lastReadTime.HasValue ? $"&Since={lastReadTime.Value.ToUniversalTime():O}" : "")}"); Logger.Debug($"Date Header value: {response.Headers.Date}"); tcsDateTime = response.Headers.Date?.UtcDateTime ?? DateTime.UtcNow; var recordsList = JsonConvert.DeserializeObject <DataWrapper>(await response.Content.ReadAsStringAsync()); maxPageNumber = recordsList.TotalPages; if (recordsList.Items == null) { yield break; } foreach (var recordMap in recordsList.Items) { var normalizedRecordMap = new Dictionary <string, object?>(); foreach (var kv in recordMap) { if ( !string.IsNullOrWhiteSpace(DetailPath) && !string.IsNullOrWhiteSpace(DetailPropertyId) && kv.Key.Equals(DetailPropertyId) && kv.Value != null) { var detailResponse = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{DetailPath.TrimStart('/')}/{kv.Value}"); var detailsRecord = JsonConvert.DeserializeObject <Dictionary <string, object> >( await detailResponse.Content.ReadAsStringAsync()); foreach (var detailKv in detailsRecord) { if (detailKv.Key.Equals(EndpointHelper.CustomFieldsId) && detailKv.Value != null) { var customFields = JsonConvert.DeserializeObject <List <CustomField> >( JsonConvert.SerializeObject(detailKv.Value)); foreach (var cf in customFields) { normalizedRecordMap.TryAdd(cf.FieldName, cf.Value); } continue; } if (detailKv.Key.Equals(EndpointHelper.LinksPropertyId)) { continue; } normalizedRecordMap.TryAdd(detailKv.Key, detailKv.Value); } continue; } if (kv.Key.Equals(EndpointHelper.CustomFieldsId) && kv.Value != null) { var customFields = JsonConvert.DeserializeObject <List <CustomField> >( JsonConvert.SerializeObject(kv.Value)); foreach (var cf in customFields) { normalizedRecordMap.TryAdd(cf.FieldName, cf.Value); } continue; } if (kv.Key.Equals(EndpointHelper.LinksPropertyId)) { continue; } normalizedRecordMap.TryAdd(kv.Key, kv.Value); } yield return(new Record { Action = Record.Types.Action.Upsert, DataJson = JsonConvert.SerializeObject(normalizedRecordMap) }); } pageNumber++; } while (pageNumber <= maxPageNumber); if (tcs != null) { Logger.Debug($"Setting tcs with value {tcsDateTime.ToUniversalTime():O}"); tcs.SetResult(tcsDateTime); } }