public virtual async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient, DateTime?lastReadTime = null, TaskCompletionSource <DateTime>?tcs = null, bool isDiscoverRead = false) { long pageNumber = 1; long maxPageNumber; DateTime tcsDateTime; do { var response = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}?PageNumber={pageNumber}{(lastReadTime.HasValue ? $"&Since={lastReadTime.Value.ToUniversalTime():O}" : "")}"); Logger.Debug($"Date Header value: {response.Headers.Date}"); tcsDateTime = response.Headers.Date?.UtcDateTime ?? DateTime.UtcNow; var recordsList = JsonConvert.DeserializeObject <DataWrapper>(await response.Content.ReadAsStringAsync()); maxPageNumber = recordsList.TotalPages; if (recordsList.Items == null) { yield break; } foreach (var recordMap in recordsList.Items) { var normalizedRecordMap = new Dictionary <string, object?>(); foreach (var kv in recordMap) { if ( !string.IsNullOrWhiteSpace(DetailPath) && !string.IsNullOrWhiteSpace(DetailPropertyId) && kv.Key.Equals(DetailPropertyId) && kv.Value != null) { var detailResponse = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{DetailPath.TrimStart('/')}/{kv.Value}"); var detailsRecord = JsonConvert.DeserializeObject <Dictionary <string, object> >( await detailResponse.Content.ReadAsStringAsync()); foreach (var detailKv in detailsRecord) { if (detailKv.Key.Equals(EndpointHelper.CustomFieldsId) && detailKv.Value != null) { var customFields = JsonConvert.DeserializeObject <List <CustomField> >( JsonConvert.SerializeObject(detailKv.Value)); foreach (var cf in customFields) { normalizedRecordMap.TryAdd(cf.FieldName, cf.Value); } continue; } if (detailKv.Key.Equals(EndpointHelper.LinksPropertyId)) { continue; } normalizedRecordMap.TryAdd(detailKv.Key, detailKv.Value); } continue; } if (kv.Key.Equals(EndpointHelper.CustomFieldsId) && kv.Value != null) { var customFields = JsonConvert.DeserializeObject <List <CustomField> >( JsonConvert.SerializeObject(kv.Value)); foreach (var cf in customFields) { normalizedRecordMap.TryAdd(cf.FieldName, cf.Value); } continue; } if (kv.Key.Equals(EndpointHelper.LinksPropertyId)) { continue; } normalizedRecordMap.TryAdd(kv.Key, kv.Value); } yield return(new Record { Action = Record.Types.Action.Upsert, DataJson = JsonConvert.SerializeObject(normalizedRecordMap) }); } pageNumber++; } while (pageNumber <= maxPageNumber); if (tcs != null) { Logger.Debug($"Setting tcs with value {tcsDateTime.ToUniversalTime():O}"); tcs.SetResult(tcsDateTime); } }
public override async IAsyncEnumerable <Record> ReadRecordsAsync(IApiClient apiClient, DateTime?lastReadTime = null, TaskCompletionSource <DateTime>?tcs = null, bool isDiscoverRead = false) { var response = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{AllPath.TrimStart('/')}"); var recordsList = JsonConvert.DeserializeObject <FiltersResponse>(await response.Content.ReadAsStringAsync()); if (recordsList.Filters == null) { yield break; } foreach (var recordMap in recordsList.Filters) { var normalizedRecordMap = new Dictionary <string, object?>(); foreach (var kv in recordMap) { if ( !string.IsNullOrWhiteSpace(DetailPath) && !string.IsNullOrWhiteSpace(DetailPropertyId) && kv.Key.Equals(DetailPropertyId) && kv.Value != null) { var detailResponse = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{DetailPath.TrimStart('/')}/{kv.Value}"); var detailsRecord = JsonConvert.DeserializeObject <Dictionary <string, object> >( await detailResponse.Content.ReadAsStringAsync()); foreach (var detailKv in detailsRecord) { if (detailKv.Key.Equals(EndpointHelper.LinksPropertyId)) { continue; } normalizedRecordMap.TryAdd(detailKv.Key, detailKv.Value); } continue; } if (kv.Key.Equals(EndpointHelper.LinksPropertyId)) { continue; } normalizedRecordMap.TryAdd(kv.Key, kv.Value); } yield return(new Record { Action = Record.Types.Action.Upsert, DataJson = JsonConvert.SerializeObject(normalizedRecordMap) }); } }
public override async Task <string> WriteRecordAsync(IApiClient apiClient, Schema schema, Record record, IServerStreamWriter <RecordAck> responseStream) { var recordMap = JsonConvert.DeserializeObject <Dictionary <string, object> >(record.DataJson); foreach (var requiredPropertyId in RequiredWritePropertyIds) { if (!recordMap.ContainsKey(requiredPropertyId)) { var errorMessage = $"Record did not contain required property {requiredPropertyId}"; var errorAck = new RecordAck { CorrelationId = record.CorrelationId, Error = errorMessage }; await responseStream.WriteAsync(errorAck); return(errorMessage); } if (recordMap.ContainsKey(requiredPropertyId) && recordMap[requiredPropertyId] == null) { var errorMessage = $"Required property {requiredPropertyId} was NULL"; var errorAck = new RecordAck { CorrelationId = record.CorrelationId, Error = errorMessage }; await responseStream.WriteAsync(errorAck); return(errorMessage); } } // get live object Dictionary <string, object> liveRecord = new Dictionary <string, object>(); var getResponse = await apiClient.GetAsync( $"{BasePath.TrimEnd('/')}/{recordMap[DetailPropertyId]}/{DetailPath.TrimStart('/')}"); if (getResponse.IsSuccessStatusCode) { liveRecord = JsonConvert.DeserializeObject <Dictionary <string, object> >(await getResponse.Content.ReadAsStringAsync()); } var putObject = new Dictionary <string, object>(); var customFieldObject = new List <CustomField>(); foreach (var property in schema.Properties) { if (property.TypeAtSource == Constants.CustomProperty) { var customField = new CustomField { FieldName = property.Id, Value = null }; if (recordMap.ContainsKey(property.Id)) { customField.Value = recordMap[property.Id]; } customFieldObject.Add(customField); } else { object value = null; if (recordMap.ContainsKey(property.Id)) { value = recordMap[property.Id]; if (property.Type == PropertyType.Json) { if (value is string s) { value = JsonConvert.DeserializeObject(s) ?? null; } } } putObject.Add(property.Id, value); } } putObject.Add("CustomFields", customFieldObject); if (putObject.ContainsKey("Lists") && putObject["Lists"] != null) { JArray j = (JArray)putObject["Lists"]; putObject["Lists"] = j.Select(x => x["ListID"]).ToList(); } else { JArray j = (JArray)liveRecord["Lists"]; putObject["Lists"] = j.Select(x => x["ListID"]).ToList(); } if (putObject.ContainsKey("Publications") && putObject["Publications"] != null) { JArray j = (JArray)putObject["Publications"]; putObject["Publications"] = j.Select(x => x["PublicationID"]).ToList(); } else { JArray j = (JArray)liveRecord["Publications"]; putObject["Publications"] = j.Select(x => x["PublicationID"]).ToList(); } if (putObject.ContainsKey("Source") && putObject["Source"] != null) { JObject j = (JObject)putObject["Source"]; if (j.ContainsKey("SourceID")) { putObject["SourceID"] = j["SourceID"]; } putObject.Remove("Source"); } else { JObject j = (JObject)liveRecord["Source"]; if (j.ContainsKey("SourceID")) { putObject["SourceID"] = j["SourceID"]; } putObject.Remove("Source"); } var json = new StringContent( JsonConvert.SerializeObject(putObject), Encoding.UTF8, "application/json" ); var response = await apiClient.PutAsync($"{BasePath.TrimEnd('/')}/{recordMap[WritePathPropertyId]}", json); if (response.StatusCode == HttpStatusCode.NotFound) { response = await apiClient.PostAsync($"{BasePath.TrimEnd('/')}", json); } if (!response.IsSuccessStatusCode) { var errorMessage = await response.Content.ReadAsStringAsync(); var errorAck = new RecordAck { CorrelationId = record.CorrelationId, Error = errorMessage }; await responseStream.WriteAsync(errorAck); return(errorMessage); } var ack = new RecordAck { CorrelationId = record.CorrelationId, Error = "" }; await responseStream.WriteAsync(ack); return(""); }