private void UpdateScannerFields(ScanRequest rpc, ScanResponsePB response) { var scanTimestamp = response.HasSnapTimestamp ? (long)response.SnapTimestamp : KuduClient.NoTimestamp; var propagatedTimestamp = response.HasPropagatedTimestamp ? (long)response.PropagatedTimestamp : KuduClient.NoTimestamp; if (SnapshotTimestamp == KuduClient.NoTimestamp && scanTimestamp != KuduClient.NoTimestamp) { // If the server-assigned timestamp is present in the tablet // server's response, store it in the scanner. The stored value // is used for read operations in READ_AT_SNAPSHOT mode at // other tablet servers in the context of the same scan. SnapshotTimestamp = scanTimestamp; } long lastPropagatedTimestamp = KuduClient.NoTimestamp; if (_readMode == ReadMode.ReadYourWrites && scanTimestamp != KuduClient.NoTimestamp) { // For READ_YOUR_WRITES mode, update the latest propagated timestamp // with the chosen snapshot timestamp sent back from the server, to // avoid unnecessarily wait for subsequent reads. Since as long as // the chosen snapshot timestamp of the next read is greater than // the previous one, the scan does not violate READ_YOUR_WRITES // session guarantees. lastPropagatedTimestamp = scanTimestamp; } else if (propagatedTimestamp != KuduClient.NoTimestamp) { // Otherwise we just use the propagated timestamp returned from // the server as the latest propagated timestamp. lastPropagatedTimestamp = propagatedTimestamp; } if (lastPropagatedTimestamp != KuduClient.NoTimestamp) { _client.LastPropagatedTimestamp = lastPropagatedTimestamp; } if (_isFaultTolerant && response.HasLastPrimaryKey) { _lastPrimaryKey = response.LastPrimaryKey; } if (response.ResourceMetrics is not null) { ResourceMetrics.Update(response.ResourceMetrics); } Tablet = rpc.Tablet; Current = rpc.TakeResultSet(); _numRowsReturned += Current.Count; }
private async Task <ScanResponse> ScanAsync(ScanRequest queryRequest) { var response = await _dynamoDbClient.ScanAsync(queryRequest); Console.WriteLine(response.ResponseMetadata); return(response); }
public async Task <List <T> > List <T>(Expression <Func <T, bool> > expression = null) where T : Base, new() { var generic = new T(); var tableName = generic.GetTableName(); var expressionAttributeNames = generic.GetExpressionAttributes(); var dynamoDbRequest = new ScanRequest { TableName = tableName, ProjectionExpression = string.Join(", ", expressionAttributeNames.Keys), ExpressionAttributeNames = expressionAttributeNames }; if (expression != null) { var expressionString = new StringBuilder(); dynamoDbRequest.ExpressionAttributeValues = Converters.ExpressionValues.ConvertExpressionValues(expression, ref expressionString); dynamoDbRequest.FilterExpression = expressionString.ToString(); } return((await amazonDynamoDB.ScanAsync(dynamoDbRequest)) .Items .Select(x => x.Map <T>()) .ToList()); }
public static async void FindProductsForPriceLessThanZero(AmazonDynamoDBClient client) { Dictionary <string, AttributeValue> lastKeyEvaluated = null; do { var request = new ScanRequest { TableName = "ProductCatalog", Limit = 2, ExclusiveStartKey = lastKeyEvaluated, ExpressionAttributeValues = new Dictionary <string, AttributeValue> { { ":val", new AttributeValue { N = "0" } } }, FilterExpression = "Price < :val", ProjectionExpression = "Id, Title, Price" }; var response = await client.ScanAsync(request); foreach (Dictionary <string, AttributeValue> item in response.Items) { Console.WriteLine("\nScanThreadTableUsePaging - printing....."); PrintItem(item); } lastKeyEvaluated = response.LastEvaluatedKey; } while (lastKeyEvaluated != null && lastKeyEvaluated.Count != 0); }
public async Task <List <DBRecord> > GetRecordsAsync(string filterColumn, string searchedNode) { Console.WriteLine($"GetItemAsync: search for {searchedNode} as {filterColumn}"); var list = new List <DBRecord>(); var request = new ScanRequest { TableName = tableName, ExpressionAttributeValues = new Dictionary <string, AttributeValue> { { ":val", new AttributeValue { S = searchedNode } } }, FilterExpression = filterColumn + "= :val", ProjectionExpression = "Id, SourceProject, TargetProject" // when delete, needs Id }; var response = await dbClient.ScanAsync(request); Console.WriteLine($"Already Existed Dependency in DB: Scan result for project {searchedNode} as {filterColumn}"); foreach (Dictionary <string, AttributeValue> item in response.Items) { Console.WriteLine($"{item["Id"].S} {item["SourceProject"].S} -> {item["TargetProject"].S}"); list.Add(new DBRecord { Id = item["Id"].S, Source = item["SourceProject"].S, Target = item["TargetProject"].S }); } return(list); }
public Response GetAllDataInDynamoDb() { try { if (null == Table) { return(new Response(false, "Table not Found")); } var request = new ScanRequest { TableName = "Affiliates", }; ScanResponse response = AmazonDynamoDBClientConnection.Client.Scan(request); List <AffiliateDataModel> identityList = CovertresponseIntoJSON(response); if (null == identityList) { new Response(false, "Identity no more exist in database"); } return(new Response(true, "Found Element", identityList)); } catch (Exception exception) { Console.WriteLine(exception.Message); return(new Response(false, exception.Message)); } }
public async Task <IEnumerable <UserModel> > GetUsersByName(string name) { var condition = new Condition { ComparisonOperator = ComparisonOperator.CONTAINS, AttributeValueList = new List <AttributeValue> { new AttributeValue { S = name } } }; var scanRequest = new ScanRequest { TableName = Constants.UserTableName, ConditionalOperator = ConditionalOperator.OR, ScanFilter = new Dictionary <string, Condition> { { "firstName", condition }, { "lastName", condition } } }; var results = await _amazonDynamoDB.ScanAsync(scanRequest); var userDaos = results.Items.Select(x => x.MapSimpleResponse <User>()); return(userDaos.Select(x => x.ToUserModel())); }
public string FunctionHandler(ILambdaContext context) { var request = new ScanRequest(new ClassificationModel().GetTable()) { FilterExpression = "attribute_not_exists(orientation)" }; ScanResponse response = null; do { if (response != null) { request.ExclusiveStartKey = response.LastEvaluatedKey; } response = DbClient.ScanAsync(request).Result; Parallel.ForEach(response.Items, new ParallelOptions { MaxDegreeOfParallelism = 2 }, item => { var modelJson = Document.FromAttributeMap(item).ToJson(); var classification = JsonConvert.DeserializeObject <ClassificationModel>(modelJson); try { Process(classification).Wait(); } catch (Exception e) { Console.WriteLine($"Failed to re-process source: {classification.Source} page id: {classification.PageId}: {e}"); } }); } while (response.LastEvaluatedKey.Any()); return("Finished re-processing all records without orientation."); }
public List <Employee> List(int page_size, int page) { ScanResponse returnIWant = null; string KeyToStart = string.Empty; Dictionary <string, AttributeValue> lastKey = null; for (int p = 1; p <= page; p++) { var query = new ScanRequest() { TableName = "employee", Limit = page_size, ExclusiveStartKey = lastKey != null? lastKey : null, }; returnIWant = dynamodbClient.ScanAsync(query).Result; lastKey = returnIWant.LastEvaluatedKey; } var employees = returnIWant?.Items.ConvertAll(new Converter <Dictionary <string, AttributeValue>, Employee>(itemRetorno => { return(new Employee() { name = itemRetorno["name"].S, department = itemRetorno["department"].S, email = itemRetorno["email"].S, id = long.Parse(itemRetorno["id"].S) }); })); return(employees); }
public async Task <IActionResult> ScanWebsite(ScanWebsiteJson obj) { try { Uri uriResult; if (Uri.TryCreate(obj.WebsiteUrl, UriKind.Absolute, out uriResult) && (uriResult.Scheme == Uri.UriSchemeHttp || uriResult.Scheme == Uri.UriSchemeHttps)) { string userId = User.Claims.First(c => c.Type == "UserID").Value; var user = await _userManager.FindByIdAsync(userId); var request = new ScanRequest() { RequestDate = DateTime.Now, WebsiteUrl = obj.WebsiteUrl, WebsiteFtp = obj.WebsiteFtp, FtpUsername = obj.FtpUsername, FtpPassword = obj.FtpPassword, UserId = user.Id }; _repository.CreateScanRequest(request); Response.StatusCode = 200; return(new EmptyResult()); } else { return(BadRequest()); } } catch (Exception e) { _logger.LogError(e, "Exception in SecSoulController, exception: "); return(BadRequest(new { message = "New scan request was unsuccessful, please try again later" })); } }
public async Task <bool> ContainsItemMatchingRoom(Classtime classtime) { ScanRequest scanRequest = new ScanRequest() { TableName = tableName, Limit = 5, ExpressionAttributeValues = new Dictionary <string, AttributeValue>() { { ":v_" + ClasstimeAsString.building, new AttributeValue() { S = classtime.building } }, { ":v_" + ClasstimeAsString.roomNumber, new AttributeValue() { S = classtime.roomNumber } } }, FilterExpression = ClasstimeAsString.building + " = " + ":v_" + ClasstimeAsString.building + " and " + ClasstimeAsString.roomNumber + " = " + ":v_" + ClasstimeAsString.roomNumber }; ScanResponse scanResponse = await dynamodbClient.ScanAsync(scanRequest); return(scanResponse.Count > 0); }
public async Task <Int32> CountStatements() { var request = new ScanRequest { TableName = "WisdomOfAvasarala", ExpressionAttributeValues = new Dictionary <string, AttributeValue> { { ":qt", new AttributeValue { S = "1" } } }, FilterExpression = "statement = :qt" //FilterExpression = "quoteType <> :qt" }; var response = await _client.ScanAsync(request); LambdaLogger.Log($"Consumed Capacity: {response.ConsumedCapacity}\n"); LambdaLogger.Log($"Content Length: {response.ContentLength}\n"); LambdaLogger.Log($"Count: {response.Count}\n"); LambdaLogger.Log($"Scanned Count: {response.ScannedCount}\n"); LambdaLogger.Log($"Items Count: {response.Items.Count}\n"); return(response.Items.Count); }
public Response GetAllDataInDynamoDb() { try { if (null == Table) { return(new Response(false, "Table not Found")); } var request = new ScanRequest { TableName = "Position", }; var response = AmazonDynamoDBClientConnection.Client.Scan(request); List <Dictionary <string, AttributeValue> > result = response.Items; if (null == result) { return(new Response(false, "Position no more exist in database")); } List <PositionData> PositionList = CovertresponseIntoJSON(response); return(new Response(true, "Found Element", PositionList)); } catch (Exception exception) { Console.WriteLine(exception.Message); return(new Response(false, exception.Message)); } }
public async Task <List <string> > Scanning(ScanRequest scanRequest) { List <Dictionary <string, AttributeValue> > listaItem = new List <Dictionary <string, AttributeValue> >(); List <string> listaJson = new List <string>(); try { //Identificar tabela table = Table.LoadTable(_dynamoClient, scanRequest.TableName); //Criar contexto DynamoDBContext db = new DynamoDBContext(_dynamoClient); ScanResponse scanResponse = await _dynamoClient.ScanAsync(scanRequest, cancellationToken); //Mapear os campos de retorno foreach (var item in scanResponse.Items) { var objDocument = table.FromAttributeMap(item); var objJson = objDocument.ToJson(); listaJson.Add(objJson); } //Converter para Json return(listaJson); } catch (Exception ex) { throw ex; } }
private async Task <List <string> > GetMobIds() { Dictionary <string, AttributeValue> startKeys = null; const string tableName = "mobs1"; const string keyName = "id"; const int limit = 10; var request = new ScanRequest { TableName = tableName, Limit = limit }; var keyList = new List <string>(); do { request.ExclusiveStartKey = startKeys; var response = await _client.ScanAsync(request); response.Items.ForEach( item => keyList.AddRange(item.Where(x => x.Key == keyName) .Select(y => y.Value.S) .ToList())); startKeys = response.LastEvaluatedKey; } while (startKeys != null && startKeys.Count != 0); return(keyList); }
public async Task <IEnumerable <AppUser> > GetAllAsync(CancellationToken cancellationToken) { var request = new ScanRequest { TableName = DynamoTables.LOGON_APPUSERS, ProjectionExpression = "Email, #Name, CreatedAtUtc, UpdatedAtUtc", ExpressionAttributeNames = new Dictionary <string, string> { { "#Name", "Name" } } }; var itemsRequested = await _client.ScanTableAsync(request, cancellationToken); if (itemsRequested.HttpStatusCode != HttpStatusCode.OK) { throw new AppException(itemsRequested.HttpStatusCode); } var appUsers = new List <AppUser>(); foreach (var itemRequested in itemsRequested.Items) { var epochCreatedAtUtc = long.Parse(itemRequested["CreatedAtUtc"].N); var epochUpdatedAtUtc = long.Parse(itemRequested["UpdatedAtUtc"].N); appUsers.Add(AppUser.CreateResponse(email: itemRequested["Email"].S, name: itemRequested["Name"].S, createdAtUtc: DateTimeOffset.FromUnixTimeSeconds(epochCreatedAtUtc).DateTime, updatedAtUtc: DateTimeOffset.FromUnixTimeSeconds(epochUpdatedAtUtc).DateTime)); } return(appUsers); }
private void FindAllStickers(Dictionary <string, AttributeValue> lastKeyEvaluated) { var request = new ScanRequest { TableName = "CaptainCillianStickers", ExclusiveStartKey = lastKeyEvaluated }; _client.ScanAsync(request, (result) => { foreach (Dictionary <string, AttributeValue> item in result.Response.Items) { // New sticker found, add info to new sticker object addItem(item); } lastKeyEvaluated = result.Response.LastEvaluatedKey; if (lastKeyEvaluated != null && lastKeyEvaluated.Count != 0) { FindAllStickers(lastKeyEvaluated); } else { retrievedAllFromDatabase = true; } }); }
public IEnumerable <Organization> GetAll() { List <Organization> returnValue = new List <Organization>(); AmazonDynamoDBClient client = DynamoUtilities.InitializeClient(); using (client) { ScanRequest request = CreateScanRequest(); ScanResponse response = client.Scan(request); if (response.Items == null || response.Items.Any() == false) { string errorMessage = string.Format(ErrorMessages.MisingResponseItem, "Get Organization"); throw new MissingFieldException(errorMessage); } foreach (Dictionary <string, AttributeValue> item in response.Items) { Organization organization = DynamoUtilities.GetItemFromAttributeStore <Organization>(item); returnValue.Add(organization); } return(returnValue); } }
public override void Invoke(AWSCredentials creds, RegionEndpoint region, int maxItems) { AmazonDynamoDBConfig config = new AmazonDynamoDBConfig(); config.RegionEndpoint = region; ConfigureClient(config); AmazonDynamoDBClient client = new AmazonDynamoDBClient(creds, config); ScanResponse resp = new ScanResponse(); do { ScanRequest req = new ScanRequest { ExclusiveStartKey = resp.LastEvaluatedKey , Limit = maxItems }; resp = client.Scan(req); CheckError(resp.HttpStatusCode, "200"); foreach (var obj in resp.Items) { AddObject(obj); } }while (resp.LastEvaluatedKey.Count > 0); }
public DataInfo Get(DataInfo item) { AmazonDynamoDBClient client = DynamoUtilities.InitializeClient(); using (client) { ScanRequest request = CreateScanRequest(item.DataId); if (request.ScanFilter.Count == 0) { throw new InvalidOperationException(ErrorMessages.NoSearchCriteria); } ScanResponse response = client.Scan(request); if (response.Items == null) { string errorMessage = string.Format(ErrorMessages.MisingResponseItem, "Get Data Info"); throw new MissingFieldException(errorMessage); } Dictionary <string, AttributeValue> userData = response.Items[0]; var returnValue = DynamoUtilities.GetItemFromAttributeStore <DataInfo>(userData); return(returnValue); } }
private IEnumerable <Dictionary <string, AttributeValue> > ScanHelper(string tableName, Dictionary <string, Condition> conditions, int?segment = null, int?totalSegments = null) { var request = new ScanRequest { TableName = tableName, ScanFilter = conditions, Limit = ScanLimit, }; if (segment.HasValue && totalSegments.HasValue) { request.Segment = segment.Value; request.TotalSegments = totalSegments.Value; } ScanResult result; do { result = Client.Scan(request); foreach (var item in result.Items) { yield return(item); } request.ExclusiveStartKey = result.LastEvaluatedKey; } while (result.LastEvaluatedKey != null && result.LastEvaluatedKey.Count > 0); }
public static async Task <User.User> RetrieveUser(string email) { var user = new User.User(); try { var dynamoDbClient = DynamoDBClientCreator.CreateClient(); var request = new ScanRequest { TableName = TableNames.user, FilterExpression = "email = :email", ExpressionAttributeValues = new Dictionary <string, AttributeValue>() { { ":email", new AttributeValue(email) } } }; var response = await dynamoDbClient.ScanAsync(request).ConfigureAwait(true); var userItm = response.Items[0]; user = User.User.deserialiseAsUser(userItm); } catch (Exception e) { LambdaLogger.Log("Unable to retrieve codes: " + e.ToString()); } return(user); }
public List <T> Scan <T>(ScanRequest request, int limit) { var to = new List <T>(); if (request.Limit == default(int)) { request.Limit = limit; } ScanResponse response = null; do { if (response != null) { request.ExclusiveStartKey = response.LastEvaluatedKey; } response = Exec(() => DynamoDb.Scan(request)); var results = response.ConvertAll <T>(); foreach (var result in results) { to.Add(result); if (to.Count >= limit) { break; } } } while (!response.LastEvaluatedKey.IsEmpty() && to.Count < limit); return(to); }
/// <summary> /// Scan a DynamoDB table by querying the entry fields. /// </summary> /// <typeparam name="TResult">The result type</typeparam> /// <param name="tableName">The name of the table to search for the entries</param> /// <param name="attributes">The attributes used on the expression</param> /// <param name="expression">The filter expression</param> /// <param name="resolver">Function that will be called to translate the returned fields into a concrete type. This Function is only called if the result is != null and will be called for each entry that match the query and added to the results list</param> /// <returns>The collection containing a list of objects translated by the resolver function</returns> public async Task <List <TResult> > ScanAsync <TResult>(string tableName, Dictionary <string, AttributeValue> attributes, string expression, Func <Dictionary <string, AttributeValue>, TResult> resolver) where TResult : class { try { var request = new ScanRequest { TableName = tableName, ConsistentRead = true, FilterExpression = expression, ExpressionAttributeValues = attributes, Select = Select.ALL_ATTRIBUTES }; var response = await ddbClient.ScanAsync(request); var resultList = new List <TResult>(); foreach (var item in response.Items) { resultList.Add(resolver(item)); } return(resultList); } catch (Exception exc) { var errorMsg = $"Failed to read table {tableName}: {exc.Message}"; Logger.Warn(ErrorCode.StorageProviderBase, errorMsg, exc); throw new OrleansException(errorMsg, exc); } }
private static async Task QueryDB(string lastName, string firstName) { AmazonDynamoDBClient client2 = new AmazonDynamoDBClient(clientRegion); var request = new ScanRequest { TableName = "Employee" }; try { var response = client2.Scan(request); string temp = ""; foreach (Dictionary<string, AttributeValue> item in response.Items) { if (!String.IsNullOrEmpty(lastName) && !String.IsNullOrEmpty(firstName)) { PrintItemByBoth(item, lastName, firstName); } if (!String.IsNullOrEmpty(lastName) && String.IsNullOrEmpty(firstName)) { PrintItemByLastName(item, lastName); } if (String.IsNullOrEmpty(lastName) && !String.IsNullOrEmpty(firstName)) { PrintItemByFirstName(item, firstName); } } } catch (ResourceNotFoundException) { } }
private void FindProductsForPriceLessThanZeroHelper(Dictionary <string, AttributeValue> lastKeyEvaluated) { var request = new ScanRequest { TableName = "ProductCatalog", Limit = 2, ExclusiveStartKey = lastKeyEvaluated, ExpressionAttributeValues = new Dictionary <string, AttributeValue> { { ":val", new AttributeValue { N = "0" } } }, FilterExpression = "Price < :val", ProjectionExpression = "Id, Title, Price" }; _client.ScanAsync(request, (result) => { foreach (Dictionary <string, AttributeValue> item in result.Response.Items) { resultText.text = ("\nScanThreadTableUsePaging - printing....."); PrintItem(item); } lastKeyEvaluated = result.Response.LastEvaluatedKey; if (lastKeyEvaluated != null && lastKeyEvaluated.Count != 0) { FindProductsForPriceLessThanZeroHelper(lastKeyEvaluated); } }); }
public async Task <T> Get <T>(Expression <Func <T, bool> > expression) where T : Base, new() { var generic = new T(); var tableName = generic.GetTableName(); var expressionAttributeNames = generic.GetExpressionAttributes(); var dynamoDbRequest = new ScanRequest { TableName = tableName, ProjectionExpression = string.Join(", ", expressionAttributeNames.Keys), ExpressionAttributeNames = expressionAttributeNames }; var expressionString = new StringBuilder(); dynamoDbRequest.ExpressionAttributeValues = Converters.ExpressionValues.ConvertExpressionValues(expression, ref expressionString); dynamoDbRequest.FilterExpression = expressionString.ToString(); var results = (await amazonDynamoDB.ScanAsync(dynamoDbRequest)).Items; if (results.Count > 1) { throw new RepositoryException($"Too many items returned by query: ({results.Count})"); } else if (results.Count == 0) { return(null); } return(results.SingleOrDefault()?.Map <T>()); }
public async Task TestDatabaseConnectionGetMaxId() { var scanRequest = new ScanRequest { TableName = "user", //ProjectionExpression = "Id, Title, #pr", ExpressionAttributeValues = new Dictionary <string, AttributeValue> { { ":q_user_id", new AttributeValue { N = TABLE_ID.ToString() } } }, ExpressionAttributeNames = new Dictionary <string, string> { { "#user_id", "user_id" } }, FilterExpression = "#user_id >= :q_user_id", Limit = 1, }; try { var response2 = await AmazonDynamoDBFactory.Client.ScanAsync(scanRequest); } catch (Exception ex) { Assert.Fail(ex.ToString()); Console.WriteLine(ex.ToString()); } }
/*-------------------------------------------------------------------------- * ClientScanning_async *--------------------------------------------------------------------------*/ public static async Task<bool> ClientScanning_async( ScanRequest sRequest ) { operationSucceeded = false; operationFailed = false; ScanResponse sResponse; Task<ScanResponse> clientScan = client.ScanAsync(sRequest); try { sResponse = await clientScan; } catch( Exception ex ) { Console.WriteLine( " -- FAILED to retrieve the movies, because:\n {0}", ex.Message ); operationFailed = true; pause( ); return( false ); } Console.WriteLine( " -- The low-level scan succeeded, and returned {0} movies!", sResponse.Items.Count ); if( !pause( ) ) { operationFailed = true; return ( false ); } Console.WriteLine( " Here are the movies retrieved:\n" + " --------------------------------------------------------------------------" ); foreach( Dictionary<string, AttributeValue> item in sResponse.Items ) showMovieAttrsShort( item ); Console.WriteLine( " -- Retrieved {0} movies.", sResponse.Items.Count ); operationSucceeded = true; return ( true ); }
public List <AppreciationCards.Models.Messages> QueryDB(ScanRequest request, AmazonDynamoDBClient raw) { List <AppreciationCards.Models.Messages> allMessages = new List <AppreciationCards.Models.Messages>(); ScanResponse response = null; do { if (response != null) { request.ExclusiveStartKey = response.LastEvaluatedKey; } response = raw.Scan(request); foreach (var item in response.Items) { string DatenTime = item["Date"].N; string format = "yyyyMMddHHmmss"; CultureInfo provider = CultureInfo.InvariantCulture; DateTime result = DateTime.ParseExact(DatenTime, format, provider); var messages = new AppreciationCards.Models.Messages() { FromName = item["From_name"].S, ToName = item["To_name"].S, Content = item["Content"].S, Value = item["Value"].S, Unread = item["Unread"].BOOL, MessageDate = result }; allMessages.Add(messages); } } while (response.LastEvaluatedKey != null && response.LastEvaluatedKey.Count > 0); return(allMessages); }