public async Task QueryAsync_IfTags()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(Constants.KB);
            await blockBlobClient.UploadAsync(stream);

            Dictionary <string, string> tags = new Dictionary <string, string>
            {
                { "coolTag", "true" }
            };
            await blockBlobClient.SetTagsAsync(tags);

            BlobQueryOptions blobQueryOptions = new BlobQueryOptions
            {
                Conditions = new BlobRequestConditions
                {
                    TagConditions = "\"coolTag\" = 'true'"
                }
            };

            // Act
            string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
            Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
                querySqlExpression : query,
                options : blobQueryOptions);

            using StreamReader streamReader = new StreamReader(response.Value.Content);
            string s = await streamReader.ReadToEndAsync();

            // Assert
            Assert.AreEqual("400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n400\n", s);
        }
        public async Task QueryAsync_ArrowConfigurationInput()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(Constants.KB);
            await blockBlobClient.UploadAsync(stream);

            string           query   = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
            BlobQueryOptions options = new BlobQueryOptions
            {
                InputTextConfiguration = new BlobQueryArrowOptions
                {
                    Schema = new List <BlobQueryArrowField>()
                    {
                        new BlobQueryArrowField
                        {
                            Type      = BlobQueryArrowFieldType.Decimal,
                            Name      = "Name",
                            Precision = 4,
                            Scale     = 2
                        }
                    }
                }
            };

            // Act
            await TestHelper.AssertExpectedExceptionAsync <ArgumentException>(
                blockBlobClient.QueryAsync(
                    query,
                    options: options),
                e => Assert.AreEqual($"{nameof(BlobQueryArrowOptions)} can only be used for output serialization.", e.Message));
        }
        public async Task QueryAsync_ParquetConfiguration()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));

            using FileStream stream = File.OpenRead(
                      $"{Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)}{Path.DirectorySeparatorChar}Resources{Path.DirectorySeparatorChar}parquet.parquet");
            await blockBlobClient.UploadAsync(stream);

            // Act
            string           query   = @"select * from blobstorage where id < 1;";
            BlobQueryOptions options = new BlobQueryOptions
            {
                InputTextConfiguration = new BlobQueryParquetTextOptions()
            };
            Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
                query,
                options : options);

            // Assert
            using StreamReader streamReader = new StreamReader(response.Value.Content);
            string s = await streamReader.ReadToEndAsync();

            // Assert
            Assert.AreEqual("0,mdifjt55.ea3,mdifjt55.ea3\n", s);
        }
        public async Task QueryAsync_IfTags_Failed()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(Constants.KB);
            await blockBlobClient.UploadAsync(stream);

            BlobQueryOptions blobQueryOptions = new BlobQueryOptions
            {
                Conditions = new BlobRequestConditions
                {
                    TagConditions = "\"coolTag\" = 'true'"
                }
            };

            string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";

            // Act
            await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
                blockBlobClient.QueryAsync(
                    querySqlExpression: query,
                    options: blobQueryOptions),
                e => Assert.AreEqual(BlobErrorCode.ConditionNotMet.ToString(), e.ErrorCode));
        }
        public async Task QueryAsync_Progress()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(Constants.KB);
            await blockBlobClient.UploadAsync(stream);

            // Act
            string           query            = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
            TestProgress     progressReporter = new TestProgress();
            BlobQueryOptions options          = new BlobQueryOptions
            {
                ProgressHandler = progressReporter
            };

            Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
                query,
                options);

            using StreamReader streamReader = new StreamReader(response.Value.Content);
            await streamReader.ReadToEndAsync();

            Assert.AreEqual(2, progressReporter.List.Count);
            Assert.AreEqual(Constants.KB, progressReporter.List[0]);
            Assert.AreEqual(Constants.KB, progressReporter.List[1]);
        }
        public async Task QueryAsync_AccessConditionsFail()
        {
            var garbageLeaseId = GetGarbageLeaseId();

            foreach (AccessConditionParameters parameters in GetAccessConditionsFail_Data(garbageLeaseId))
            {
                // Arrange
                await using DisposingContainer test = await GetTestContainerAsync();

                BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
                Stream          stream          = CreateDataStream(Constants.KB);
                await blockBlobClient.UploadAsync(stream);

                parameters.NoneMatch = await SetupBlobMatchCondition(blockBlobClient, parameters.NoneMatch);

                BlobRequestConditions accessConditions = BuildAccessConditions(parameters);
                BlobQueryOptions      options          = new BlobQueryOptions
                {
                    Conditions = accessConditions
                };

                string query = @"SELECT * from BlobStorage";

                // Act
                await TestHelper.AssertExpectedExceptionAsync <RequestFailedException>(
                    blockBlobClient.QueryAsync(
                        query,
                        options),
                    e => { });
            }
        }
        public async Task QueryAsync_Large()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(250 * Constants.MB);
            await blockBlobClient.UploadAsync(stream);

            string query = @"SELECT * from BlobStorage";

            // Act
            TestProgress     progressReporter = new TestProgress();
            BlobQueryOptions options          = new BlobQueryOptions
            {
                ProgressHandler = progressReporter
            };
            Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
                query,
                options);

            stream.Seek(0, SeekOrigin.Begin);
            using StreamReader expectedStreamReader = new StreamReader(stream);
            string expected = await expectedStreamReader.ReadToEndAsync();

            using StreamReader actualStreamReader = new StreamReader(response.Value.Content);
            string actual = await actualStreamReader.ReadToEndAsync();

            // Assert
            // Check we got back the same content that we uploaded.
            Assert.AreEqual(expected, actual);
        }
Ejemplo n.º 8
0
        public async Task QueryAsync_ArrowConfiguration()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(Constants.KB);
            await blockBlobClient.UploadAsync(stream);

            // Act
            string           query   = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
            BlobQueryOptions options = new BlobQueryOptions
            {
                OutputTextConfiguration = new BlobQueryArrowOptions
                {
                    Schema = new List <BlobQueryArrowField>()
                    {
                        new BlobQueryArrowField
                        {
                            Type      = BlobQueryArrowFieldType.Decimal,
                            Name      = "Name",
                            Precision = 4,
                            Scale     = 2
                        }
                    }
                }
            };

            // Act
            await blockBlobClient.QueryAsync(
                query,
                options : options);
        }
        public async Task QueryAsync_FatalError()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(Constants.KB);
            await blockBlobClient.UploadAsync(stream);

            string query = @"SELECT * from BlobStorage;";
            BlobQueryJsonTextOptions jsonTextConfiguration = new BlobQueryJsonTextOptions
            {
                RecordSeparator = "\n"
            };
            BlobQueryOptions options = new BlobQueryOptions
            {
                InputTextConfiguration = jsonTextConfiguration
            };

            // Act - with no IBlobQueryErrorReceiver
            Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
                query,
                options);

            using StreamReader streamReader = new StreamReader(response.Value.Content);
            string s = await streamReader.ReadToEndAsync();

            // Act - with  IBlobQueryErrorReceiver
            BlobQueryError expectedBlobQueryError = new BlobQueryError
            {
                IsFatal     = true,
                Name        = "ParseError",
                Description = "Unexpected token ',' at [byte: 3]. Expecting tokens '{', or '['.",
                Position    = 0
            };

            BlobQueryErrorHandler errorHandler = new BlobQueryErrorHandler(expectedBlobQueryError);

            options = new BlobQueryOptions
            {
                InputTextConfiguration = jsonTextConfiguration,
            };
            options.ErrorHandler += errorHandler.Handle;

            response = await blockBlobClient.QueryAsync(
                query,
                options);

            using StreamReader streamReader2 = new StreamReader(response.Value.Content);
            s = await streamReader2.ReadToEndAsync();
        }
        public async Task QueryAsync_ParquetOutputError()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));

            // Act
            string           query   = @"select * from blobstorage where id < 1;";
            BlobQueryOptions options = new BlobQueryOptions
            {
                OutputTextConfiguration = new BlobQueryParquetTextOptions()
            };

            await TestHelper.AssertExpectedExceptionAsync <ArgumentException>(
                blockBlobClient.QueryAsync(
                    query, options),
                e => Assert.AreEqual($"{nameof(BlobQueryParquetTextOptions)} can only be used for input serialization.", e.Message));
        }
        public async Task QueryAsync_QueryTextConfigurations()
        {
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(Constants.KB);
            await blockBlobClient.UploadAsync(stream);

            // Act
            string query = @"SELECT _2 from BlobStorage WHERE _1 > 250;";

            BlobQueryCsvTextOptions csvTextConfiguration = new BlobQueryCsvTextOptions
            {
                ColumnSeparator    = ",",
                QuotationCharacter = '"',
                EscapeCharacter    = '\\',
                RecordSeparator    = "\n",
                HasHeaders         = false
            };

            BlobQueryJsonTextOptions jsonTextConfiguration = new BlobQueryJsonTextOptions
            {
                RecordSeparator = "\n"
            };

            BlobQueryOptions options = new BlobQueryOptions
            {
                InputTextConfiguration  = csvTextConfiguration,
                OutputTextConfiguration = jsonTextConfiguration
            };

            // Act
            Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
                query,
                options);

            using StreamReader streamReader = new StreamReader(response.Value.Content);
            string s = await streamReader.ReadToEndAsync();

            // Assert
            Assert.AreEqual("{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n{\"_1\":\"400\"}\n", s);
        }
Ejemplo n.º 12
0
        public async Task QueryAsync_NonFatalError()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));

            byte[] data = Encoding.UTF8.GetBytes("100,pizza,300,400\n300,400,500,600\n");
            using MemoryStream stream = new MemoryStream(data);
            await blockBlobClient.UploadAsync(stream);

            string query = @"SELECT _1 from BlobStorage WHERE _2 > 250;";

            // Act - with no IBlobQueryErrorReceiver
            Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(query);

            using StreamReader streamReader = new StreamReader(response.Value.Content);
            string s = await streamReader.ReadToEndAsync();


            // Act - with  IBlobQueryErrorReceiver
            BlobQueryError expectedBlobQueryError = new BlobQueryError
            {
                IsFatal     = false,
                Name        = "InvalidTypeConversion",
                Description = "Invalid type conversion.",
                Position    = 0
            };

            BlobQueryErrorHandler errorHandler = new BlobQueryErrorHandler(expectedBlobQueryError);

            BlobQueryOptions options = new BlobQueryOptions();

            options.ErrorHandler += errorHandler.Handle;

            response = await blockBlobClient.QueryAsync(
                query,
                options);

            using StreamReader streamReader2 = new StreamReader(response.Value.Content);
            s = await streamReader2.ReadToEndAsync();
        }
        public async Task QueryAsync_ArrowConfiguration()
        {
            // Arrange
            await using DisposingContainer test = await GetTestContainerAsync();

            BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
            Stream          stream          = CreateDataStream(Constants.KB);
            await blockBlobClient.UploadAsync(stream);

            // Act
            string           query   = @"SELECT _2 from BlobStorage WHERE _1 > 250;";
            BlobQueryOptions options = new BlobQueryOptions
            {
                OutputTextConfiguration = new BlobQueryArrowOptions
                {
                    Schema = new List <BlobQueryArrowField>()
                    {
                        new BlobQueryArrowField
                        {
                            Type      = BlobQueryArrowFieldType.Decimal,
                            Name      = "Name",
                            Precision = 4,
                            Scale     = 2
                        }
                    }
                }
            };
            Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
                query,
                options : options);

            MemoryStream memoryStream = new MemoryStream();
            await response.Value.Content.CopyToAsync(memoryStream);

            // Assert
            Assert.AreEqual("/////4AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABAwAMAAAACAAIAAAABAAIAAAABAAAAAEAAAAUAAAAEAAUAAgABgAHAAwAAAAQABAAAAAAAAEHJAAAABQAAAAEAAAAAAAAAAgADAAEAAgACAAAAAQAAAACAAAABAAAAE5hbWUAAAAAAAAAAP////9wAAAAEAAAAAAACgAOAAYABQAIAAoAAAAAAwMAEAAAAAAACgAMAAAABAAIAAoAAAAwAAAABAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAP////+IAAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMDABgAAAAAAgAAAAAAAAAACgAYAAwABAAIAAoAAAA8AAAAEAAAACAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAAAAAABAAAAIAAAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAAkAEAAAAAAAAAAAAAAAAAAJABAAAAAAAAAAAAAAAAAACQAQAAAAAAAAAAAAAAAAAA", Convert.ToBase64String(memoryStream.ToArray()));
        }
        public async Task QueryAsync_AccessConditions()
        {
            var garbageLeaseId = GetGarbageLeaseId();

            foreach (AccessConditionParameters parameters in AccessConditions_Data)
            {
                // Arrange
                await using DisposingContainer test = await GetTestContainerAsync();

                BlockBlobClient blockBlobClient = InstrumentClient(test.Container.GetBlockBlobClient(GetNewBlobName()));
                Stream          stream          = CreateDataStream(Constants.KB);
                await blockBlobClient.UploadAsync(stream);

                parameters.Match = await SetupBlobMatchCondition(blockBlobClient, parameters.Match);

                parameters.LeaseId = await SetupBlobLeaseCondition(blockBlobClient, parameters.LeaseId, garbageLeaseId);

                BlobRequestConditions accessConditions = BuildAccessConditions(
                    parameters: parameters,
                    lease: true);
                BlobQueryOptions options = new BlobQueryOptions
                {
                    Conditions = accessConditions
                };

                string query = @"SELECT * from BlobStorage";

                // Act
                Response <BlobDownloadInfo> response = await blockBlobClient.QueryAsync(
                    query,
                    options);

                // Assert
                Assert.IsNotNull(response.Value.Details.ETag);
            }
        }
        private static async Task <List <OpportunityHistory> > DumpQueryCsv(BlockBlobClient blob, string query, bool headers, ILogger log, List <string> attributes)
        {
            try
            {
                List <OpportunityHistory> opportunities = new List <OpportunityHistory>();
                var id             = attributes.FindIndex(str => str == "Id");
                var snapshotdate   = attributes.FindIndex(str => str == "SinkModifiedOn");
                var estimatedvalue = attributes.FindIndex(str => str == "estimatedvalue");
                var name           = attributes.FindIndex(str => str == "name");

                var counter = 0;

                var options = new BlobQueryOptions()
                {
                    InputTextConfiguration = new BlobQueryCsvTextOptions()
                    {
                        HasHeaders = headers
                    },
                    OutputTextConfiguration = new BlobQueryCsvTextOptions()
                    {
                        HasHeaders = true
                    },
                    ProgressHandler = new Progress <long>((finishedBytes) => Console.Error.WriteLine($"Data read: {finishedBytes}"))
                };
                options.ErrorHandler += (BlobQueryError err) => {
                };
                // BlobDownloadInfo exposes a Stream that will make results available when received rather than blocking for the entire response.
                using (var reader = new StreamReader((await blob.QueryAsync(
                                                          query,
                                                          options)).Value.Content))
                {
                    using (var parser = new CsvReader(reader, new CsvConfiguration(CultureInfo.CurrentCulture)
                    {
                        HasHeaderRecord = true
                    }))
                    {
                        while (await parser.ReadAsync())
                        {
                            //get indexOf prop


                            log.LogInformation(String.Join(" ", parser.Context.Record));
                            DateTime parsedDT;
                            decimal  parsedDec;
                            opportunities.Add(new OpportunityHistory
                            {
                                Id = new Guid(parser.Context.Record[0]),
                                EstimatedRevenue = decimal.TryParse(parser.Context.Record[2], out parsedDec)? parsedDec:0,
                                SnapshotDate     = DateTime.TryParse(parser.Context.Record[1], out parsedDT) ? (DateTime?)parsedDT : null,
                                Topic            = parser.Context.Record[3]
                            });
                            counter++;
                            log.LogInformation("Counter: " + counter.ToString());
                        }
                    }
                }

                return(opportunities);
            }
            catch (Exception ex)
            {
                log.LogError("Exception: " + ex.ToString());
                return(null);
            }
        }