public void CSVSelectObjectGzipDataTest() { const string key = "gzip_data.csv.gz"; try { var buffer = new byte[256]; _ossClient.PutObject(_bucketName, key, _test_csv_gzip_file); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select * from ossobject"; var inputFormat = new SelectObjectCSVInputFormat(); inputFormat.FileHeaderInfo = FileHeaderInfo.None; inputFormat.RecordDelimiter = "\n"; inputFormat.FieldDelimiter = ","; inputFormat.QuoteCharacter = "\""; inputFormat.CompressionType = CompressionType.GZIP; request.InputFormat = inputFormat; var result = _ossClient.SelectObject(request); var md5 = FileUtils.ComputeContentMd5(result.Content); Assert.AreEqual(md5, "2e39f40a7d65fcf0acc95654638482da"); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectBigCSVDataTest() { const string key = "gzip_data.csv"; try { var buffer = new byte[256]; _ossClient.PutObject(_bucketName, key, _test_csv_file); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select Year, StateAbbr,StateDesc from ossobject limit 200"; var inputFormat = new SelectObjectCSVInputFormat(); inputFormat.FileHeaderInfo = FileHeaderInfo.Use; request.InputFormat = inputFormat; var result = _ossClient.SelectObject(request); StreamReader sr = new StreamReader(result.Content); int cnt = 0; while (sr.ReadLine() != null) { cnt++; } Assert.AreEqual(200, cnt); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectCommentCharacterTest() { const string key = "CSVSelectObjectCommentCharacterTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n`123,456\n#ghi,jkl\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var metaRequest = new CreateSelectObjectMetaRequest(_bucketName, key); metaRequest.InputFormat = new CreateSelectObjectMetaCSVInputFormat(); _ossClient.CreateSelectObjectMeta(metaRequest); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _1 from ossobject"; var inputFormat = new SelectObjectCSVInputFormat(); inputFormat.CommentCharacter = "`"; request.InputFormat = inputFormat; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc\n#ghi\n", Encoding.Default.GetString(buffer, 0, got)); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectInputDelimitersTest() { const string key = "CSVSelectObjectInputDelimitersTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("abc,def|123,456|7891334\n\n777,888|999,222|012345\n\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _2,_3 from ossobject"; var inputFormat = new SelectObjectCSVInputFormat(); inputFormat.RecordDelimiter = "\n\n"; inputFormat.FieldDelimiter = "|"; request.InputFormat = inputFormat; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("123,456|7891334\n\n999,222|012345\n\n", Encoding.Default.GetString(buffer, 0, got)); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectSkipPartialDataTest() { const string key = "CSVSelectObjectSkipPartialDataTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\nefg\nhij,klm\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _1, _2 from ossobject"; //with SkipPartial var inputFormat = new SelectObjectCSVInputFormat(); request.InputFormat = inputFormat; var options = new SelectObjectOptions(); options.SkipPartialDataRecord = true; request.Options = options; try { _ossClient.SelectObject(request); Assert.IsTrue(false); } catch (OssException e) { Assert.AreEqual("InvalidCsvLine", e.ErrorCode); } //without SkipPartial binaryData = Encoding.ASCII.GetBytes("abc,def\nefg\nhij,klm\n123,456\n"); stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _1, _2 from ossobject"; inputFormat = new SelectObjectCSVInputFormat(); request.InputFormat = inputFormat; options = new SelectObjectOptions(); options.SkipPartialDataRecord = false; request.Options = options; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc,def\nefg,\nhij,klm\n123,456\n", Encoding.Default.GetString(buffer, 0, got)); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectHeaderTest() { const string key = "CSVSelectObjectWithHeaderTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("name,job\nabc,def\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select name from ossobject"; // with header var inputFormat = new SelectObjectCSVInputFormat(); inputFormat.FileHeaderInfo = FileHeaderInfo.Use; request.InputFormat = inputFormat; var outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.OutputHeader = true; request.OutputFormat = outputFormat; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("name\nabc\n", Encoding.Default.GetString(buffer, 0, got)); // without header inputFormat = new SelectObjectCSVInputFormat(); inputFormat.FileHeaderInfo = FileHeaderInfo.Use; request.InputFormat = inputFormat; outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.OutputHeader = false; request.OutputFormat = outputFormat; result = _ossClient.SelectObject(request); got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got)); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectOutputDelimitersTest() { const string key = "CSVSelectObjectOutputDelimitersTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _1, _2 from ossobject"; //with delimiter var inputFormat = new SelectObjectCSVInputFormat(); request.InputFormat = inputFormat; var outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.RecordDelimiter = "\r\n"; outputFormat.FieldDelimiter = "|"; request.OutputFormat = outputFormat; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc|def\r\n", Encoding.Default.GetString(buffer, 0, got)); //without delimiter request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _1, _2 from ossobject"; inputFormat = new SelectObjectCSVInputFormat(); request.InputFormat = inputFormat; outputFormat = new SelectObjectCSVOutputFormat(); request.OutputFormat = outputFormat; result = _ossClient.SelectObject(request); got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc,def\n", Encoding.Default.GetString(buffer, 0, got)); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectRawTest() { const string key = "CSVSelectObjectRawTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _1 from ossobject"; //with raw var inputFormat = new SelectObjectCSVInputFormat(); request.InputFormat = inputFormat; var outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.OutputRawData = true; request.OutputFormat = outputFormat; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got)); Assert.AreEqual(true, result.Metadata.HttpMetadata.ContainsKey("x-oss-select-output-raw")); Assert.AreEqual("true", result.Metadata.HttpMetadata["x-oss-select-output-raw"]); // without raw outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.OutputRawData = false; request.OutputFormat = outputFormat; result = _ossClient.SelectObject(request); got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got)); Assert.AreEqual(true, result.Metadata.HttpMetadata.ContainsKey("x-oss-select-output-raw")); Assert.AreEqual("false", result.Metadata.HttpMetadata["x-oss-select-output-raw"]); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectColumnsTest() { const string key = "CSVSelectObjectColumnsTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("abc,def,ghi,jkl\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _1, _4 from ossobject"; //with columns var inputFormat = new SelectObjectCSVInputFormat(); request.InputFormat = inputFormat; var outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.KeepAllColumns = true; request.OutputFormat = outputFormat; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc,,,jkl\n", Encoding.Default.GetString(buffer, 0, got)); // without columns outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.KeepAllColumns = false; request.OutputFormat = outputFormat; result = _ossClient.SelectObject(request); got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("abc,jkl\n", Encoding.Default.GetString(buffer, 0, got)); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void NormalSelectObjectWithOutputRawTest() { const string key = "SqlObjectWithoutOutputRaw"; try { byte[] binaryData = Encoding.ASCII.GetBytes(_sqlMessage); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select * from ossobject"; var inputFormat = new SelectObjectCSVInputFormat(); inputFormat.FileHeaderInfo = FileHeaderInfo.Use; inputFormat.RecordDelimiter = "\r\n"; inputFormat.FieldDelimiter = ","; inputFormat.QuoteCharacter = "\""; inputFormat.CommentCharacter = "#"; inputFormat.CompressionType = CompressionType.None; request.InputFormat = inputFormat; var outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.KeepAllColumns = false; outputFormat.OutputRawData = true; outputFormat.OutputHeader = false; outputFormat.EnablePayloadCrc = false; outputFormat.FieldDelimiter = ","; outputFormat.RecordDelimiter = "\n"; request.OutputFormat = outputFormat; _ossClient.SelectObject(request); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectQuoteCharacterTest() { const string key = "CSVSelectObjectQuoteCharacterTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("'abc','def\n123','456'\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _2 from ossobject"; var inputFormat = new SelectObjectCSVInputFormat(); request.InputFormat = inputFormat; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("'def\n'456'\n", Encoding.Default.GetString(buffer, 0, got)); //use ' inputFormat = new SelectObjectCSVInputFormat(); inputFormat.QuoteCharacter = "'"; request.InputFormat = inputFormat; result = _ossClient.SelectObject(request); got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("'def\n123'\n", Encoding.Default.GetString(buffer, 0, got)); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectLineRangeTest() { const string key = "CSVSelectObjectLineRangeTest"; try { var buffer = new byte[256]; byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n123,456\n789,efg\nhij,klm\n"); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var metaRequest = new CreateSelectObjectMetaRequest(_bucketName, key); metaRequest.InputFormat = new CreateSelectObjectMetaCSVInputFormat(); _ossClient.CreateSelectObjectMeta(metaRequest); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select _1, _2 from ossobject"; var inputFormat = new SelectObjectCSVInputFormat(); inputFormat.Range = "line-range=1-2"; request.InputFormat = inputFormat; var outputFormat = new SelectObjectCSVOutputFormat(); request.OutputFormat = outputFormat; var result = _ossClient.SelectObject(request); var got = result.Content.Read(buffer, 0, buffer.Length); Assert.AreEqual("123,456\n789,efg\n", Encoding.Default.GetString(buffer, 0, got)); } finally { _ossClient.DeleteObject(_bucketName, key); } }
public void CSVSelectObjectNormalTest() { const string key = "SqlObjectWithCsvData"; try { byte[] binaryData = Encoding.ASCII.GetBytes(_sqlMessage); var stream = new MemoryStream(binaryData); _ossClient.PutObject(_bucketName, key, stream); var request = new SelectObjectRequest(_bucketName, key); request.Expression = "select * from ossobject"; var inputFormat = new SelectObjectCSVInputFormat(); inputFormat.FileHeaderInfo = FileHeaderInfo.Use; inputFormat.RecordDelimiter = "\r\n"; inputFormat.FieldDelimiter = ","; inputFormat.QuoteCharacter = "\""; inputFormat.CommentCharacter = "#"; inputFormat.CompressionType = CompressionType.None; request.InputFormat = inputFormat; var outputFormat = new SelectObjectCSVOutputFormat(); outputFormat.EnablePayloadCrc = true; outputFormat.OutputRawData = false; outputFormat.KeepAllColumns = false; outputFormat.OutputHeader = false; outputFormat.FieldDelimiter = ","; outputFormat.RecordDelimiter = "\n"; request.OutputFormat = outputFormat; var result = _ossClient.SelectObject(request); string expect = "Lora Francis,School A,Staples Inc,27\n" + "Eleanor Little,School B,\"Conectiv, Inc\",43\n" + "Rosie Hughes,School C,Western Gas Resources Inc,44\n" + "Lawrence Ross,School D,MetLife Inc.,24\n"; var buf = new byte[256]; int offset = 0; for (int i = 1; i < 30; i++) { int got = result.Content.Read(buf, offset, i); offset += got; } Assert.AreEqual(expect.Length, offset); string str = Encoding.Default.GetString(buf, 0, offset); Assert.AreEqual(expect, str); } finally { _ossClient.DeleteObject(_bucketName, key); } }