public void CSVSelectObjectHeaderTest()
{
const string key = "CSVSelectObjectWithHeaderTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("name,job\nabc,def\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select name from ossobject";
// with header
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.OutputHeader = true;
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("name\nabc\n", Encoding.Default.GetString(buffer, 0, got));
// without header
inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
request.InputFormat = inputFormat;
outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.OutputHeader = false;
request.OutputFormat = outputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectOutputDelimitersTest()
{
const string key = "CSVSelectObjectOutputDelimitersTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _2 from ossobject";
//with delimiter
var inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.RecordDelimiter = "\r\n";
outputFormat.FieldDelimiter = "|";
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc|def\r\n", Encoding.Default.GetString(buffer, 0, got));
//without delimiter
request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _2 from ossobject";
inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
outputFormat = new SelectObjectCSVOutputFormat();
request.OutputFormat = outputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc,def\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void NormalSelectObjectWithJsonType()
{
const string key = "SqlObjectWithJsonType";
try
{
byte[] binaryData = Encoding.ASCII.GetBytes(_jsonMessage);
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select * from ossobject";
var inputFormat = new SelectObjectJSONInputFormat();
inputFormat.CompressionType = CompressionType.None;
inputFormat.Type = JSONType.LINES;
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.KeepAllColumns = false;
outputFormat.OutputRawData = false;
outputFormat.OutputHeader = false;
outputFormat.EnablePayloadCrc = true;
outputFormat.RecordDelimiter = "\n";
request.OutputFormat = outputFormat;
_ossClient.SelectObject(request);
var metarequest = new CreateSelectObjectMetaRequest(_bucketName, key);
metarequest.OverwriteIfExists = false;
var input = new CreateSelectObjectMetaJSONInputFormat();
input.CompressionType = CompressionType.None;
input.Type = JSONType.LINES;
metarequest.InputFormat = input;
var metaresult = _ossClient.CreateSelectObjectMeta(metarequest);
Assert.AreEqual(metaresult.SplitsCount, 1);
Assert.AreEqual(metaresult.RowsCount, 4);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectRawTest()
{
const string key = "CSVSelectObjectRawTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1 from ossobject";
//with raw
var inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.OutputRawData = true;
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got));
Assert.AreEqual(true, result.Metadata.HttpMetadata.ContainsKey("x-oss-select-output-raw"));
Assert.AreEqual("true", result.Metadata.HttpMetadata["x-oss-select-output-raw"]);
// without raw
outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.OutputRawData = false;
request.OutputFormat = outputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got));
Assert.AreEqual(true, result.Metadata.HttpMetadata.ContainsKey("x-oss-select-output-raw"));
Assert.AreEqual("false", result.Metadata.HttpMetadata["x-oss-select-output-raw"]);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectColumnsTest()
{
const string key = "CSVSelectObjectColumnsTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def,ghi,jkl\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _4 from ossobject";
//with columns
var inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.KeepAllColumns = true;
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc,,,jkl\n", Encoding.Default.GetString(buffer, 0, got));
// without columns
outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.KeepAllColumns = false;
request.OutputFormat = outputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc,jkl\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void NormalSelectObjectWithOutputRawTest()
{
const string key = "SqlObjectWithoutOutputRaw";
try
{
byte[] binaryData = Encoding.ASCII.GetBytes(_sqlMessage);
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select * from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
inputFormat.RecordDelimiter = "\r\n";
inputFormat.FieldDelimiter = ",";
inputFormat.QuoteCharacter = "\"";
inputFormat.CommentCharacter = "#";
inputFormat.CompressionType = CompressionType.None;
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.KeepAllColumns = false;
outputFormat.OutputRawData = true;
outputFormat.OutputHeader = false;
outputFormat.EnablePayloadCrc = false;
outputFormat.FieldDelimiter = ",";
outputFormat.RecordDelimiter = "\n";
request.OutputFormat = outputFormat;
_ossClient.SelectObject(request);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectLineRangeTest()
{
const string key = "CSVSelectObjectLineRangeTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n123,456\n789,efg\nhij,klm\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var metaRequest = new CreateSelectObjectMetaRequest(_bucketName, key);
metaRequest.InputFormat = new CreateSelectObjectMetaCSVInputFormat();
_ossClient.CreateSelectObjectMeta(metaRequest);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _2 from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.Range = "line-range=1-2";
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("123,456\n789,efg\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectNormalTest()
{
const string key = "SqlObjectWithCsvData";
try
{
byte[] binaryData = Encoding.ASCII.GetBytes(_sqlMessage);
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select * from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
inputFormat.RecordDelimiter = "\r\n";
inputFormat.FieldDelimiter = ",";
inputFormat.QuoteCharacter = "\"";
inputFormat.CommentCharacter = "#";
inputFormat.CompressionType = CompressionType.None;
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.EnablePayloadCrc = true;
outputFormat.OutputRawData = false;
outputFormat.KeepAllColumns = false;
outputFormat.OutputHeader = false;
outputFormat.FieldDelimiter = ",";
outputFormat.RecordDelimiter = "\n";
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
string expect = "Lora Francis,School A,Staples Inc,27\n" +
"Eleanor Little,School B,\"Conectiv, Inc\",43\n" +
"Rosie Hughes,School C,Western Gas Resources Inc,44\n" +
"Lawrence Ross,School D,MetLife Inc.,24\n";
var buf = new byte[256];
int offset = 0;
for (int i = 1; i < 30; i++)
{
int got = result.Content.Read(buf, offset, i);
offset += got;
}
Assert.AreEqual(expect.Length, offset);
string str = Encoding.Default.GetString(buf, 0, offset);
Assert.AreEqual(expect, str);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
