public void CSVSelectObjectGzipDataTest()
{
const string key = "gzip_data.csv.gz";
try
{
var buffer = new byte[256];
_ossClient.PutObject(_bucketName, key, _test_csv_gzip_file);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select * from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.None;
inputFormat.RecordDelimiter = "\n";
inputFormat.FieldDelimiter = ",";
inputFormat.QuoteCharacter = "\"";
inputFormat.CompressionType = CompressionType.GZIP;
request.InputFormat = inputFormat;
var result = _ossClient.SelectObject(request);
var md5 = FileUtils.ComputeContentMd5(result.Content);
Assert.AreEqual(md5, "2e39f40a7d65fcf0acc95654638482da");
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectBigCSVDataTest()
{
const string key = "gzip_data.csv";
try
{
var buffer = new byte[256];
_ossClient.PutObject(_bucketName, key, _test_csv_file);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select Year, StateAbbr,StateDesc from ossobject limit 200";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
request.InputFormat = inputFormat;
var result = _ossClient.SelectObject(request);
StreamReader sr = new StreamReader(result.Content);
int cnt = 0;
while (sr.ReadLine() != null)
{
cnt++;
}
Assert.AreEqual(200, cnt);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
private SelectObjectCommand(IServiceClient client, Uri endpoint, ExecutionContext context,
IDeserializer <ServiceResponse, OssObject> deserializer,
SelectObjectRequest selectObjectRequest)
: base(client, endpoint, context, deserializer)
{
_request = selectObjectRequest;
}
public void CSVSelectObjectInputDelimitersTest()
{
const string key = "CSVSelectObjectInputDelimitersTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def|123,456|7891334\n\n777,888|999,222|012345\n\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _2,_3 from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.RecordDelimiter = "\n\n";
inputFormat.FieldDelimiter = "|";
request.InputFormat = inputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("123,456|7891334\n\n999,222|012345\n\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectCommentCharacterTest()
{
const string key = "CSVSelectObjectCommentCharacterTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n`123,456\n#ghi,jkl\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var metaRequest = new CreateSelectObjectMetaRequest(_bucketName, key);
metaRequest.InputFormat = new CreateSelectObjectMetaCSVInputFormat();
_ossClient.CreateSelectObjectMeta(metaRequest);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1 from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.CommentCharacter = "`";
request.InputFormat = inputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc\n#ghi\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectSkipPartialDataTest()
{
const string key = "CSVSelectObjectSkipPartialDataTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\nefg\nhij,klm\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _2 from ossobject";
//with SkipPartial
var inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
var options = new SelectObjectOptions();
options.SkipPartialDataRecord = true;
request.Options = options;
try
{
_ossClient.SelectObject(request);
Assert.IsTrue(false);
}
catch (OssException e)
{
Assert.AreEqual("InvalidCsvLine", e.ErrorCode);
}
//without SkipPartial
binaryData = Encoding.ASCII.GetBytes("abc,def\nefg\nhij,klm\n123,456\n");
stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _2 from ossobject";
inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
options = new SelectObjectOptions();
options.SkipPartialDataRecord = false;
request.Options = options;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc,def\nefg,\nhij,klm\n123,456\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public static SelectObjectCommand Create(IServiceClient client, Uri endpoint,
ExecutionContext context,
SelectObjectRequest request)
{
OssUtils.CheckBucketName(request.BucketName);
OssUtils.CheckObjectKey(request.Key);
return(new SelectObjectCommand(client, endpoint, context,
DeserializerFactory.GetFactory().CreateSelectObjectRequestDeserializer(request),
request));
}
public void CSVSelectObjectHeaderTest()
{
const string key = "CSVSelectObjectWithHeaderTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("name,job\nabc,def\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select name from ossobject";
// with header
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.OutputHeader = true;
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("name\nabc\n", Encoding.Default.GetString(buffer, 0, got));
// without header
inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
request.InputFormat = inputFormat;
outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.OutputHeader = false;
request.OutputFormat = outputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectOutputDelimitersTest()
{
const string key = "CSVSelectObjectOutputDelimitersTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _2 from ossobject";
//with delimiter
var inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.RecordDelimiter = "\r\n";
outputFormat.FieldDelimiter = "|";
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc|def\r\n", Encoding.Default.GetString(buffer, 0, got));
//without delimiter
request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _2 from ossobject";
inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
outputFormat = new SelectObjectCSVOutputFormat();
request.OutputFormat = outputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc,def\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectRawTest()
{
const string key = "CSVSelectObjectRawTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1 from ossobject";
//with raw
var inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.OutputRawData = true;
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got));
Assert.AreEqual(true, result.Metadata.HttpMetadata.ContainsKey("x-oss-select-output-raw"));
Assert.AreEqual("true", result.Metadata.HttpMetadata["x-oss-select-output-raw"]);
// without raw
outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.OutputRawData = false;
request.OutputFormat = outputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc\n", Encoding.Default.GetString(buffer, 0, got));
Assert.AreEqual(true, result.Metadata.HttpMetadata.ContainsKey("x-oss-select-output-raw"));
Assert.AreEqual("false", result.Metadata.HttpMetadata["x-oss-select-output-raw"]);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void NormalSelectObjectWithJsonType()
{
const string key = "SqlObjectWithJsonType";
try
{
byte[] binaryData = Encoding.ASCII.GetBytes(_jsonMessage);
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select * from ossobject";
var inputFormat = new SelectObjectJSONInputFormat();
inputFormat.CompressionType = CompressionType.None;
inputFormat.Type = JSONType.LINES;
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.KeepAllColumns = false;
outputFormat.OutputRawData = false;
outputFormat.OutputHeader = false;
outputFormat.EnablePayloadCrc = true;
outputFormat.RecordDelimiter = "\n";
request.OutputFormat = outputFormat;
_ossClient.SelectObject(request);
var metarequest = new CreateSelectObjectMetaRequest(_bucketName, key);
metarequest.OverwriteIfExists = false;
var input = new CreateSelectObjectMetaJSONInputFormat();
input.CompressionType = CompressionType.None;
input.Type = JSONType.LINES;
metarequest.InputFormat = input;
var metaresult = _ossClient.CreateSelectObjectMeta(metarequest);
Assert.AreEqual(metaresult.SplitsCount, 1);
Assert.AreEqual(metaresult.RowsCount, 4);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void CSVSelectObjectColumnsTest()
{
const string key = "CSVSelectObjectColumnsTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def,ghi,jkl\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _4 from ossobject";
//with columns
var inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.KeepAllColumns = true;
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc,,,jkl\n", Encoding.Default.GetString(buffer, 0, got));
// without columns
outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.KeepAllColumns = false;
request.OutputFormat = outputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("abc,jkl\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void NormalSelectObjectWithOutputRawTest()
{
const string key = "SqlObjectWithoutOutputRaw";
try
{
byte[] binaryData = Encoding.ASCII.GetBytes(_sqlMessage);
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select * from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
inputFormat.RecordDelimiter = "\r\n";
inputFormat.FieldDelimiter = ",";
inputFormat.QuoteCharacter = "\"";
inputFormat.CommentCharacter = "#";
inputFormat.CompressionType = CompressionType.None;
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.KeepAllColumns = false;
outputFormat.OutputRawData = true;
outputFormat.OutputHeader = false;
outputFormat.EnablePayloadCrc = false;
outputFormat.FieldDelimiter = ",";
outputFormat.RecordDelimiter = "\n";
request.OutputFormat = outputFormat;
_ossClient.SelectObject(request);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
public void testSelectObjectToFile()
{
try
{
string key = "select_target.json";
SelectObjectRequest request = new SelectObjectRequest(bucket, key);
ObjectSelectionFormat.JSONFormat jSONFormat = new ObjectSelectionFormat.JSONFormat();
jSONFormat.Type = "DOCUMENT";
jSONFormat.RecordDelimiter = "\n";
string outputFile = "select_local_file.json";
request.setExpression("Select * from COSObject")
.setInputFormat(new ObjectSelectionFormat(null, jSONFormat))
.setOutputFormat(new ObjectSelectionFormat(null, jSONFormat))
.SetCosProgressCallback(delegate(long progress, long total) {
Console.WriteLine("OnProgress : " + progress + "," + total);
})
.outputToFile(outputFile)
;
SelectObjectResult selectObjectResult = cosXml.selectObject(request);
Console.WriteLine(selectObjectResult.stat);
Assert.AreEqual(selectObjectResult.stat.BytesReturned, new FileInfo(outputFile).Length);
}
catch (COSXML.CosException.CosClientException clientEx)
{
Console.WriteLine("CosClientException: " + clientEx.StackTrace);
Console.WriteLine("CosClientException: " + clientEx.Message);
Assert.True(false);
}
catch (COSXML.CosException.CosServerException serverEx)
{
Console.WriteLine("CosServerException: " + serverEx.GetInfo());
Assert.True(false);
}
}
public void CSVSelectObjectQuoteCharacterTest()
{
const string key = "CSVSelectObjectQuoteCharacterTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("'abc','def\n123','456'\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _2 from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
request.InputFormat = inputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("'def\n'456'\n", Encoding.Default.GetString(buffer, 0, got));
//use '
inputFormat = new SelectObjectCSVInputFormat();
inputFormat.QuoteCharacter = "'";
request.InputFormat = inputFormat;
result = _ossClient.SelectObject(request);
got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("'def\n123'\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
/// 检索对象内容
public void SelectObject()
{
//.cssg-snippet-body-start:[select-object]
try
{
string bucket = "examplebucket-1250000000"; //存储桶，格式：BucketName-APPID
string key = "exampleobject"; //对象键
SelectObjectRequest request = new SelectObjectRequest(bucket, key);
ObjectSelectionFormat.JSONFormat jSONFormat = new ObjectSelectionFormat.JSONFormat();
jSONFormat.Type = "DOCUMENT";
jSONFormat.RecordDelimiter = "\n";
string outputFile = "select_local_file.json";
request.setExpression("Select * from COSObject")
.setInputFormat(new ObjectSelectionFormat(null, jSONFormat))
.setOutputFormat(new ObjectSelectionFormat(null, jSONFormat))
.SetCosProgressCallback(delegate(long progress, long total) {
Console.WriteLine("OnProgress : " + progress + "," + total);
})
.outputToFile(outputFile)
;
SelectObjectResult selectObjectResult = cosXml.selectObject(request);
Console.WriteLine(selectObjectResult.stat);
}
catch (COSXML.CosException.CosClientException clientEx)
{
Console.WriteLine("CosClientException: " + clientEx.StackTrace);
Console.WriteLine("CosClientException: " + clientEx.Message);
}
catch (COSXML.CosException.CosServerException serverEx)
{
Console.WriteLine("CosServerException: " + serverEx.GetInfo());
}
//.cssg-snippet-body-end
}
public void CSVSelectObjectLineRangeTest()
{
const string key = "CSVSelectObjectLineRangeTest";
try
{
var buffer = new byte[256];
byte[] binaryData = Encoding.ASCII.GetBytes("abc,def\n123,456\n789,efg\nhij,klm\n");
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var metaRequest = new CreateSelectObjectMetaRequest(_bucketName, key);
metaRequest.InputFormat = new CreateSelectObjectMetaCSVInputFormat();
_ossClient.CreateSelectObjectMeta(metaRequest);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select _1, _2 from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.Range = "line-range=1-2";
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
var got = result.Content.Read(buffer, 0, buffer.Length);
Assert.AreEqual("123,456\n789,efg\n", Encoding.Default.GetString(buffer, 0, got));
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
void CosXml.SelectObjectAsync(SelectObjectRequest request, OnSuccessCallback <CosResult> successCallback, OnFailedCallback failCallback)
{
Schedue(request, new Model.Object.SelectObjectResult(), successCallback, failCallback);
}
SelectObjectResult CosXml.SelectObject(SelectObjectRequest request)
{
return((Model.Object.SelectObjectResult)Excute(request, new Model.Object.SelectObjectResult()));
}
public SelectObjectRequestDeserializer(SelectObjectRequest selectObjectRequest)
: base(null)
{
_request = selectObjectRequest;
}
public IDeserializer <ServiceResponse, OssObject> CreateSelectObjectRequestDeserializer(SelectObjectRequest request)
{
return(new SelectObjectRequestDeserializer(request));
}
public void CSVSelectObjectNormalTest()
{
const string key = "SqlObjectWithCsvData";
try
{
byte[] binaryData = Encoding.ASCII.GetBytes(_sqlMessage);
var stream = new MemoryStream(binaryData);
_ossClient.PutObject(_bucketName, key, stream);
var request = new SelectObjectRequest(_bucketName, key);
request.Expression = "select * from ossobject";
var inputFormat = new SelectObjectCSVInputFormat();
inputFormat.FileHeaderInfo = FileHeaderInfo.Use;
inputFormat.RecordDelimiter = "\r\n";
inputFormat.FieldDelimiter = ",";
inputFormat.QuoteCharacter = "\"";
inputFormat.CommentCharacter = "#";
inputFormat.CompressionType = CompressionType.None;
request.InputFormat = inputFormat;
var outputFormat = new SelectObjectCSVOutputFormat();
outputFormat.EnablePayloadCrc = true;
outputFormat.OutputRawData = false;
outputFormat.KeepAllColumns = false;
outputFormat.OutputHeader = false;
outputFormat.FieldDelimiter = ",";
outputFormat.RecordDelimiter = "\n";
request.OutputFormat = outputFormat;
var result = _ossClient.SelectObject(request);
string expect = "Lora Francis,School A,Staples Inc,27\n" +
"Eleanor Little,School B,\"Conectiv, Inc\",43\n" +
"Rosie Hughes,School C,Western Gas Resources Inc,44\n" +
"Lawrence Ross,School D,MetLife Inc.,24\n";
var buf = new byte[256];
int offset = 0;
for (int i = 1; i < 30; i++)
{
int got = result.Content.Read(buf, offset, i);
offset += got;
}
Assert.AreEqual(expect.Length, offset);
string str = Encoding.Default.GetString(buf, 0, offset);
Assert.AreEqual(expect, str);
}
finally
{
_ossClient.DeleteObject(_bucketName, key);
}
}
