public override void Prepare()
{
// Connection should be open at this point, but just in case
using (DataManager.Current.OpenConnection())
{
using (SqlBulkCopy bulk = DataManager.CreateBulkCopy(SqlBulkCopyOptions.Default))
{
bool tableInit = true;
bulk.BatchSize = BulkBatchSize;
bulk.DestinationTableName = "TEMP_ImportedData";
using (RowReader<ImportFileRow> reader = Activator.CreateInstance(_readerType))
{
while (reader.Read())
{
if (tableInit)
{
// Init the temp table using the first row
SqlCommand createTableCmd = DataManager.CreateCommand(
"create table #TEMP_ImportedData
}
//bulk.WriteToServer(
}
}
}
}
}
public static TestDoc FromHybridRowStream(Stream stream)
{
uint length = 0;
using (BinaryReader binaryReader = new BinaryReader(stream, Encoding.Default, leaveOpen: true))
{
TestDoc.SkipBinaryField(binaryReader); // binaryId
TestDoc.SkipBinaryField(binaryReader); // EPK
binaryReader.ReadByte();
length = binaryReader.ReadUInt32();
}
RowBuffer row = new RowBuffer((int)length);
Assert.IsTrue(row.ReadFrom(stream, (int)length, HybridRowVersion.V1, BatchTestBase.LayoutResolver));
RowReader reader = new RowReader(ref row);
TestDoc testDoc = new TestDoc();
while (reader.Read())
{
Result r;
switch (reader.Path)
{
case "Id":
r = reader.ReadString(out string id);
Assert.AreEqual(Result.Success, r);
testDoc.Id = id;
break;
case "Cost":
r = reader.ReadInt32(out int cost);
Assert.AreEqual(Result.Success, r);
testDoc.Cost = cost;
break;
case "Status":
r = reader.ReadString(out string status);
Assert.AreEqual(Result.Success, r);
testDoc.Status = status;
break;
case "Description":
r = reader.ReadString(out string description);
Assert.AreEqual(Result.Success, r);
testDoc.Description = description;
break;
}
}
return(testDoc);
}
protected override IEnumerable <FullName> Read(RowReader reader)
{
using (reader)
while (reader.Read())
{
yield return new FullName
{
First = reader.Get <string>("First"),
Last = reader.Get <string>("Last")
}
}
;
}
}
public void TestWriteOperationWithBinaryIdByteArray()
{
ISpanResizer <byte> resizer = new MemorySpanResizer <byte>(100);
RowBuffer row = new RowBuffer(capacity: 100, resizer: resizer);
row.InitLayout(HybridRowVersion.V1, BatchSchemaProvider.BatchOperationLayout, BatchSchemaProvider.BatchLayoutResolver);
byte[] testBinaryId = new byte[] { 1, 2, 3, 4, };
ItemRequestOptions requestOptions = new();
requestOptions.Properties = new Dictionary <string, object>()
{
{ WFConstants.BackendHeaders.BinaryId, testBinaryId },
};
TransactionalBatchItemRequestOptions transactionalBatchItemRequestOptions =
TransactionalBatchItemRequestOptions.FromItemRequestOptions(requestOptions);
ItemBatchOperation operation = new ItemBatchOperation(
operationType: OperationType.Patch,
operationIndex: 0,
partitionKey: Cosmos.PartitionKey.Null,
requestOptions: transactionalBatchItemRequestOptions);
int length = operation.GetApproximateSerializedLength();
Assert.AreEqual(testBinaryId.Length, length);
Result r = RowWriter.WriteBuffer(ref row, operation, ItemBatchOperation.WriteOperation);
if (r != Result.Success)
{
Assert.Fail(r.ToString());
}
bool foundBinaryId = false;
RowReader reader = new RowReader(ref row);
while (reader.Read())
{
if (reader.PathSpan == Utf8String.TranscodeUtf16("binaryId"))
{
foundBinaryId = true;
reader.ReadBinary(out byte[] binaryId);
CollectionAssert.AreEqual(testBinaryId, binaryId);
}
}
Assert.IsTrue(foundBinaryId);
}
private void ProcessData(Delivery delivery)
{
Type objType = null;
switch (Instance.Configuration.Options["ReportReaderType"])
{
case "BingKeywordReportReader":
objType = typeof(BingKeywordReportReader);
break;
case "BingAdPerformanceReportReader":
objType = typeof(BingAdPerformanceReportReader);
break;
}
string[] filesPath = new string[2];
for (int i = 0; delivery.Files.Count >= i + 1; i++)
{
Type dRederType = Type.GetType(delivery.Files[i].ReaderType);
if (dRederType == typeof(BingAdPerformanceReportReader))
{
filesPath[1] = delivery.Files[i].FilePath;
}
else if (dRederType == typeof(BingKeywordReportReader))
{
filesPath[0] = delivery.Files[i].FilePath;
}
}
using (RowReader <PpcDataUnit> reader = (RowReader <PpcDataUnit>)Activator.CreateInstance(objType, new object[] { filesPath }))
{
using (DataManager.Current.OpenConnection())
{
DataManager.Current.StartTransaction();
while (reader.Read())
{
if (reader.CurrentRow.AccountID > 0)
{
reader.CurrentRow.Save();
}
}
DataManager.Current.CommitTransaction();
}
}
}
private static Result ReadOperationResult(ref RowReader reader, out BatchOperationResult batchOperationResult)
{
batchOperationResult = new BatchOperationResult();
while (reader.Read())
{
Result r;
switch (reader.Path)
{
case "statusCode":
r = reader.ReadInt32(out int statusCode);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.StatusCode = (HttpStatusCode)statusCode;
break;
case "subStatusCode":
r = reader.ReadInt32(out int subStatusCode);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.SubStatusCode = (SubStatusCodes)subStatusCode;
break;
case "eTag":
r = reader.ReadString(out string eTag);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.ETag = eTag;
break;
case "resourceBody":
r = reader.ReadBinary(out byte[] resourceBody);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.ResourceStream = new MemoryStream(
buffer: resourceBody, index: 0, count: resourceBody.Length, writable: false, publiclyVisible: true);
break;
case "retryAfterMilliseconds":
r = reader.ReadUInt32(out uint retryAfterMilliseconds);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.RetryAfter = TimeSpan.FromMilliseconds(retryAfterMilliseconds);
break;
}
}
return(Result.Success);
}
private static Result ReadOperationResult(ref RowReader reader, out TransactionalBatchOperationResult batchOperationResult)
{
batchOperationResult = new TransactionalBatchOperationResult();
while (reader.Read())
{
Result r;
switch (reader.Path)
{
case "statusCode":
r = reader.ReadInt32(out int statusCode);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.StatusCode = (HttpStatusCode)statusCode;
break;
case "subStatusCode":
r = reader.ReadInt32(out int subStatusCode);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.SubStatusCode = (SubStatusCodes)subStatusCode;
break;
case "eTag":
r = reader.ReadString(out string eTag);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.ETag = eTag;
break;
case "resourceBody":
r = reader.ReadBinary(out byte[] resourceBody);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.ResourceStream = new MemoryStream(
buffer: resourceBody, index: 0, count: resourceBody.Length, writable: false, publiclyVisible: true);
break;
case "requestCharge":
r = reader.ReadFloat64(out double requestCharge);
if (r != Result.Success)
{
return(r);
}
// Round request charge to 2 decimals on the operation results
// similar to how we round them for the full response.
batchOperationResult.RequestCharge = Math.Round(requestCharge, 2);
break;
case "retryAfterMilliseconds":
r = reader.ReadUInt32(out uint retryAfterMilliseconds);
if (r != Result.Success)
{
return(r);
}
batchOperationResult.RetryAfter = TimeSpan.FromMilliseconds(retryAfterMilliseconds);
break;
}
}
return(Result.Success);
}
public IEnumerable <TModel> Read()
{
_errors.Clear();
var rowIndex = 0;
var firstRow = RowReader.Read();
if (firstRow == null)
{
if (HasHeader)
{
throw new TableIOException(new[] { new ErrorDetail
{
Type = "NoTableHeader",
Message = "Table header is none."
} });
}
else
{
yield break;
}
}
// decide valid column size.
// ** all row's column size must be valid column size. **
var validColumnSize = ColumnSize ?? firstRow.Count;
if (PropertyMapper.RequiredHeaderOnRead && !HasHeader)
{
throw new TableIOException(new[] { new ErrorDetail
{
Type = "HeaderRequired",
Message = $"Header is required on read.",
RowIndex = rowIndex
} });
}
var propertyMaps = PropertyMapper.CreatePropertyMaps(typeof(TModel), HasHeader ? firstRow.Select(f => $"{f}").ToArray() : null);
var propertyMapMaxColumnIndex = propertyMaps.Any() ? propertyMaps.Max(m => m.ColumnIndex) : -1;
if (propertyMapMaxColumnIndex >= validColumnSize)
{
throw new TableIOException(new[] { new ErrorDetail
{
Type = "OutOfRangeColumnIndexMapping",
Message = $"Max column index({propertyMapMaxColumnIndex}) of property mapping is greater than or equal to valid column size({validColumnSize}).",
RowIndex = rowIndex
} });
}
var row = firstRow;
if (HasHeader)
{
rowIndex++;
row = RowReader.Read();
}
while (row != null)
{
if (row.Count != validColumnSize)
{
_errors.Add(new ErrorDetail
{
Type = "InvalidColumnSize",
Message = "Column size is invalid.",
RowIndex = rowIndex
});
if (_errors.Count >= ErrorLimit)
{
throw new TableIOException(_errors);
}
}
else
{
yield return(ConvertFromRow(row, rowIndex, propertyMaps));
}
rowIndex++;
row = RowReader.Read();
}
if (_errors.Any())
{
throw new TableIOException(_errors);
}
}
private static Result ReadOperation(ref RowReader reader, int operationIndex, out ItemBatchOperation operation)
{
operation = null;
OperationType operationType = OperationType.Invalid;
string partitionKeyJson = null;
byte[] effectivePartitionKey = null;
string id = null;
byte[] binaryId = null;
byte[] resourceBody = null;
Cosmos.IndexingDirective?indexingDirective = null;
string ifMatch = null;
string ifNoneMatch = null;
int? ttlInSeconds = null;
while (reader.Read())
{
Result r;
switch (reader.Path)
{
case "operationType":
r = reader.ReadInt32(out int operationTypeInt);
if (r != Result.Success)
{
return(r);
}
operationType = (OperationType)operationTypeInt;
break;
case "resourceType":
r = reader.ReadInt32(out int resourceType);
if (r != Result.Success)
{
return(r);
}
Assert.AreEqual(ResourceType.Document, (ResourceType)resourceType);
break;
case "partitionKey":
r = reader.ReadString(out partitionKeyJson);
if (r != Result.Success)
{
return(r);
}
break;
case "effectivePartitionKey":
r = reader.ReadBinary(out effectivePartitionKey);
if (r != Result.Success)
{
return(r);
}
break;
case "id":
r = reader.ReadString(out id);
if (r != Result.Success)
{
return(r);
}
break;
case "binaryId":
r = reader.ReadBinary(out binaryId);
if (r != Result.Success)
{
return(r);
}
break;
case "resourceBody":
r = reader.ReadBinary(out resourceBody);
if (r != Result.Success)
{
return(r);
}
break;
case "indexingDirective":
r = reader.ReadString(out string indexingDirectiveStr);
if (r != Result.Success)
{
return(r);
}
if (!Enum.TryParse <Cosmos.IndexingDirective>(indexingDirectiveStr, out Cosmos.IndexingDirective indexingDirectiveEnum))
{
return(Result.Failure);
}
indexingDirective = indexingDirectiveEnum;
break;
case "ifMatch":
r = reader.ReadString(out ifMatch);
if (r != Result.Success)
{
return(r);
}
break;
case "ifNoneMatch":
r = reader.ReadString(out ifNoneMatch);
if (r != Result.Success)
{
return(r);
}
break;
case "timeToLiveInSeconds":
r = reader.ReadInt32(out int ttl);
if (r != Result.Success)
{
return(r);
}
ttlInSeconds = ttl;
break;
}
}
// Ensure the mandatory fields were populated
if (operationType == OperationType.Invalid)
{
return(Result.Failure);
}
TransactionalBatchItemRequestOptions requestOptions = null;
if (indexingDirective.HasValue || ifMatch != null || ifNoneMatch != null || binaryId != null || effectivePartitionKey != null || ttlInSeconds.HasValue)
{
requestOptions = new TransactionalBatchItemRequestOptions();
if (indexingDirective.HasValue)
{
requestOptions.IndexingDirective = indexingDirective;
}
if (ifMatch != null)
{
requestOptions.IfMatchEtag = ifMatch;
}
else if (ifNoneMatch != null)
{
requestOptions.IfNoneMatchEtag = ifNoneMatch;
}
if (binaryId != null || effectivePartitionKey != null || ttlInSeconds.HasValue)
{
requestOptions.Properties = new Dictionary <string, object>();
if (binaryId != null)
{
requestOptions.Properties.Add(WFConstants.BackendHeaders.BinaryId, binaryId);
}
if (effectivePartitionKey != null)
{
requestOptions.Properties.Add(WFConstants.BackendHeaders.EffectivePartitionKey, effectivePartitionKey);
}
if (ttlInSeconds.HasValue)
{
requestOptions.Properties.Add(WFConstants.BackendHeaders.TimeToLiveInSeconds, ttlInSeconds.ToString());
}
}
}
Documents.PartitionKey parsedPartitionKey = null;
if (partitionKeyJson != null)
{
parsedPartitionKey = Documents.PartitionKey.FromJsonString(partitionKeyJson);
}
operation = new ItemBatchOperation(
operationType: operationType,
operationIndex: operationIndex,
id: id,
requestOptions: requestOptions)
{
ParsedPartitionKey = parsedPartitionKey,
ResourceBody = resourceBody
};
return(Result.Success);
}