public override void ProcessInput(int inputID, PipelineBuffer buffer) { bool pbCancel = false; if (String.IsNullOrEmpty(ComponentMetaData.CustomPropertyCollection[MONGO_COLL].Value)) { ComponentMetaData.FireError(0, "ProcessInput", "Missing MongoDB collection name", "", 0, out pbCancel); return; } int iBatchSize; if (!Int32.TryParse(ComponentMetaData.CustomPropertyCollection[MONGO_BATCH_SIZE].Value, out iBatchSize)) { ComponentMetaData.FireError(0, "ProcessInput", "Failed to parse MongoDB insert batch size", "", 0, out pbCancel); return; } MongoCollection collection = _db.GetCollection(ComponentMetaData.CustomPropertyCollection[MONGO_COLL].Value); List <BsonDocument> batch = new List <BsonDocument>(); int j = 1; // Create an empty template document using input column specification BsonInsertDocument template = new BsonInsertDocument(_colInfos); BsonDocument doc = template.Create(); while (buffer.NextRow()) { doc = template.DeepClone(); for (int i = 0; i < _colInfos.Count; i++) { doc.InsertDotNotated(_colInfos[i], CreateBsonValue(buffer[_colInfos[i].BufferColumnIndex], _colInfos[i])); } batch.Add(doc); j++; if (j > iBatchSize) { collection.InsertBatch(batch); ComponentMetaData.IncrementPipelinePerfCounter(103, (uint)batch.Count); j = 1; batch = new List <BsonDocument>(); } } if (batch.Count > 0) { collection.InsertBatch(batch); ComponentMetaData.IncrementPipelinePerfCounter(103, (uint)batch.Count); } }
public override void ProcessInput(int inputID, PipelineBuffer buffer) { bool pbCancel = false; if (String.IsNullOrEmpty(ComponentMetaData.CustomPropertyCollection[MONGO_COLL].Value)) { ComponentMetaData.FireError(0, "ProcessInput", "Missing MongoDB collection name", "", 0, out pbCancel); return; } int iBatchSize; if (!Int32.TryParse(ComponentMetaData.CustomPropertyCollection[MONGO_BATCH_SIZE].Value, out iBatchSize)) { ComponentMetaData.FireError(0, "ProcessInput", "Failed to parse MongoDB insert batch size", "", 0, out pbCancel); return; } MongoCollection collection = _db.GetCollection(ComponentMetaData.CustomPropertyCollection[MONGO_COLL].Value); List<BsonDocument> batch = new List<BsonDocument>(); int j = 1; // Create an empty template document using input column specification BsonInsertDocument template = new BsonInsertDocument(_colInfos); BsonDocument doc = template.Create(); while (buffer.NextRow()) { doc = template.DeepClone(); for (int i = 0; i < _colInfos.Count; i++) { doc.InsertDotNotated(_colInfos[i], CreateBsonValue(buffer[_colInfos[i].BufferColumnIndex], _colInfos[i])); } batch.Add(doc); j++; if (j > iBatchSize) { collection.InsertBatch(batch); ComponentMetaData.IncrementPipelinePerfCounter(103, (uint)batch.Count); j = 1; batch = new List<BsonDocument>(); } } if (batch.Count > 0) { collection.InsertBatch(batch); ComponentMetaData.IncrementPipelinePerfCounter(103, (uint)batch.Count); } }