public ReaderLoadTest(ReaderAdapter adapter, BlockingQueue <QueryDescription> queries)
: base(adapter)
{
Contract.Requires(queries != null);
_adapter = adapter;
_queries = queries;
}
private DatabaseReader(string name, string connectionString, SchemaParameters schemaParameters)
{
_db = new DatabaseSchema(connectionString, name);
_schemaParameters = schemaParameters;
_schemaParameters.DatabaseSchema = _db;
_readerAdapter = ReaderAdapterFactory.Create(_schemaParameters);
}
/// <summary>
/// Initializes a new instance of the <see cref="DatabaseReader"/> class from a DbConnection.
/// </summary>
/// <param name="connection">The connection.</param>
public DatabaseReader(System.Data.Common.DbConnection connection)
{
var name = connection.GetType().Namespace;
_db = new DatabaseSchema(connection.ConnectionString, name);
_schemaParameters = new SchemaParameters(connection)
{
DatabaseSchema = _db
};
_readerAdapter = ReaderAdapterFactory.Create(_schemaParameters);
}
/// <summary>
/// Initializes a new instance of the <see cref="DatabaseReader"/> class for one of the standard providers.
/// </summary>
/// <param name="connectionString">The connection string.</param>
/// <param name="sqlType">Type of the SQL.</param>
public DatabaseReader(string connectionString, SqlType sqlType)
{
if (connectionString == null)
{
throw new ArgumentNullException("connectionString");
}
_schemaParameters = new SchemaParameters(connectionString, sqlType);
_readerAdapter = ReaderAdapterFactory.Create(_schemaParameters);
//_schemaReader = SchemaReaderFactory.Create(connectionString, sqlType);
_db = new DatabaseSchema(connectionString, _schemaParameters.ProviderName);
_schemaParameters.DatabaseSchema = _db;
}
/// <summary>
/// Initializes a new instance of the <see cref="DatabaseReader"/> class for one of the standard providers.
/// </summary>
/// <param name="connectionString">The connection string.</param>
/// <param name="sqlType">Type of the SQL.</param>
/// <param name="additionalParameters"></param>
public DatabaseReader(string connectionString, SqlType sqlType, int commandTimeout, IAdditionalProperties additionalParameters = null)
{
if (connectionString == null)
{
throw new ArgumentNullException("connectionString");
}
_schemaParameters = new SchemaParameters(connectionString, sqlType);
_readerAdapter = ReaderAdapterFactory.Create(_schemaParameters, commandTimeout);
_readerAdapter.AdditionalParameters = additionalParameters;
//_schemaReader = SchemaReaderFactory.Create(connectionString, sqlType);
_db = new DatabaseSchema(connectionString, _schemaParameters.ProviderName);
_schemaParameters.DatabaseSchema = _db;
}
/// <summary>
/// Initializes a new instance of the <see cref="DatabaseReader"/> class using an existing <see cref="DatabaseSchema"/>.
/// </summary>
/// <param name="databaseSchema">The database schema. Can be a subclassed version.</param>
public DatabaseReader(DatabaseSchema databaseSchema)
{
if (databaseSchema == null)
{
throw new ArgumentNullException("databaseSchema");
}
if (databaseSchema.ConnectionString == null)
{
throw new ArgumentException("No connectionString");
}
_schemaParameters = new SchemaParameters(databaseSchema.ConnectionString, databaseSchema.Provider);
_schemaParameters.DatabaseSchema = databaseSchema;
_schemaParameters.Owner = databaseSchema.Owner;
_readerAdapter = ReaderAdapterFactory.Create(_schemaParameters);
_db = databaseSchema;
}
/// <summary>
/// Initializes a new instance of the <see cref="DatabaseReader"/> class using an existing <see cref="DatabaseSchema"/>.
/// </summary>
/// <param name="databaseSchema">The database schema. Can be a subclassed version.</param>
/// <param name="commandTimeout"></param>
/// <param name="additionalParameters"></param>
public DatabaseReader(DatabaseSchema databaseSchema, int commandTimeout, IAdditionalProperties additionalParameters = null)
{
if (databaseSchema == null)
{
throw new ArgumentNullException("databaseSchema");
}
if (databaseSchema.ConnectionString == null)
{
throw new ArgumentException("No connectionString");
}
_schemaParameters = new SchemaParameters(databaseSchema.ConnectionString, databaseSchema.Provider);
_schemaParameters.DatabaseSchema = databaseSchema;
_schemaParameters.Owner = databaseSchema.Owner;
_readerAdapter = ReaderAdapterFactory.Create(_schemaParameters, commandTimeout);
_readerAdapter.AdditionalParameters = additionalParameters;
_db = databaseSchema;
}
protected override MetricsUnit GetSampleMetrics()
{
try
{
// load sample file, read only one row in order to create metrics table by sample metric unit
ReaderAdapter.Init(FileManager.Open(Configuration.SampleFilePath), Configuration);
ReaderAdapter.Reader.Read();
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
return(CurrentMetricsUnit);
}
catch (Exception ex)
{
throw new ConfigurationErrorsException(String.Format("Failed to create metrics by sample file '{0}', ex: {1}", Configuration.SampleFilePath, ex.Message));
}
}
protected override ServiceOutcome DoPipelineWork()
{
InitMappings();
LoadConfiguration();
// Import data
using (ReaderAdapter)
{
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, _importManagerOptions)
{
OnLog = Log
})
{
// create objects tables and metrics table according to sample metrics
ImportManager.BeginImport(Delivery, GetSampleMetrics());
Log("ImportManager.BeginImport() executed successfully", LogMessageType.Debug);
// open delivery file
using (var stream = _deliveryFile.OpenContents(compression: _compression))
{
ReaderAdapter.Init(stream, Configuration);
// for each row in file read and import into metrics table
var readSuccess = false;
while (CheckEndOfFile())
{
readSuccess = true;
ProcessMetrics();
}
if (!readSuccess)
{
Log("Could Not read data from file!, check file mapping and configuration", Core.Utilities.LogMessageType.Warning);
}
ImportManager.EndImport();
Log("ImportManager.EndImport() executed successfully", LogMessageType.Debug);
}
}
}
return(ServiceOutcome.Success);
}
public ProcedureBuilder(ReaderAdapter readerAdapter, DatabaseSchema databaseSchema, Exclusions exclusions)
{
_readerAdapter = readerAdapter;
_databaseSchema = databaseSchema;
_exclusions = exclusions;
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
// Setup/defaults/configuration/etc.
// ------------------------------------------
string checksumThreshold = Configuration.Parameters.Get <T>(Consts.ConfigurationOptions.ChecksumTheshold);
var importManagerOptions = new MetricsDeliveryManagerOptions()
{
SqlTransformCommand = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlTransformCommand),
SqlStageCommand = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlStageCommand),
SqlRollbackCommand = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlRollbackCommand),
ChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold)
};
string fileName;
if (!this.Configuration.Options.TryGetValue(Const.DeliveryServiceConfigurationOptions.DeliveryFileName, out fileName))
{
throw new ConfigurationException(String.Format("{0} is missing in the service configuration options.", Const.DeliveryServiceConfigurationOptions.DeliveryFileName));
}
DeliveryFile file = this.Delivery.Files[fileName];
if (file == null)
{
throw new Exception(String.Format("Could not find delivery file '{0}' in the delivery.", fileName));
}
FileCompression compression;
string compressionOption;
if (this.Configuration.Options.TryGetValue(Const.DeliveryServiceConfigurationOptions.Compression, out compressionOption))
{
if (!Enum.TryParse <FileCompression>(compressionOption, out compression))
{
throw new ConfigurationException(String.Format("Invalid compression type '{0}'.", compressionOption));
}
}
else
{
compression = FileCompression.None;
}
// Create format processor from configuration
string adapterTypeName = Configuration.GetOption(Consts.ConfigurationOptions.ReaderAdapterType);
Type readerAdapterType = Type.GetType(adapterTypeName, true);
this.ReaderAdapter = (ReaderAdapter)Activator.CreateInstance(readerAdapterType);
this.Mappings.OnFieldRequired = this.ReaderAdapter.GetField;
LoadConfiguration();
// ------------------------------------------
using (var stream = file.OpenContents(compression: compression))
{
using (this.ReaderAdapter)
{
this.ReaderAdapter.Init(stream, Configuration);
using (this.ImportManager = CreateImportManager(InstanceID, importManagerOptions))
{
this.ImportManager.BeginImport(this.Delivery);
bool readSuccess = false;
while (this.ReaderAdapter.Reader.Read())
{
readSuccess = true;
OnRead();
}
if (!readSuccess)
{
Edge.Core.Utilities.Log("Could Not read data from file!, check file mapping and configuration", Core.Utilities.LogMessageType.Warning);
}
this.ImportManager.EndImport();
}
}
}
return(Core.Services.ServiceOutcome.Success);
}
public TableBuilder(ReaderAdapter readerAdapter)
{
_readerAdapter = readerAdapter;
}
public ViewBuilder(ReaderAdapter readerAdapter, Exclusions exclusions)
{
_readerAdapter = readerAdapter;
_exclusions = exclusions;
}
protected override ServiceOutcome DoPipelineWork()
{
// init mapping and load configuration
Log("Starting Google.AdWords.CampaignCriterionProcessorService", LogMessageType.Debug);
InitMappings();
Mappings.OnMappingApplied = SetEdgeType;
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings))
{
throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object");
}
LoadConfiguration();
Progress = 0.1;
// get nessesary EdgeFields
var relationField = GetEdgeFieldByName("Relation");
var locationField = GetEdgeFieldByName("Location");
var languageField = GetEdgeFieldByName("Language");
var isLocationNegativeField = GetEdgeFieldByName("IsLocationNegative");
var isLanguageNegativeField = GetEdgeFieldByName("IsLanguageNegative");
using (ReaderAdapter)
{
using (var stream = _deliveryFile.OpenContents(compression: _compression))
{
ReaderAdapter.Init(stream, Configuration);
while (ReaderAdapter.Reader.Read())
{
// load metrics unit which contains Relation object of Campaign-Language or Campaign-Location
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
// insert loaded relation to List per Campaign
var relation = CurrentMetricsUnit.Dimensions[relationField] as RelationObject;
if (relation != null)
{
var campaign = relation.Object1 as Campaign;
if (campaign != null)
{
if (!_campaignRelationMap.ContainsKey(campaign.OriginalID))
{
_campaignRelationMap.Add(campaign.OriginalID, new List <RelationObject>());
}
_campaignRelationMap[campaign.OriginalID].Add(relation);
}
}
}
}
}
Progress = 0.5;
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, _importManagerOptions)
{
OnLog = Log
})
{
// create object tables
ImportManager.BeginImport(Delivery, null);
// add objects to EdgeObjectsManager cache
PrepareImportObjects(x => x is Location, locationField, isLocationNegativeField);
PrepareImportObjects(x => x is Language, languageField, isLanguageNegativeField);
// import objects
ImportManager.EndImport();
}
return(ServiceOutcome.Success);
}
private bool CheckEndOfFile()
{
// if defined EOF indication read till the indication, otherwise read till the end of the file
return((_eofIndication != null && _eofFieldName != null) ? ReaderAdapter.Reader.Read() && ReaderAdapter.GetField(_eofFieldName).ToString() != _eofIndication :
ReaderAdapter.Reader.Read());
}
