/*=========================*/
protected override void OnBeginImport(MetricsUnit sampleMetrics)
{
// set table prefix
_tablePrefix = string.Format("{0}_{1}_{2}_{3}", CurrentDelivery.Account.ID, CurrentDelivery.Name, DateTime.Now.ToString("yyyMMdd_HHmmss"), CurrentDelivery.DeliveryID.ToString("N").ToLower());
CurrentDelivery.Parameters[Consts.DeliveryHistoryParameters.TablePerfix] = _tablePrefix;
// create delivery object tables (should be Usid instead of GK)
_edgeObjectsManager.CreateDeliveryObjectTables(_tablePrefix);
Log(String.Format("Delivery object tables created for delivery {0}", CurrentDelivery.DeliveryID));
// create metrics table using metrics table manager and sample metrics
_metricsTableManager.CreateDeliveryMetricsTable(_tablePrefix, sampleMetrics);
Log(String.Format("Delivery Metrics table '{0}' created for delivery {1}", _metricsTableManager.TableName, CurrentDelivery.DeliveryID));
// store delivery and staging (best match) table names in delivery
CurrentDelivery.Parameters[Consts.DeliveryHistoryParameters.DeliveryMetricsTableName] = _metricsTableManager.TableName;
CurrentDelivery.Parameters[Consts.DeliveryHistoryParameters.StagingMetricsTableName] = _metricsTableManager.FindStagingTable();
Log(String.Format("Best match metrics table is '{0}' for delivery {1}", CurrentDelivery.Parameters[Consts.DeliveryHistoryParameters.StagingMetricsTableName], CurrentDelivery.DeliveryID));
// CHECKSUMMANAGER: setup
// MAPPER: setup bulks for objects and metrics
}
protected override MetricsUnit GetSampleMetrics()
{
LoadCampaigns(new DeliveryFile {
Location = Configuration.Parameters.Get <string>("CampaignSampleFile")
});
LoadCreatives(new DeliveryFile {
Location = Configuration.Parameters.Get <string>("CreativeSampleFile")
});
LoadAds(new DeliveryFile {
Location = Configuration.Parameters.Get <string>("AdGroupSampleFile")
});
using (var reader = new JsonDynamicReader(Configuration.SampleFilePath, "$.data[*].*"))
{
Mappings.OnFieldRequired = fieldName => reader.Current[fieldName];
if (reader.Read())
{
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
return(CurrentMetricsUnit);
}
}
throw new ConfigurationErrorsException(String.Format("Failed to read sample metrics from file: {0}", Configuration.SampleFilePath));
}
protected void LoadCurrentMetrics()
{
// fill the metrics using mapping
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField
};
MetricsMappings.Apply(CurrentMetricsUnit);
var signature = new Signature();
SignatureMappings.Apply(signature);
// check if signature is already exists in delivery outputs
var output = Delivery.Outputs.FirstOrDefault(x => x.Signature == signature.Value);
// attach output to Metrics: take existing or create new
if (output != null)
{
CurrentMetricsUnit.Output = output;
}
else
{
var deliveryOutput = new DeliveryOutput
{
Signature = signature.Value,
TimePeriodStart = CurrentMetricsUnit.TimePeriodStart,
TimePeriodEnd = CurrentMetricsUnit.TimePeriodEnd,
Account = CurrentMetricsUnit.Account,
Channel = CurrentMetricsUnit.Channel
};
Delivery.Outputs.Add(deliveryOutput);
CurrentMetricsUnit.Output = deliveryOutput;
}
}
protected override MetricsUnit GetSampleMetrics()
{
var headers = new[] { AdWordsConst.AdPreRequiredHeader };
// load sample keywords
var file = new DeliveryFile {
Location = Configuration.Parameters.Get <string>("KeywordSampleFile")
};
LoadKeywords(file, headers, FileCompression.None);
file = new DeliveryFile {
Location = Configuration.Parameters.Get <string>("PlacementSampleFile")
};
LoadPlacements(file, new[] { AdWordsConst.AutoPlacRequiredHeader }, FileCompression.None);
// load ad
using (_adsReader = new CsvDynamicReader(Configuration.Parameters.Get <string>("AdSampleFile"), headers))
{
Mappings.OnFieldRequired = fieldName => _adsReader.Current[fieldName];
if (_adsReader.Read())
{
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
return(CurrentMetricsUnit);
}
}
throw new ConfigurationErrorsException(String.Format("Failed to read sample metrics from file: {0}", Configuration.Parameters.Get <string>("AdSampleFile")));
}
public void ImportMetrics(DeliveryOutput targetOutput, MetricsUnit metrics)
{
EnsureBeginImport();
foreach (EdgeObject obj in metrics.GetObjectDimensions())
{
}
}
public void BeginImport(Delivery delivery, MetricsUnit sampleMetrics)
{
ThrowIfNotIdle();
State = DeliveryManagerState.Importing;
CurrentDelivery = delivery;
OnBeginImport(sampleMetrics);
}
/// <summary>
/// Create flat list of objects which compose metrics data object
/// </summary>
/// <param name="metricsUnit"></param>
/// <returns></returns>
public List <object> GetFlatObjectList(MetricsUnit metricsUnit)
{
// clear all containers
ClearObjects();
var level = 0;
var flatList = new List <object>();
// add all metrics dimensions to object structure
foreach (var dimension in metricsUnit.GetObjectDimensions().Where(dimension => dimension != null))
{
Add(dimension, level);
}
// go over object composition levels and add objects to flat list
while (ContainsKey(level) && this[level] != null && this[level].Count > 0)
{
// add all objects of this level to flat list (check if it not exists already)
foreach (var obj in this[level])
{
var objDim = obj as ObjectDimension;
if (objDim == null)
{
continue;
}
if (objDim.Field == null || !flatList.Any(x => x is ObjectDimension &&
((x as ObjectDimension).Field == objDim.Field || // check if object already added by field definition
(x as ObjectDimension).Value == objDim.Value) // check if object already added by object value
)) // (for example creative ref in creative definition should not be added, already added in composite creative)
{
flatList.Add(obj);
}
}
foreach (var obj in this[level])
{
var dimension = obj as ObjectDimension;
if (dimension != null && dimension.Value is EdgeObject)
{
// add all dimensions of the EdgeObject
foreach (var childDimension in (dimension.Value as EdgeObject).GetObjectDimensions())
{
Add(childDimension, level + 1);
}
}
}
level++;
}
// add measures
if (metricsUnit.MeasureValues != null)
{
flatList.AddRange(metricsUnit.MeasureValues.Cast <object>());
}
return(Normalize(flatList, metricsUnit));
}
private MetricsUnit CreateMetricsUnitNoMapping(CsvDynamicReader adsReader)
{
// get already existing or create new Ad
string adId = adsReader.Current[AdWordsConst.AdIDFieldName];
Ad ad = _importedAds.ContainsKey(adId) ? _importedAds[adId] : CreateAd(adsReader);
// create metrics unit
var metricsUnit = new MetricsUnit
{
GetEdgeField = GetEdgeField,
Ad = ad,
Channel = GetChannel("Google"),
Account = GetAccount("Bbinary"),
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd,
Currency = new EdgeCurrency {
Code = Convert.ToString(adsReader.Current.Currency)
}
};
// add keyword or placement as a target to metrics
var target = GetKeywordTarget();
metricsUnit.Dimensions.Add(GetTargetField("TargetMatch"), new TargetMatch
{
Target = target,
EdgeType = GetEdgeType("TargetMatch"),
TK = target.TK
});
// metrics measures
metricsUnit.MeasureValues = new Dictionary <Measure, double>();
metricsUnit.MeasureValues.Add(GetMeasure("Clicks"), Convert.ToInt64(adsReader.Current.Clicks));
metricsUnit.MeasureValues.Add(GetMeasure("Cost"), Convert.ToDouble(adsReader.Current.Cost) / 1000000);
metricsUnit.MeasureValues.Add(GetMeasure("Impressions"), Convert.ToInt64(adsReader.Current.Impressions));
metricsUnit.MeasureValues.Add(GetMeasure("AveragePosition"), Convert.ToDouble(adsReader.Current[AdWordsConst.AvgPositionFieldName]));
metricsUnit.MeasureValues.Add(GetMeasure(_googleMeasuresDic[AdWordsConst.ConversionOnePerClickFieldName]), Convert.ToDouble(adsReader.Current[AdWordsConst.ConversionOnePerClickFieldName]));
metricsUnit.MeasureValues.Add(GetMeasure(_googleMeasuresDic[AdWordsConst.ConversionManyPerClickFieldName]), Convert.ToDouble(adsReader.Current[AdWordsConst.ConversionManyPerClickFieldName]));
// add conversion values
string conversionKey = String.Format("{0}#{1}", ad.OriginalID, adsReader.Current[AdWordsConst.KeywordIdFieldName]);
if (_importedAdsWithConv.ContainsKey(conversionKey))
{
var conversionDic = _importedAdsWithConv[conversionKey];
foreach (var pair in conversionDic.Where(pair => _googleMeasuresDic.ContainsKey(pair.Key)))
{
metricsUnit.MeasureValues.Add(GetMeasure(_googleMeasuresDic[pair.Key]), pair.Value);
}
}
return(metricsUnit);
}
/// <summary>
/// Save metrics (sigle row) into metrics table
/// </summary>
/// <param name="metricsUnit"></param>
public void ImportMetrics(MetricsUnit metricsUnit)
{
//if (_insertMetricsCommand == null || String.IsNullOrEmpty(_insertMetricsCommand.CommandText))
// throw new Exception(String.Format("Insert command is not ready to import metrics for Table '{0}'", TableName));
var flatObjectList = _edgeObjectsManger.GetFlatObjectList(metricsUnit);
var columnList = GetColumnList(flatObjectList);
// no need to import metrics if there are only dimensions no measures (import objects only)
if (metricsUnit.MeasureValues != null)
{
ImportMetricsData(columnList);
}
}
protected override MetricsUnit GetSampleMetrics()
{
using (var reader = new JsonDynamicReader(Configuration.SampleFilePath, "$.records[*].*"))
{
Mappings.OnFieldRequired = fieldName => reader.Current[fieldName];
if (reader.Read())
{
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
return(CurrentMetricsUnit);
}
}
throw new ConfigurationErrorsException(String.Format("Failed to read sample metrics from file: {0}", Configuration.SampleFilePath));
}
/// <summary>
/// Create delivery metric table named by table prefix using sample metric unit structure
/// </summary>
/// <param name="tablePrefix"></param>
/// <param name="metricsUnit">sample metric unit for table structure</param>
/// <returns></returns>
public void CreateDeliveryMetricsTable(string tablePrefix, MetricsUnit metricsUnit)
{
// no need to create metrics table if there are only dimensions no measures (import objects only)
if (metricsUnit == null || metricsUnit.MeasureValues == null)
{
return;
}
TableName = string.Format("[DBO].[{0}_Metrics]", tablePrefix);
var flatObjectList = _edgeObjectsManger.GetFlatObjectList(metricsUnit);
SaveMetricsMetadata(flatObjectList);
var columnList = GetColumnList(flatObjectList, false); // sampe metrics objects are not added to object cache
CreateTable(columnList);
}
protected override MetricsUnit GetSampleMetrics()
{
try
{
// load sample file, read only one row in order to create metrics table by sample metric unit
ReaderAdapter.Init(FileManager.Open(Configuration.SampleFilePath), Configuration);
ReaderAdapter.Reader.Read();
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
return(CurrentMetricsUnit);
}
catch (Exception ex)
{
throw new ConfigurationErrorsException(String.Format("Failed to create metrics by sample file '{0}', ex: {1}", Configuration.SampleFilePath, ex.Message));
}
}
/// <summary>
/// Normalize flat object list to set:
/// Account and Channel to all objects according to Metrics unit definition
/// </summary>
/// <param name="flatObjectList"></param>
/// <param name="metricsUnit"></param>
/// <returns></returns>
private List <object> Normalize(List <object> flatObjectList, MetricsUnit metricsUnit)
{
foreach (var obj in flatObjectList)
{
// edge object can be an object or an extra field
var edgeObj = obj as EdgeObject;
if (obj is ObjectDimension && (obj as ObjectDimension).Value is EdgeObject)
{
edgeObj = (obj as ObjectDimension).Value as EdgeObject;
}
if (edgeObj != null)
{
// set account and channeL
edgeObj.Account = metricsUnit.Account;
if (edgeObj is ChannelSpecificObject)
{
(edgeObj as ChannelSpecificObject).Channel = metricsUnit.Channel;
}
}
}
return(flatObjectList);
}
public virtual void ImportMetrics(MetricsUnit metrics)
{
EnsureBeginImport();
_metricsTableManager.ImportMetrics(metrics);
}
public abstract void ImportMetrics(MetricsUnit metrics);
protected override ServiceOutcome DoPipelineWork()
{
// init mapping and load configuration
Log("Starting Google.AdWords.CampaignCriterionProcessorService", LogMessageType.Debug);
InitMappings();
Mappings.OnMappingApplied = SetEdgeType;
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings))
{
throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object");
}
LoadConfiguration();
Progress = 0.1;
// get nessesary EdgeFields
var relationField = GetEdgeFieldByName("Relation");
var locationField = GetEdgeFieldByName("Location");
var languageField = GetEdgeFieldByName("Language");
var isLocationNegativeField = GetEdgeFieldByName("IsLocationNegative");
var isLanguageNegativeField = GetEdgeFieldByName("IsLanguageNegative");
using (ReaderAdapter)
{
using (var stream = _deliveryFile.OpenContents(compression: _compression))
{
ReaderAdapter.Init(stream, Configuration);
while (ReaderAdapter.Reader.Read())
{
// load metrics unit which contains Relation object of Campaign-Language or Campaign-Location
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
// insert loaded relation to List per Campaign
var relation = CurrentMetricsUnit.Dimensions[relationField] as RelationObject;
if (relation != null)
{
var campaign = relation.Object1 as Campaign;
if (campaign != null)
{
if (!_campaignRelationMap.ContainsKey(campaign.OriginalID))
{
_campaignRelationMap.Add(campaign.OriginalID, new List <RelationObject>());
}
_campaignRelationMap[campaign.OriginalID].Add(relation);
}
}
}
}
}
Progress = 0.5;
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, _importManagerOptions)
{
OnLog = Log
})
{
// create object tables
ImportManager.BeginImport(Delivery, null);
// add objects to EdgeObjectsManager cache
PrepareImportObjects(x => x is Location, locationField, isLocationNegativeField);
PrepareImportObjects(x => x is Language, languageField, isLanguageNegativeField);
// import objects
ImportManager.EndImport();
}
return(ServiceOutcome.Success);
}
protected virtual void OnBeginImport(MetricsUnit sampleMetrics)
{
}
protected override ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
_currentOutput = Delivery.Outputs.First();
_currentOutput.Checksum = new Dictionary <string, double>();
var columns = new Dictionary <string, int>();
foreach (var reportFile in Delivery.Files)
{
//Get Columns
var reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.columnHeaders[*].*");
using (reportReader)
{
var colIndex = 0;
while (reportReader.Read())
{
columns.Add(reportReader.Current.name, colIndex);
colIndex++;
}
}
// get sample unit to create metrics table
reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*");
using (reportReader)
{
Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
reportReader.Read();
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField
};
metricsUnitMapping.Apply(CurrentMetricsUnit);
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes))
{
ImportManager.BeginImport(Delivery, CurrentMetricsUnit);
//Get values
reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*");
using (reportReader)
{
Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
while (reportReader.Read())
{
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField
};
metricsUnitMapping.Apply(CurrentMetricsUnit);
ProcessMetrics();
}
}
// finish import (insert edge objects)
ImportManager.EndImport();
}
}
}
return(ServiceOutcome.Success);
}
