protected override ServiceOutcome DoPipelineWork()
{
if (Configuration.Parameters["AccountID"] != null)
{
int.TryParse(Configuration.Parameters["AccountID"].ToString(), out _accountId);
}
var checksumThreshold = Configuration.Parameters.Get <string>(Consts.ConfigurationOptions.ChecksumTheshold, false);
var identityInDebug = Configuration.Parameters.ContainsKey("IdentityInDebug") && Configuration.Parameters.Get <bool>("IdentityInDebug", false);
var options = new MetricsDeliveryManagerOptions
{
SqlTransformCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlTransformCommand),
ChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold),
IdentityInDebug = identityInDebug
};
using (var importManager = new MetricsDeliveryManager(InstanceID, options: options))
{
// TODO: need this? Only check tickets, don't check conflicts
HandleConflicts(importManager, DeliveryConflictBehavior.Ignore, getBehaviorFromConfiguration: false);
// perform transform
Log(String.Format("Start transform deliver '{0}'", Delivery.DeliveryID), LogMessageType.Information);
importManager.Transform(new[] { Delivery });
Log(String.Format("Finished transform deliver '{0}'", Delivery.DeliveryID), LogMessageType.Information);
}
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
var checksumThreshold = Configuration.Parameters.Get <string>(Consts.ConfigurationOptions.ChecksumTheshold, false);
var identityInDebug = Configuration.Parameters.ContainsKey("IdentityInDebug") && Configuration.Parameters.Get <bool>("IdentityInDebug", false);
var identityConfig = Configuration.Parameters.ContainsKey("IdentityConfig") ? Configuration.Parameters.Get <string>("IdentityConfig") : null;
var options = new MetricsDeliveryManagerOptions
{
SqlStageCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlStageCommand),
SqlRollbackCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlRollbackCommand),
ChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold),
IdentityInDebug = identityInDebug,
IdentityConfig = identityConfig
};
using (var importManager = new MetricsDeliveryManager(InstanceID, options: options))
{
var success = false;
do
{
try
{
// perform staging
Log("Staging: start", LogMessageType.Information);
importManager.Stage(new[] { Delivery });
Log("Staging: end", LogMessageType.Information);
success = true;
}
catch (DeliveryConflictException dceex)
{
// rollback in case of exception
Log("Rollback: start", LogMessageType.Information);
importManager.RollbackOutputs(dceex.ConflictingOutputs);
Log("Rollback: end", LogMessageType.Information);
}
catch (Exception ex)
{
throw new Exception(String.Format("Delivery {0} failed during staging.", Delivery.DeliveryID), ex);
}
}while (!success);
}
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
InitMappings();
LoadConfiguration();
// Import data
using (ReaderAdapter)
{
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, _importManagerOptions)
{
OnLog = Log
})
{
// create objects tables and metrics table according to sample metrics
ImportManager.BeginImport(Delivery, GetSampleMetrics());
Log("ImportManager.BeginImport() executed successfully", LogMessageType.Debug);
// open delivery file
using (var stream = _deliveryFile.OpenContents(compression: _compression))
{
ReaderAdapter.Init(stream, Configuration);
// for each row in file read and import into metrics table
var readSuccess = false;
while (CheckEndOfFile())
{
readSuccess = true;
ProcessMetrics();
}
if (!readSuccess)
{
Log("Could Not read data from file!, check file mapping and configuration", Core.Utilities.LogMessageType.Warning);
}
ImportManager.EndImport();
Log("ImportManager.EndImport() executed successfully", LogMessageType.Debug);
}
}
}
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
// init mapping and load configuration
Log("Starting Google.AdWords.CampaignCriterionProcessorService", LogMessageType.Debug);
InitMappings();
Mappings.OnMappingApplied = SetEdgeType;
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings))
{
throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object");
}
LoadConfiguration();
Progress = 0.1;
// get nessesary EdgeFields
var relationField = GetEdgeFieldByName("Relation");
var locationField = GetEdgeFieldByName("Location");
var languageField = GetEdgeFieldByName("Language");
var isLocationNegativeField = GetEdgeFieldByName("IsLocationNegative");
var isLanguageNegativeField = GetEdgeFieldByName("IsLanguageNegative");
using (ReaderAdapter)
{
using (var stream = _deliveryFile.OpenContents(compression: _compression))
{
ReaderAdapter.Init(stream, Configuration);
while (ReaderAdapter.Reader.Read())
{
// load metrics unit which contains Relation object of Campaign-Language or Campaign-Location
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
// insert loaded relation to List per Campaign
var relation = CurrentMetricsUnit.Dimensions[relationField] as RelationObject;
if (relation != null)
{
var campaign = relation.Object1 as Campaign;
if (campaign != null)
{
if (!_campaignRelationMap.ContainsKey(campaign.OriginalID))
{
_campaignRelationMap.Add(campaign.OriginalID, new List <RelationObject>());
}
_campaignRelationMap[campaign.OriginalID].Add(relation);
}
}
}
}
}
Progress = 0.5;
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, _importManagerOptions)
{
OnLog = Log
})
{
// create object tables
ImportManager.BeginImport(Delivery, null);
// add objects to EdgeObjectsManager cache
PrepareImportObjects(x => x is Location, locationField, isLocationNegativeField);
PrepareImportObjects(x => x is Language, languageField, isLanguageNegativeField);
// import objects
ImportManager.EndImport();
}
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
Log("Starting Google.AdWords.ProcessorService", LogMessageType.Debug);
InitMappings();
Mappings.OnMappingApplied = SetEdgeType;
if (!Mappings.Objects.TryGetValue(typeof(KeywordTarget), out KeywordMappings))
{
throw new MappingConfigurationException("Missing mapping definition for KeywordTarget.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(PlacementTarget), out PlacementMappings))
{
throw new MappingConfigurationException("Missing mapping definition for PlacementTarget.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings))
{
throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings))
{
throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object");
}
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, new MetricsDeliveryManagerOptions()))
//{
// MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
// MeasureOptionsOperator = OptionsOperator.Not,
// SegmentOptions = Data.Objects.SegmentOptions.All,
// SegmentOptionsOperator = OptionsOperator.And
//}))
{
ImportManager.BeginImport(Delivery, GetSampleMetrics());
Log("Objects and Metrics tables are created", LogMessageType.Debug);
Progress = 0.1;
var requiredHeaders = new[] { AdWordsConst.AdPreRequiredHeader };
// Getting Keywords Data
Log("Start loading keywords", LogMessageType.Debug);
LoadKeywords(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]], requiredHeaders);
Log("Finished loading keywords", LogMessageType.Debug);
Progress = 0.3;
// Getting Placements Data
Log("Start loading placements", LogMessageType.Debug);
LoadPlacements(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]], requiredHeaders);
Log("Finished loading placements", LogMessageType.Debug);
// Getting Conversions Data ( for ex. signup , purchase )
Log("Start loading conversions", LogMessageType.Debug);
LoadConversions(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"], requiredHeaders);
Log("Finished loading conversions", LogMessageType.Debug);
Progress = 0.4;
#region Getting Ads Data and Import Metrics
Log("Start loading Ads", LogMessageType.Debug);
var adPerformanceFile = Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]];
_adsReader = new CsvDynamicReader(adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
Mappings.OnFieldRequired = field => _adsReader.Current[field];
_importedAds.Clear();
using (_adsReader)
{
Mappings.OnFieldRequired = fieldName => _adsReader.Current[fieldName];
while (_adsReader.Read() && _adsReader.Current[AdWordsConst.AdIDFieldName] != AdWordsConst.EOF)
{
ProcessMetrics();
// add Ad if it is not exists yet
if (!_importedAds.ContainsKey(CurrentMetricsUnit.Ad.OriginalID))
{
_importedAds.Add(CurrentMetricsUnit.Ad.OriginalID, CurrentMetricsUnit.Ad);
}
}
//Progress = 0.8;
//Log("Start importing objects", LogMessageType.Debug);
ImportManager.EndImport();
//Log("Finished importing objects", LogMessageType.Debug);
}
#endregion
}
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
Log("Starting SalesForce.ProcessorService", LogMessageType.Debug);
InitMappings();
Mappings.OnMappingApplied = SetEdgeType;
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings))
{
throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings))
{
throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object");
}
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, new MetricsDeliveryManagerOptions()))
{
// create objects and metrics table according to the sample metrics
ImportManager.BeginImport(Delivery, GetSampleMetrics());
Log("Objects and Metrics tables are created", LogMessageType.Debug);
Progress = 0.1;
var metricsMap = new Dictionary <string, MetricsUnit>();
foreach (var file in Delivery.Files)
{
using (var reader = new JsonDynamicReader(file.OpenContents(), "$.records[*].*"))
{
Mappings.OnFieldRequired = fieldName => reader.Current[fieldName];
Log(String.Format("Start reading metrics for file '{0}'", file.Name), LogMessageType.Debug);
while (reader.Read())
{
// load current metrics unit
LoadCurrentMetrics();
if (CurrentMetricsUnit.Dimensions.ContainsKey(GetEdgeField("Tracker")) && CurrentMetricsUnit.Dimensions[GetEdgeField("Tracker")] is StringValue)
{
var tracker = CurrentMetricsUnit.Dimensions[GetEdgeField("Tracker")] as StringValue;
// check if metrics unit already exists by tracker
if (metricsMap.ContainsKey(tracker.Value))
{
// summaries all measure values
foreach (var measure in CurrentMetricsUnit.MeasureValues)
{
metricsMap[tracker.Value].MeasureValues[measure.Key] += measure.Value;
}
}
else
{
// add new metrics unit by tracker
metricsMap.Add(tracker.Value, CurrentMetricsUnit);
}
}
else
{
Log(String.Format("Failed to get tracker in file '{0}'", file.Name), LogMessageType.Warning);
}
}
}
}
Progress = 0.5;
// import summarized metrics
Log("Start importing summarizzed metrics", LogMessageType.Debug);
foreach (var metrics in metricsMap)
{
ImportManager.ImportMetrics(metrics.Value);
}
Progress = 0.8;
Log("Start importing objects", LogMessageType.Debug);
ImportManager.EndImport();
}
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
MappingContainer metricsUnitMapping;
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out metricsUnitMapping))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.");
}
_currentOutput = Delivery.Outputs.First();
_currentOutput.Checksum = new Dictionary <string, double>();
var columns = new Dictionary <string, int>();
foreach (var reportFile in Delivery.Files)
{
//Get Columns
var reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.columnHeaders[*].*");
using (reportReader)
{
var colIndex = 0;
while (reportReader.Read())
{
columns.Add(reportReader.Current.name, colIndex);
colIndex++;
}
}
// get sample unit to create metrics table
reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*");
using (reportReader)
{
Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
reportReader.Read();
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField
};
metricsUnitMapping.Apply(CurrentMetricsUnit);
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes))
{
ImportManager.BeginImport(Delivery, CurrentMetricsUnit);
//Get values
reportReader = new JsonDynamicReader(reportFile.OpenContents(), "$.rows[*].*");
using (reportReader)
{
Mappings.OnFieldRequired = field => reportReader.Current["array"][columns[field]];
while (reportReader.Read())
{
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField
};
metricsUnitMapping.Apply(CurrentMetricsUnit);
ProcessMetrics();
}
}
// finish import (insert edge objects)
ImportManager.EndImport();
}
}
}
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
Log("Starting Facebook.GraphApi.ProcessorService", LogMessageType.Debug);
InitMappings();
Mappings.OnMappingApplied = SetEdgeType;
if (!Mappings.Objects.TryGetValue(typeof(Ad), out AdMappings))
{
throw new MappingConfigurationException("Missing mapping definition for Ad.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(Campaign), out CampaignMappings))
{
throw new MappingConfigurationException("Missing mapping definition for Campaign.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(CompositeCreative), out CreativeMappings))
{
throw new MappingConfigurationException("Missing mapping definition for CompositeCreative.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings))
{
throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings))
{
throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object");
}
using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, new MetricsDeliveryManagerOptions()))
{
// create objects and metrics table according to the sample metrics
ImportManager.BeginImport(Delivery, GetSampleMetrics());
Log("Objects and Metrics tables are created", LogMessageType.Debug);
Progress = 0.1;
// load Campaigns, Creatives and Ads into local Cache
ClearLocalCache();
var filesByType = Delivery.Parameters["FilesByType"] as IDictionary <Consts.FileTypes, List <string> >;
LoadFiles(filesByType[Consts.FileTypes.Campaigns], LoadCampaigns);
LoadFiles(filesByType[Consts.FileTypes.Creatives], LoadCreatives);
LoadFiles(filesByType[Consts.FileTypes.AdGroups], LoadAds);
Log("Campaigns, Creatives and Ads are loaded into local cache", LogMessageType.Debug);
Progress = 0.3;
// start processing metrics
foreach (var filePath in filesByType[Consts.FileTypes.AdGroupStats])
{
using (var reader = new JsonDynamicReader(Delivery.Files[filePath].OpenContents(), "$.data[*].*"))
{
Mappings.OnFieldRequired = fieldName => reader.Current[fieldName];
while (reader.Read())
{
ProcessMetrics();
}
}
}
Progress = 0.8;
Log("Start importing objects", LogMessageType.Debug);
ImportManager.EndImport();
Log("Finished importing objects", LogMessageType.Debug);
}
return(ServiceOutcome.Success);
}
