private Ad CreateAd(dynamic values, AdMetricsImportManager session)
{
Ad ad = new Ad()
{
OriginalID = values[WS.AdPerformanceReportColumn.AdId.ToString()],
Creatives = new List <Creative>()
{
new TextCreative()
{
TextType = TextCreativeType.Title, Text = values[WS.AdPerformanceReportColumn.AdTitle.ToString()]
},
new TextCreative()
{
TextType = TextCreativeType.Body, Text = values[WS.AdPerformanceReportColumn.AdDescription.ToString()]
},
new TextCreative()
{
TextType = TextCreativeType.DisplayUrl, Text = string.Empty
}
},
DestinationUrl = values[WS.AdPerformanceReportColumn.DestinationUrl.ToString()],
Account = new Account()
{
ID = this.Delivery.Account.ID, OriginalID = this.Delivery.Account.OriginalID
},
Channel = new Channel()
{
ID = this.Delivery.Channel.ID
}
};
return(ad);
}
private AdMetricsUnit CreateMetrics(dynamic values, string timePeriodColumn, AdMetricsImportManager session)
{
var metricsUnit = new AdMetricsUnit();
metricsUnit.Output = currentOutput;
metricsUnit.MeasureValues = new Dictionary <Measure, double>();
metricsUnit.Ad = _adCache[Convert.ToInt64(values[WS.KeywordPerformanceReportColumn.AdId.ToString()])];
metricsUnit.Currency = new Currency()
{
Code = values[WS.KeywordPerformanceReportColumn.CurrencyCode.ToString()]
};
metricsUnit.TimePeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
metricsUnit.TimePeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
metricsUnit.MeasureValues[session.Measures[Measure.Common.Clicks]] = double.Parse(string.IsNullOrEmpty(values[WS.KeywordPerformanceReportColumn.Clicks.ToString()]) ? "0" : values[WS.KeywordPerformanceReportColumn.Clicks.ToString()]);
metricsUnit.MeasureValues[session.Measures[Measure.Common.Cost]] = double.Parse(string.IsNullOrEmpty(values[WS.KeywordPerformanceReportColumn.Spend.ToString()]) ? "0" : values[WS.KeywordPerformanceReportColumn.Spend.ToString()]);
metricsUnit.MeasureValues[session.Measures[Measure.Common.Impressions]] = double.Parse(string.IsNullOrEmpty(values[WS.KeywordPerformanceReportColumn.Impressions.ToString()]) ? "0" : values[WS.KeywordPerformanceReportColumn.Impressions.ToString()]);
metricsUnit.MeasureValues[session.Measures[Measure.Common.AveragePosition]] = double.Parse(string.IsNullOrEmpty(values[WS.KeywordPerformanceReportColumn.AveragePosition.ToString()]) ? "0" : values[WS.KeywordPerformanceReportColumn.AveragePosition.ToString()]);
metricsUnit.MeasureValues[session.Measures[MeasureNames.AdCenterConversions]] = double.Parse(string.IsNullOrEmpty(values[WS.KeywordPerformanceReportColumn.Conversions.ToString()]) ? "0" : values[WS.KeywordPerformanceReportColumn.Conversions.ToString()]);
metricsUnit.TargetDimensions = new List <Target>();
//CREATING KEYWORD TARGET
KeywordTarget kwdTarget = new KeywordTarget();
kwdTarget.DestinationUrl = values[WS.KeywordPerformanceReportColumn.DestinationUrl.ToString()];
kwdTarget.Keyword = values[WS.KeywordPerformanceReportColumn.Keyword.ToString()];
kwdTarget.QualityScore = values[WS.KeywordPerformanceReportColumn.QualityScore.ToString()];
string macthType = values["BiddedMatchType"];// KeywordMatchType.Phrase;
//string macthType = values[WS.KeywordPerformanceReportColumn.BidMatchType.ToString()];
switch (macthType)
{
case "Exact":
kwdTarget.MatchType = KeywordMatchType.Exact;
break;
case "Broad":
kwdTarget.MatchType = KeywordMatchType.Broad;
break;
case "Phrase":
kwdTarget.MatchType = KeywordMatchType.Phrase;
break;
default:
kwdTarget.MatchType = KeywordMatchType.Unidentified;
break;
}
kwdTarget.OriginalID = values[WS.KeywordPerformanceReportColumn.KeywordId.ToString()];
metricsUnit.TargetDimensions.Add(kwdTarget);
return(metricsUnit);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"];
StringBuilder warningsStr = new StringBuilder();
Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>();
Dictionary <string, AdGroup> adGroupsData = new Dictionary <string, AdGroup>();
Dictionary <string, Ad> ads = new Dictionary <string, Ad>();
Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >();
var adStatIds = new Dictionary <string, string>();
var insertedAds = new Dictionary <string, string>();
var storyIds = new Dictionary <string, Dictionary <string, string> >();
var photoIds = new Dictionary <string, Dictionary <string, string> >();
DeliveryOutput currentOutput = Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
#region AdSets
if (filesByType.ContainsKey(Consts.FileTypes.CampaignGroups))
{
List <string> campaignsFiles = filesByType[Consts.FileTypes.CampaignGroups];
foreach (var campaignFile in campaignsFiles)
{
DeliveryFile campaigns = this.Delivery.Files[campaignFile];
var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*");
using (campaignsReader)
{
while (campaignsReader.Read())
{
Campaign camp = new Campaign()
{
Name = campaignsReader.Current.name,
OriginalID = Convert.ToString(campaignsReader.Current.id),
};
campaignsData.Add(camp.OriginalID, camp);
}
}
}
}
#endregion
#region AdGroups
if (filesByType.ContainsKey(Consts.FileTypes.AdSets))
{
List <string> adSetList = filesByType[Consts.FileTypes.AdSets];
foreach (var adSet in adSetList)
{
DeliveryFile adSetDF = this.Delivery.Files[adSet];
var adSetReader = new JsonDynamicReader(adSetDF.OpenContents(), "$.data[*].*");
using (adSetReader)
{
while (adSetReader.Read())
{
var adGroupObj = new AdGroup()
{
Value = adSetReader.Current.name,
OriginalID = Convert.ToString(adSetReader.Current.id),
};
if (campaignsData.ContainsKey(adSetReader.Current.campaign_group_id))
{
adGroupObj.Campaign = campaignsData[adSetReader.Current.campaign_group_id];
}
adGroupsData.Add(adGroupObj.OriginalID, adGroupObj);
}
}
}
}
#endregion
#region adGroups And Targeting
//******************************************************************************************************************************************************
if (filesByType.ContainsKey(Consts.FileTypes.AdGroups))
{
List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups];
foreach (var adGroup in adGroupsFiles)
{
#region foreach
DeliveryFile adGroups = this.Delivery.Files[adGroup];
var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*");
using (adGroupsReader)
{
while (adGroupsReader.Read())
{
var campaignId = Convert.ToString(adGroupsReader.Current.campaign_id);
if (adGroupsData.ContainsKey(campaignId) && ((AdGroup)adGroupsData[campaignId]).Campaign != null)
{
Ad ad = new Ad();
ad.OriginalID = Convert.ToString(adGroupsReader.Current.id);
ad.Segments = new Dictionary <Segment, SegmentObject>();
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] =
((AdGroup)adGroupsData[Convert.ToString(adGroupsReader.Current.campaign_id)]).Campaign;
ad.Name = adGroupsReader.Current.name;
ad.Channel = new Channel()
{
ID = 6
};
ad.Account = new Account()
{
ID = this.Delivery.Account.ID,
OriginalID = this.Delivery.Account.OriginalID.ToString()
};
if (adGroupsData.ContainsKey(adGroupsReader.Current.campaign_id))
{
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] =
adGroupsData[adGroupsReader.Current.campaign_id];
}
// adgroup targeting
string age_min = string.Empty;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min"))
{
age_min = adGroupsReader.Current.targeting["age_min"];
}
if (!string.IsNullOrEmpty(age_min))
{
AgeTarget ageTarget = new AgeTarget()
{
FromAge = int.Parse(age_min),
ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"])
};
ad.Targets.Add(ageTarget);
}
List <object> genders = null;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders"))
{
genders = adGroupsReader.Current.targeting["genders"];
}
if (genders != null)
{
foreach (object gender in genders)
{
GenderTarget genderTarget = new GenderTarget();
if (gender.ToString() == "1")
{
genderTarget.Gender = Gender.Male;
}
else if (gender.ToString() == "2")
{
genderTarget.Gender = Gender.Female;
}
else
{
genderTarget.Gender = Gender.Unspecified;
}
genderTarget.OriginalID = gender.ToString();
ad.Targets.Add(genderTarget);
}
}
if (adGroupsReader.Current.creative_ids != null)
{
foreach (string creative in adGroupsReader.Current.creative_ids)
{
if (!adsBycreatives.ContainsKey(creative))
{
adsBycreatives.Add(creative, new List <Ad>());
}
adsBycreatives[creative].Add(ad);
}
}
ads.Add(ad.OriginalID, ad);
}
}
}
#endregion
}
}
//******************************************************************************************************************************************************
#endregion adGroups And Targeting
//GetAdGroupStats
#region for validation
foreach (var measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(measure.Key))
{
currentOutput.Checksum.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP
}
}
}
//**************************************************************************
#endregion
Dictionary <string, List <Dictionary <string, object> > > conversion_data = new Dictionary <string, List <Dictionary <string, object> > >();
if (filesByType.ContainsKey(Consts.FileTypes.ConversionsStats))
{
#region Conversions
List <string> conversionFiles = filesByType[Consts.FileTypes.ConversionsStats];
foreach (string conversionFile in conversionFiles)
{
DeliveryFile creativeFile = Delivery.Files[conversionFile];
var conversionCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (conversionCreativesReader)
{
this.Mappings.OnFieldRequired = field => conversionCreativesReader.Current[field];
while (conversionCreativesReader.Read())
{
if (((Edge.Data.Pipeline.DynamicDictionaryObject)(conversionCreativesReader.Current)).Values == null)
{
break; // in case of empty data (e.g. data [] )
}
if (!conversion_data.ContainsKey(conversionCreativesReader.Current.adgroup_id))
{
Dictionary <string, string> adConversionData = new Dictionary <string, string>();
//string TotalConversionsManyPerClick, Leads, Signups, Purchases;
List <object> actions = conversionCreativesReader.Current.actions;
if (actions.Count > 0)
{
List <Dictionary <string, object> > action_list = actions.Select(s => (Dictionary <string, object>)s).ToList();
List <Dictionary <string, object> > action_items = action_list.Where(dict => dict.ContainsKey("action_type") && dict.ContainsKey("value") &&
(
dict["action_type"].ToString().Equals("offsite_conversion") ||
dict["action_type"].ToString().Equals("offsite_conversion.lead") ||
dict["action_type"].ToString().Equals("offsite_conversion.registration") ||
dict["action_type"].ToString().Equals("offsite_conversion.add_to_cart") ||
dict["action_type"].ToString().Equals("offsite_conversion.checkout") ||
dict["action_type"].ToString().Equals("offsite_conversion.key_page_view") ||
dict["action_type"].ToString().Equals("offsite_conversion.other")
)
).ToList();
if (action_items.Count > 0)
{
conversion_data.Add(conversionCreativesReader.Current.adgroup_id, action_items);
}
}
}
} //End of While read
} // End of Using reader
} //End Foreach conversion file
#endregion
} //End if contains conversion file
#region AdGroupStats start new import session
List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats];
foreach (var adGroupStat in adGroupStatsFiles)
{
DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat];
var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*");
using (adGroupStatsReader)
{
while (adGroupStatsReader.Read())
{
#region Create Metrics
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
adMetricsUnit.Output = currentOutput;
adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
Ad tempAd;
try
{
var x = adGroupStatsReader.Current.adgroup_id;
}
catch (Exception)
{
continue;
}
if (adGroupStatsReader.Current.adgroup_id != null)
{
adStatIds[adGroupStatsReader.Current.adgroup_id] = adGroupStatsReader.Current.adgroup_id;
if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd))
{
adMetricsUnit.Ad = tempAd;
//adMetricsUnit.PeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
//adMetricsUnit.PeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
// Common and Facebook specific meausures
/* Sets totals for validations */
if (currentOutput.Checksum.ContainsKey(Measure.Common.Clicks))
{
currentOutput.Checksum[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Impressions))
{
currentOutput.Checksum[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Cost))
{
currentOutput.Checksum[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d;
}
/* Sets measures values */
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(adGroupStatsReader.Current.clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueClicks], Convert.ToInt64(adGroupStatsReader.Current.unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(adGroupStatsReader.Current.impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueImpressions], Convert.ToInt64(adGroupStatsReader.Current.unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d);
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Actions], 0);
//adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections));
//Setting conversions from conversion files data
List <Dictionary <string, object> > adgroup_conversion_data;
if (conversion_data.TryGetValue(adGroupStatsReader.Current.adgroup_id, out adgroup_conversion_data))
{
var TotalConversionsManyPerClick = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion") select element["value"];
var Leads = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.lead") select element["value"];
var Signups = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.registration") select element["value"];
//TO DO : Get field from Configuration
var Purchases = from element in adgroup_conversion_data where element["action_type"].Equals(this.Delivery.Parameters["API_Purchases_Field_Name"]) select element["value"];
//add values to metrics unit
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.TotalConversionsManyPerClick], Convert.ToDouble(TotalConversionsManyPerClick.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Leads], Convert.ToDouble(Leads.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Signups], Convert.ToDouble(Signups.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Purchases], Convert.ToDouble(Purchases.DefaultIfEmpty("0").FirstOrDefault()));
}
this.ImportManager.ImportMetrics(adMetricsUnit);
}
else
{
warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID));
}
}
else
{
warningsStr.AppendLine("adGroupStatsReader.Current.id=null");
}
#endregion
}
}
}
#endregion AdGroupStats start new import session
#region Creatives
List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives];
Dictionary <string, string> usedCreatives = new Dictionary <string, string>();
foreach (string creative in creativeFiles)
{
DeliveryFile creativeFile = Delivery.Files[creative];
var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (adGroupCreativesReader)
{
//this.Mappings.OnFieldRequired = field => if((field == "object_url" && adGroupCreativesReader.Current[field] != null) || field != "object_url")adGroupCreativesReader.Current[field];
this.Mappings.OnFieldRequired = field => adGroupCreativesReader.Current[field];
while (adGroupCreativesReader.Read())
{
List <Ad> adsByCreativeID = null;
if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
usedCreatives.Add(adGroupCreativesReader.Current.id, adGroupCreativesReader.Current.id);
adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.id];
}
}
if (adsByCreativeID != null)
{
foreach (Ad ad in adsByCreativeID)
{
if (!adStatIds.ContainsKey(ad.OriginalID))
{
continue;
}
ad.Creatives = new List <Creative>();
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_type))
{
string objectType = adGroupCreativesReader.Current.object_type;
switch (objectType.ToUpper())
{
case "SHARE":
{
#region Ads Type SHARE
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_story_id))
{
Dictionary <string, string> shareCreativeData;
string object_story_id = adGroupCreativesReader.Current.object_story_id;
if (storyIds.ContainsKey(object_story_id))
{
shareCreativeData = storyIds[object_story_id];
}
else
{
var accessToken = this.Instance.Configuration.Options[FacebookConfigurationOptions.Auth_AccessToken];
shareCreativeData = GetShareCreativeData(object_story_id, accessToken);
}
ad.DestinationUrl = shareCreativeData["link"];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
var trackersReadCommands = from n in this.Mappings.Objects[typeof(Ad)].ReadCommands
where n.Field.Equals("link_url")
select n;
foreach (var command in trackersReadCommands)
{
string trackerValue = ApplyRegex(ad.DestinationUrl, command.RegexPattern);
if (!String.IsNullOrEmpty(trackerValue))
{
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value = trackerValue;
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].OriginalID = trackerValue;
}
}
}
ad.Creatives.Add(GetTextCreative(shareCreativeData["text"], adGroupCreativesReader));
ad.Creatives.Add(GetBodyCreative(shareCreativeData["description"], adGroupCreativesReader));
ad.Creatives.Add(GetImageCreative(shareCreativeData["picture"], adGroupCreativesReader));
}
#endregion
break;
}
case "DOMAIN":
{
#region Ads Type DOMAIN
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_url))
{
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields))
{
ad.DestinationUrl = adGroupCreativesReader.Current.object_url;
}
}
else if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.link_url))
{
ad.DestinationUrl = adGroupCreativesReader.Current.link_url;
}
else
{
ad.DestinationUrl = "UnKnown Url";
}
/*Get Data from Mapping E.g Tracker*/
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
if (adGroupCreativesReader.Current.image_url != null)
{
CreateImageCreatives(ad, adGroupCreativesReader);
}
ad.Creatives.Add(GetTextCreative(adGroupCreativesReader));
ad.Creatives.Add(GetBodyCreative(adGroupCreativesReader));
#endregion
break;
}
default:
{
#region Ads Type not handeled
string adtype = objectType.ToUpper();
ad.DestinationUrl = adtype + " Ad Type";
/*Get Data from Mapping E.g Tracker*/
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
TextCreative unhandeledTextAd_title = new TextCreative()
{
OriginalID = ad.OriginalID,
TextType = TextCreativeType.Title,
Text = adtype + " Ad Type"
};
ad.Creatives.Add(unhandeledTextAd_title);
#endregion
break;
}
}
if (!insertedAds.ContainsKey(ad.OriginalID))
{
insertedAds[ad.OriginalID] = ad.OriginalID;
this.ImportManager.ImportAd(ad);
}
}
}
}
//TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS)
}
}
#endregion
}
// End foreach creative
currentOutput.Status = DeliveryOutputStatus.Imported;
this.ImportManager.EndImport();
if (!string.IsNullOrEmpty(warningsStr.ToString()))
{
Log.Write(warningsStr.ToString(), LogMessageType.Warning);
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
#region Init General
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
// This is for finding conflicting services
this.Delivery.Signature = Delivery.CreateSignature(String.Format("facebook-[{0}]-[{1}]-[{2}]",
this.Instance.AccountID,
this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString(),
this.TargetPeriod.ToAbsolute()));
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.ImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Common.Importing.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
// ...............................
// Now that we have a new delivery, start adding values
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
OriginalID = this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString()
};
this.Delivery.TargetPeriod = this.TargetPeriod;
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = 6
};
this.Delivery.TargetLocationDirectory = Instance.Configuration.Options["DeliveryFilesDir"];
if (string.IsNullOrEmpty(this.Delivery.TargetLocationDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
// Copy some options as delivery parameters
var configOptionsToCopyToDelivery = new string[] {
FacebookConfigurationOptions.Account_ID,
FacebookConfigurationOptions.Account_Name,
FacebookConfigurationOptions.Auth_ApiKey,
FacebookConfigurationOptions.Auth_AppSecret,
FacebookConfigurationOptions.Auth_SessionKey,
FacebookConfigurationOptions.Auth_SessionSecret,
FacebookConfigurationOptions.Auth_RedirectUri,
FacebookConfigurationOptions.Auth_AuthenticationUrl
};
foreach (string option in configOptionsToCopyToDelivery)
{
this.Delivery.Parameters[option] = this.Instance.Configuration.Options[option];
}
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]))
{
throw new Exception("facebook base url must be configured!");
}
_baseAddress = new Uri(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]);
this.ReportProgress(0.2);
#endregion
#region Init Delivery Files
Dictionary <string, string> methodParams = new Dictionary <string, string>();
string methodUrl;
DeliveryFile deliveryFile = new DeliveryFile();
#region adgroupstats
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupStats;
methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TargetPeriod.Start.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TargetPeriod.End.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupStats);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.AdGroupStats.ToString()));
this.Delivery.Files.Add(deliveryFile);
#endregion
this.ReportProgress(0.4);
#region adgroup
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroup;
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroups);
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.AdGroups);
this.Delivery.Files.Add(deliveryFile);
#endregion
this.ReportProgress(0.6);
#region Campaigns
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Campaigns;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaigns);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Campaigns);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region Creatives
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Creatives;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupCreatives);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Creatives);
this.Delivery.Files.Add(deliveryFile);
#endregion
//#region AdGroupTargeting
//deliveryFile = new DeliveryFile();
//deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupTargeting;
//methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupTargeting);
//deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
//deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Data);
//this.Delivery.Files.Add(deliveryFile);
//#endregion
#endregion
this.ReportProgress(0.9);
this.Delivery.Save();
this.ReportProgress(1);
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
#region Init General
// ...............................
// SETUP
this.Delivery = NewDelivery();
// This is for finding conflicting services
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
OriginalID = this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString()
};
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = 6
};
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("facebook-[{0}]-[{1}]-[{2}]",
this.Instance.AccountID,
this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString(),
this.Delivery.TimePeriodDefinition.ToAbsolute())),
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd,
Account = this.Delivery.Account,
Channel = this.Delivery.Channel
});
// Now that we have a new delivery, start adding values
this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory];
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]))
{
throw new Exception("facebook base url must be configured!");
}
_baseAddress = new Uri(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]);
//this.ReportProgress(0.2);
#endregion
#region Init Delivery Files
Dictionary <string, string> methodParams = new Dictionary <string, string>();
string methodUrl;
DeliveryFile deliveryFile = new DeliveryFile();
#region adgroupstats
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupStats;
methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TimePeriod.Start.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TimePeriod.End.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupStats);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.AdGroupStats.ToString()));
this.Delivery.Files.Add(deliveryFile);
#endregion
//this.ReportProgress(0.4);
#region Conversions
//======================================================================================
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.ConversionsStats;
methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TimePeriod.Start.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TimePeriod.End.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetConversionStats);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.ConversionsStats.ToString()));
this.Delivery.Files.Add(deliveryFile);
//======================================================================================
#endregion Conversions
//this.ReportProgress(0.4);
#region adgroup
/*
* Summary
* An ad group contains the data necessary for an ad, such as bid type, bid info,
* targeting data, creative elements, and campaign information. Each ad group is
* associated with a campaign and all ad groups in a campaign have the same daily
* or lifetime budget and schedule.
* */
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroup;
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroups);
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.AdGroupFields].ToString());
}
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.AdGroups);
this.Delivery.Files.Add(deliveryFile);
#endregion
//this.ReportProgress(0.6);
#region AdSet- Formally Campaigns
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Campaigns;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.CampaignFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.CampaignFields].ToString());
}
// methodParams.Add("redownload", "true");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaignsAdSets);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Campaigns);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region Campaigns - New Structure
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.CampaignGroups;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.CampaignGroupsFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.CampaignGroupsFields].ToString());
}
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaignsGroups);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.CampaignGroups);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region Creatives
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Creatives;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.AdGroupCreativeFields].ToString());
}
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupCreatives);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Creatives);
this.Delivery.Files.Add(deliveryFile);
#endregion
//#region AdGroupTargeting
//deliveryFile = new DeliveryFile();
//deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupTargeting;
//methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupTargeting);
//deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
//deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Data);
//this.Delivery.Files.Add(deliveryFile);
#endregion
//this.ReportProgress(0.9);
this.Delivery.Save();
this.ReportProgress(1);
return(ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
string baseAddress;
#region Init General
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
// This is for finding conflicting services
this.Delivery.Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]",
this.Instance.AccountID,
this.TargetPeriod.ToAbsolute()));
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.ImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Common.Importing.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
// ...............................
// Now that we have a new delivery, start adding values
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
};
this.Delivery.TargetPeriod = this.TargetPeriod;
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = -1
};
this.Delivery.TargetLocationDirectory = Instance.Configuration.Options["DeliveryFilesDir"];
if (string.IsNullOrEmpty(this.Delivery.TargetLocationDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress]))
{
throw new Exception("base url must be configured!");
}
baseAddress = this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress];
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.UserName]))
{
throw new Exception("base url must be configured!");
}
this.Delivery.Parameters["UserName"] = this.Instance.Configuration.Options[BoConfigurationOptions.UserName];
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.Password]))
{
throw new Exception("base url must be configured!");
}
this.Delivery.Parameters["Password"] = this.Instance.Configuration.Options[BoConfigurationOptions.Password];
int utcOffset = 0;
if (!string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]))
{
utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]);
}
this.ReportProgress(0.2);
#endregion
#region DeliveryFile
Dictionary <string, string> UrlParams = new Dictionary <string, string>();
DeliveryFile boFile = new DeliveryFile();
boFile.Parameters[BoConfigurationOptions.BO_XPath_Trackers] = Instance.Configuration.Options[BoConfigurationOptions.BO_XPath_Trackers];
boFile.Name = BoConfigurationOptions.BoFileName;
UrlParams.Add("from", utcOffset == 0 ? Delivery.TargetPeriod.Start.ToDateTime().ToString("yyyy-MM-ddTHH:MMZ") : ConvertToTimeZone(utcOffset, TargetPeriod.Start.ToDateTime()));
UrlParams.Add("to", utcOffset == 0 ? Delivery.TargetPeriod.End.ToDateTime().ToString("yyyy-MM-ddTHH:MMZ") : ConvertToTimeZone(utcOffset, TargetPeriod.Start.ToDateTime()));
UrlParams.Add("vendor", this.Delivery.Parameters["UserName"].ToString());
UrlParams.Add("Password", this.Delivery.Parameters["Password"].ToString());
boFile.SourceUrl = CreateUrl(UrlParams, baseAddress.ToString());
this.Delivery.Files.Add(boFile);
this.Delivery.Save();
#endregion
return(Core.Services.ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
this.Delivery = this.NewDelivery(); // setup delivery
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["FilterDeleted"]))
{
throw new Exception("Missing Configuration Param , FilterDeleted");
}
this.Delivery.Parameters["FilterDeleted"] = this.Instance.Configuration.Options["FilterDeleted"];
if (this.Instance.Configuration.Options.ContainsKey("AppendSitelinks"))
{
this.Delivery.Parameters.Add("AppendSitelinks", this.Instance.Configuration.Options["AppendSitelinks"]);
}
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["KeywordContentId"]))
{
throw new Exception("Missing Configuration Param , KeywordContentId");
}
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["Adwords.MccEmail"]))
{
throw new Exception("Missing Configuration Param , Adwords.MccEmail");
}
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["Adwords.ClientID"]))
{
throw new Exception("Missing Configuration Param , Adwords.ClientID");
}
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["SubChannelName"]))
{
throw new Exception("Missing Configuration Param , SubChannelName");
}
//checking for conflicts
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("{4}-[{0}]-[{1}]-[{2}]-[{3}]",//-[{4}]",//EdgeAccountID , MCC Email ,AdwordsClientID , TimePeriod
this.Instance.AccountID,
this.Instance.Configuration.Options["Adwords.MccEmail"].ToString(),
this.Instance.Configuration.Options["Adwords.ClientID"].ToString(),
this.TimePeriod.ToAbsolute(),
this.Instance.Configuration.Options["SubChannelName"].ToString()
)),
Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID, OriginalID = this.Instance.Configuration.Options["Adwords.ClientID"]
},
Channel = new Data.Objects.Channel()
{
ID = 1
},
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd
}
);
this.Delivery.FileDirectory = Instance.Configuration.Options[Edge.Data.Pipeline.Services.Const.DeliveryServiceConfigurationOptions.FileDirectory];
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery FileDirectory must be configured in configuration file (DeliveryFilesDir)");
}
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.Account = new Edge.Data.Objects.Account()
{
ID = this.Instance.AccountID, OriginalID = this.Instance.Configuration.Options["Adwords.ClientID"]
};
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = 1
};
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Consts.AppSettings.SqlRollbackCommand]
});
// will use ConflictBehavior configuration option to abort or rollback if any conflicts occur
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
#region Must Have Params
this.Delivery.Parameters["IncludeStatus"] = this.Instance.Configuration.Options["IncludeStatus"];
//Get MCC Paramerters
this.Delivery.Parameters["DeveloperToken"] = this.Instance.Configuration.Options["DeveloperToken"];
this.Delivery.Parameters["MccEmail"] = this.Instance.Configuration.Options["Adwords.MccEmail"];
//this.Delivery.Parameters["MccPass"] = Core.Utilities.Encryptor.Dec(this.Instance.Configuration.Options["Adwords.MccPass"].ToString());
this.Delivery.Parameters["KeywordContentId"] = this.Instance.Configuration.Options["KeywordContentId"];
// Get Report types
string[] reportTypeNames = this.Instance.Configuration.Options["Adwords.ReportType"].Split('|');
List <GA.ReportDefinitionReportType> reportTypes = new List <GA.ReportDefinitionReportType>();
foreach (string reportTypeName in reportTypeNames)
{
if (Enum.IsDefined(typeof(GA.ReportDefinitionReportType), reportTypeName))
{
reportTypes.Add((GA.ReportDefinitionReportType)Enum.Parse(typeof(GA.ReportDefinitionReportType), reportTypeName, true));
}
else
{
throw new Exception("Undefined Google Adwords ReportType");
}
}
this.Delivery.Parameters["reportTypes"] = reportTypes;
//Get Account Client Id's
string[] adwordsClientIds = this.Instance.Configuration.Options["Adwords.ClientID"].Split('|');
this.Delivery.Parameters["AdwordsClientIDs"] = adwordsClientIds;
#endregion
#region Nice to have params
//Check for includeZeroImpression
string includeZeroImpression;
if (!String.IsNullOrEmpty(includeZeroImpression = Instance.Configuration.Options["includeZeroImpression"]))
{
this.Delivery.Parameters["includeZeroImpression"] = includeZeroImpression;
}
else
{
this.Delivery.Parameters["includeZeroImpression"] = false;
}
//Check for includeConversionTypes
string includeConversionTypes;
if (!String.IsNullOrEmpty(includeConversionTypes = Instance.Configuration.Options["includeConversionTypes"]))
{
this.Delivery.Parameters["includeConversionTypes"] = includeConversionTypes;
}
else
{
this.Delivery.Parameters["includeConversionTypes"] = false; // deafult
}
//Check for includeDisplaytData
string includeDisplaytData;
if (!String.IsNullOrEmpty(includeDisplaytData = Instance.Configuration.Options["includeDisplaytData"]))
{
this.Delivery.Parameters["includeDisplaytData"] = includeDisplaytData;
}
else
{
this.Delivery.Parameters["includeDisplaytData"] = false; // deafult
}
#endregion
//Creating Delivery files Per Client ID
foreach (string clientId in adwordsClientIds)
{
foreach (GA.ReportDefinitionReportType reportType in (List <GA.ReportDefinitionReportType>) this.Delivery.Parameters["reportTypes"])
{
DeliveryFile file = new DeliveryFile();
file.Name = GoogleStaticReportsNamesUtill._reportNames[reportType];
file.Parameters.Add("ReportType", reportType.ToString());
file.Parameters.Add("ReportFieldsType", ReportDefinitionReportFieldsType.DEFAULT);
file.Parameters.Add("AdwordsClientID", clientId);
this.Delivery.Files.Add(file);
//Handelling conversion
if (Boolean.Parse(this.Delivery.Parameters["includeConversionTypes"].ToString())) // if AD Performance With conversion type is required
{
bool addFile = true;
DeliveryFile conversionFile = new DeliveryFile();
conversionFile.Name = GoogleStaticReportsNamesUtill._reportNames[reportType] + "_Conv";
switch (reportType)
{
case GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT:
conversionFile.Parameters.Add("ReportType", GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT.ToString());
break;
case GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT:
conversionFile.Parameters.Add("ReportType", GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT.ToString());
break;
default: addFile = false; break;
}
if (addFile)
{
conversionFile.Parameters.Add("ReportFieldsType", ReportDefinitionReportFieldsType.CONVERSION);
conversionFile.Parameters.Add("AdwordsClientID", clientId);
this.Delivery.Files.Add(conversionFile);
}
}
//Add Status Reports
DeliveryFile AD_Camp_Adgroups_StatusFile = new DeliveryFile();
AD_Camp_Adgroups_StatusFile.Parameters.Add("ReportType", GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT.ToString());
}
if (Boolean.Parse(this.Delivery.Parameters["includeDisplaytData"].ToString())) // if AD Performance With conversion type is required
{
DeliveryFile file = new DeliveryFile();
file.Name = GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT];
file.Parameters.Add("ReportType", GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT.ToString());
file.Parameters.Add("ReportFieldsType", ReportDefinitionReportFieldsType.DEFAULT);
file.Parameters.Add("AdwordsClientID", clientId);
this.Delivery.Files.Add(file);
}
#region Sitelinks
if (this.Delivery.Parameters.ContainsKey("AppendSitelinks"))
{
if (Boolean.Parse(this.Delivery.Parameters["AppendSitelinks"].ToString()))
{
DeliveryFile siteLinkFile = new DeliveryFile();
siteLinkFile.Name = GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEHOLDER_FEED_ITEM_REPORT];
siteLinkFile.Parameters.Add("ReportType", GA.ReportDefinitionReportType.PLACEHOLDER_FEED_ITEM_REPORT.ToString());
siteLinkFile.Parameters.Add("ReportFieldsType", ReportDefinitionReportFieldsType.DEFAULT);
siteLinkFile.Parameters.Add("AdwordsClientID", clientId);
this.Delivery.Files.Add(siteLinkFile);
}
}
#endregion
#region Status Reports
if (Boolean.Parse(this.Delivery.Parameters["IncludeStatus"].ToString())) // if AD Performance With conversion type is required
{
//1. create file for **** Ad performance with status
DeliveryFile adPerformaceStatusfile = new DeliveryFile();
adPerformaceStatusfile.Name = GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Status";
adPerformaceStatusfile.Parameters.Add("ReportType", GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT.ToString());
adPerformaceStatusfile.Parameters.Add("ReportFieldsType", ReportDefinitionReportFieldsType.STATUS);
adPerformaceStatusfile.Parameters.Add("AdwordsClientID", clientId);
this.Delivery.Files.Add(adPerformaceStatusfile);
//2. create file for **** KWD performance with status
DeliveryFile kwdPerformaceStatusfile = new DeliveryFile();
kwdPerformaceStatusfile.Name = GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT] + "_Status";
kwdPerformaceStatusfile.Parameters.Add("ReportType", GA.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT.ToString());
kwdPerformaceStatusfile.Parameters.Add("ReportFieldsType", ReportDefinitionReportFieldsType.STATUS);
kwdPerformaceStatusfile.Parameters.Add("AdwordsClientID", clientId);
this.Delivery.Files.Add(kwdPerformaceStatusfile);
//3. create file for **** Managed performance with status
DeliveryFile ManagedGDNStatusfile = new DeliveryFile();
ManagedGDNStatusfile.Name = GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT] + "_Status";
ManagedGDNStatusfile.Parameters.Add("ReportType", GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT.ToString());
ManagedGDNStatusfile.Parameters.Add("ReportFieldsType", ReportDefinitionReportFieldsType.STATUS);
ManagedGDNStatusfile.Parameters.Add("AdwordsClientID", clientId);
this.Delivery.Files.Add(ManagedGDNStatusfile);
}
#endregion
}
this.Delivery.Save();
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
bool includeConversionTypes = Boolean.Parse(this.Delivery.Parameters["includeConversionTypes"].ToString());
bool includeDisplaytData = Boolean.Parse(this.Delivery.Parameters["includeDisplaytData"].ToString());
bool ConvertToUSD = Boolean.Parse(this.Delivery.Parameters["ConvertToUSD"].ToString());
//double ConstCurrencyRate = this.Delivery.Parameters.ContainsKey("ConstCurrencyRate") ? Convert.ToDouble(this.Delivery.Parameters["ConstCurrencyRate"]) : 1;
//Status Members
Dictionary <string, ObjectStatus> kwd_Status_Data = new Dictionary <string, ObjectStatus>();
Dictionary <string, ObjectStatus> placement_kwd_Status_Data = new Dictionary <string, ObjectStatus>();
Dictionary <Int64, ObjectStatus> adGroup_Status_Data = new Dictionary <Int64, ObjectStatus>();
Dictionary <Int64, ObjectStatus> ad_Status_Data = new Dictionary <Int64, ObjectStatus>();
Dictionary <Int64, ObjectStatus> campaign_Status_Data = new Dictionary <Int64, ObjectStatus>();
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
string[] requiredHeaders = new string[1];
requiredHeaders[0] = Const.AdPreRequiredHeader;
#region Getting Keywords Data
Dictionary <string, double> _totals = new Dictionary <string, double>();
DeliveryFile _keyWordsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]];
requiredHeaders[0] = Const.AdPreRequiredHeader;
var _keywordsReader = new CsvDynamicReader(_keyWordsFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
_keywordsReader.MatchExactColumns = false;
Dictionary <string, KeywordTarget> _keywordsData = new Dictionary <string, KeywordTarget>();
this.ImportManager.BeginImport(this.Delivery);
using (_keywordsReader)
{
while (_keywordsReader.Read())
{
this.Mappings.OnFieldRequired = field => _keywordsReader.Current[field];
if (_keywordsReader.Current[Const.KeywordIdFieldName] == Const.EOF)
{
break;
}
KeywordPrimaryKey keywordPrimaryKey = new KeywordPrimaryKey()
{
KeywordId = Convert.ToInt64(_keywordsReader.Current[Const.KeywordIdFieldName]),
AdgroupId = Convert.ToInt64(_keywordsReader.Current[Const.AdGroupIdFieldName]),
CampaignId = Convert.ToInt64(_keywordsReader.Current[Const.CampaignIdFieldName])
};
KeywordTarget keyword = new KeywordTarget()
{
OriginalID = _keywordsReader.Current[Const.KeywordIdFieldName],
Keyword = _keywordsReader.Current[Const.KeywordFieldName],
//Status = kwd_Status_Data[keywordPrimaryKey.ToString()]
};
keyword.QualityScore = Convert.ToString(_keywordsReader.Current[Const.QualityScoreFieldName]);
string matchType = _keywordsReader.Current[Const.MatchTypeFieldName];
keyword.MatchType = (KeywordMatchType)Enum.Parse(typeof(KeywordMatchType), matchType, true);
//Setting Tracker for Keyword
if (!String.IsNullOrWhiteSpace(Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName])))
{
keyword.DestinationUrl = Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]);
//setting kwd tracker
//if (((String)(_keywordsReader.Current[Const.DestUrlFieldName])).IndexOf(Const.CreativeIDTrackingValue, StringComparison.OrdinalIgnoreCase) >= 0)
if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"]))
{
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(KeywordTarget)))
{
this.Mappings.Objects[typeof(KeywordTarget)].Apply(keyword);
}
}
}
_keywordsData.Add(keywordPrimaryKey.ToString(), keyword);
}
}
#endregion
Dictionary <string, PlacementTarget> _placementsData = new Dictionary <string, PlacementTarget>();
#region Getting Placements Data
string[] _placementsFileRequiredHeaders = new string[1];
_placementsFileRequiredHeaders[0] = Const.PlacementCriteriaID;
DeliveryFile _PlacementsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]];
var _PlacementsReader = new CsvDynamicReader(_PlacementsFile.OpenContents(compression: FileCompression.Gzip), _placementsFileRequiredHeaders);
using (_PlacementsReader)
{
while (_PlacementsReader.Read())
{
if (_PlacementsReader.Current[Const.PlacementCriteriaID] == Const.EOF)
{
break;
}
//Read data only if managed GDN, otherwise it is an automatic GDN so skip
if (!((String)(_PlacementsReader.Current[Const.PlacementCriteriaID])).Trim().Equals("--"))
{
KeywordPrimaryKey placementPrimaryKey = new KeywordPrimaryKey()
{
KeywordId = Convert.ToInt64(_PlacementsReader.Current[Const.PlacementCriteriaID]),
AdgroupId = Convert.ToInt64(_PlacementsReader.Current[Const.AdGroupIdFieldName]),
CampaignId = Convert.ToInt64(_PlacementsReader.Current[Const.CampaignIdFieldName])
};
PlacementTarget placement = new PlacementTarget()
{
OriginalID = _PlacementsReader.Current[Const.PlacementCriteriaID],
Placement = _PlacementsReader.Current[Const.PlacementFieldName],
PlacementType = PlacementType.Managed,
// Status = placement_kwd_Status_Data[placementPrimaryKey.ToString()]
};
//Setting Tracker for placment
if (!String.IsNullOrWhiteSpace(Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName])))
{
placement.DestinationUrl = Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]);
}
_placementsData.Add(placementPrimaryKey.ToString(), placement);
}
}
}
#endregion
#region Getting Conversions Data
//Get Ads Conversion ( for ex. signup , purchase )
DeliveryFile _conversionFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"];
var _conversionsReader = new CsvDynamicReader(_conversionFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
Dictionary <string, Dictionary <string, long> > importedAdsWithConv = new Dictionary <string, Dictionary <string, long> >();
using (_conversionsReader)
{
while (_conversionsReader.Read())
{
if (_conversionsReader.Current[Const.AdIDFieldName] == Const.EOF) // if end of report
{
break;
}
string conversionKey = String.Format("{0}#{1}", _conversionsReader.Current[Const.AdIDFieldName], _conversionsReader.Current[Const.KeywordIdFieldName]);
Dictionary <string, long> conversionDic = new Dictionary <string, long>();
if (!importedAdsWithConv.TryGetValue(conversionKey, out conversionDic))
{
//ADD conversionKey to importedAdsWithConv
//than add conversion field to importedAdsWithConv : <conversion name , conversion value>
Dictionary <string, long> conversion = new Dictionary <string, long>();
conversion.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]));
importedAdsWithConv.Add(conversionKey, conversion);
}
else // if Key exists
{
// if current add already has current conversion type than add value to the current type
if (!conversionDic.ContainsKey(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])))
{
conversionDic.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]));
}
// else create new conversion type and add the value
else
{
conversionDic[Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])] += Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]);
}
}
}
}
#endregion
#region Getting Ads Data
DeliveryFile _adPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]];
var _adsReader = new CsvDynamicReader(_adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
Dictionary <string, Ad> importedAds = new Dictionary <string, Ad>();
//session.Begin(false);
//this.ImportManager.BeginImport(this.Delivery);
DeliveryOutput currentOutput = Delivery.Outputs.First();
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
_totals.Add(measure.Key, 0);
}
}
using (_adsReader)
{
this.Mappings.OnFieldRequired = field => _adsReader.Current[field];
while (_adsReader.Read())
{
string currencyCode = ((string)(_adsReader.Current.Currency)).ToUpper();
// Adding totals line for validation (checksum)
if (_adsReader.Current[Const.AdIDFieldName] == Const.EOF)
{
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
continue;
}
switch (measure.Key)
{
case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_adsReader.Current.Clicks); break;
case Measure.Common.Cost: _totals[Measure.Common.Cost] =
ConvertToUSD ? this.ConvertToUSD(this.Delivery.Parameters["CurrencyCode"].ToString().ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000;
break;
case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_adsReader.Current.Impressions); break;
}
}
break;
}
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
adMetricsUnit.Output = currentOutput;
Ad ad;
#region Try Get SearchKWD
/***********************************************************************************/
//Creating kwd primary key
KeywordPrimaryKey kwdKey = new KeywordPrimaryKey()
{
AdgroupId = Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName]),
KeywordId = Convert.ToInt64(_adsReader.Current[Const.KeywordIdFieldName]),
CampaignId = Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])
};
//Check if keyword file contains this kwdkey and not a GDN Keyword
String[] GdnKwdIds = this.Delivery.Parameters["KeywordContentId"].ToString().Split(',');
bool IsSearchKwd = false;
KeywordTarget kwd = null;
PlacementTarget placement = null;
if (!GdnKwdIds.Contains(kwdKey.KeywordId.ToString()) && _keywordsData.ContainsKey(kwdKey.ToString()))
{
kwd = new KeywordTarget();
try
{
kwd = _keywordsData[kwdKey.ToString()];
}
catch (Exception)
{
//Creating KWD with OriginalID , since the KWD doesnt exists in KWD report.
kwd = new KeywordTarget {
OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName])
};
}
IsSearchKwd = true;
}
else
{
placement = new PlacementTarget();
try
{
placement = _placementsData[kwdKey.ToString()];
}
catch (Exception)
{
placement.OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]);
placement.PlacementType = PlacementType.Automatic;
placement.Placement = Const.AutoDisplayNetworkName;
}
}
/***********************************************************************************/
#endregion
string adId = _adsReader.Current[Const.AdIDFieldName];
if (!importedAds.ContainsKey(adId))
{
ad = new Ad();
ad.OriginalID = adId;
ad.Channel = new Channel()
{
ID = 1
};
ad.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"]
};
// ad.Status = ad_Status_Data[Convert.ToInt64(adId)];
#region Ad Type
/****************************************************************/
string adTypeColumnValue = Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]);
string devicePreferenceColumnValue = Convert.ToString(_adsReader.Current[Const.AdDevicePreferenceFieldName]);
if (!GoogleAdTypeDic.ContainsKey(adTypeColumnValue))
{
continue;
}
string adTypeEdgeValue = GoogleAdTypeDic[adTypeColumnValue].ToString();
//EdgeAdType atv = (EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);
//is mobile ad ?
if (devicePreferenceColumnValue.Equals(Const.AdDevicePreferenceMobileFieldValue))
{
string mobileValue = string.Format("Mobile {0}", Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]));
//Check if this mobile value exists on dictionary
if (GoogleAdTypeDic.ContainsKey(mobileValue))
{
adTypeEdgeValue = GoogleAdTypeDic[mobileValue].ToString();
}
else
{
adTypeEdgeValue = GoogleAdTypeDic[Const.AdTypeValues.Mobile_ad].ToString();
}
}
ad.ExtraFields[AdType] = (int)(EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);;
/****************************************************************/
#endregion
//Creative
ad.Creatives.Add(new TextCreative {
TextType = TextCreativeType.DisplayUrl, Text = _adsReader.Current[Const.DisplayURLFieldName]
});
#region Ad Tracker segment
/******************************************************/
////Setting Tracker for Ad
if (!String.IsNullOrWhiteSpace(_adsReader.Current[Const.DestUrlFieldName]))
{
ad.DestinationUrl = _adsReader.Current[Const.DestUrlFieldName];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
}
//if Ad doesnt contains tracker than check for kwd tracker
if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"]))
{
if (kwd != null && kwd.Segments != null && kwd.Segments.ContainsKey(this.ImportManager.SegmentTypes[Segment.Common.Tracker]))
{
if (kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] != null && kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value != null)
{
SegmentObject tracker = kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]];
//if value contains ADID than replace ADID with AD original id
tracker.Value = tracker.Value.Replace("ADID", ad.OriginalID);
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] = tracker;
}
}
}
/******************************************************/
#endregion
#region Campaign
/****************************************************************/
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign()
{
OriginalID = _adsReader.Current[Const.CampaignIdFieldName],
Name = _adsReader.Current[Const.CampaignFieldName],
//Status = campaign_Status_Data[Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])]
};
/****************************************************************/
#endregion
#region Image
/****************************************************************/
//Image Type > Create Image
if (String.Equals(Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]), "Image ad"))
{
string adNameField = _adsReader.Current[Const.AdFieldName];
string[] imageParams = adNameField.Trim().Split(new Char[] { ':', ';' }); // Ad name: 468_60_Test7options_Romanian.swf; 468 x 60
ad.Name = imageParams[1].Trim();
ad.Creatives.Add(new ImageCreative()
{
ImageUrl = imageParams[1].Trim(),
ImageSize = imageParams[2].Trim()
});
}
/****************************************************************/
#endregion
#region Text od Display
/****************************************************************/
else //Text ad or Display ad
{
ad.Name = _adsReader.Current[Const.AdFieldName];
ad.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Title,
Text = _adsReader.Current.Ad,
});
ad.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Body,
Text = _adsReader.Current["Description line 1"],
Text2 = _adsReader.Current["Description line 2"]
});
}
/****************************************************************/
#endregion
#region Adgroup
/****************************************************************/
//Insert Adgroup
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
Value = _adsReader.Current[Const.AdGroupFieldName],
OriginalID = _adsReader.Current[Const.AdGroupIdFieldName],
// Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])]
};
/****************************************************************/
#endregion
#region Network
/****************************************************************/
//Insert Network Type Display Network / Search Network
//string networkType = Convert.ToString(_adsReader.Current[Const.NetworkFieldName]);
//if (networkType.Equals(Const.GoogleSearchNetwork))
// networkType = Const.SystemSearchNetwork;
//else if (networkType.Equals(Const.GoogleDisplayNetwork))
// networkType = Const.SystemDisplayNetwork;
//ad.ExtraFields[NetworkType] = networkType;
/****************************************************************/
#endregion
importedAds.Add(adId, ad);
this.ImportManager.ImportAd(ad);
}
else
{
ad = importedAds[adId];
}
adMetricsUnit.Ad = ad;
//INSERTING SEARCH KEYWORD INTO METRICS
if (IsSearchKwd & kwd != null)
{
adMetricsUnit.TargetDimensions = new List <Target>();
adMetricsUnit.TargetDimensions.Add(kwd);
}
//INSERTING GDN KEYWORD INTO METRICS
else if (placement != null)
{
//INSERTING KEYWORD INTO METRICS
adMetricsUnit.TargetDimensions = new List <Target>();
adMetricsUnit.TargetDimensions.Add(placement);
}
//INSERTING METRICS DATA
adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_adsReader.Current.Clicks));
//Currencies Conversion Support
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost],
ConvertToUSD ? this.ConvertToUSD(currencyCode.ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000);
if (ConvertToUSD)
{
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.CostBeforeConversion], (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000);
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.USDConversionRate], Convert.ToDouble(this.ImportManager.CurrencyRates[currencyCode].RateValue));
}
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_adsReader.Current.Impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], Convert.ToDouble(_adsReader.Current[Const.AvgPositionFieldName]));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionOnePerClickFieldName]));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionManyPerClickFieldName]));
//Inserting conversion values
string conversionKey = String.Format("{0}#{1}", ad.OriginalID, _adsReader.Current[Const.KeywordIdFieldName]);
Dictionary <string, long> conversionDic = new Dictionary <string, long>();
if (importedAdsWithConv.TryGetValue(conversionKey, out conversionDic))
{
foreach (var pair in conversionDic)
{
if (GoogleMeasuresDic.ContainsKey(pair.Key))
{
adMetricsUnit.MeasureValues[this.ImportManager.Measures[GoogleMeasuresDic[pair.Key]]] = pair.Value;
}
}
}
adMetricsUnit.Currency = new Currency
{
Code = Convert.ToString(_adsReader.Current.Currency)
};
this.ImportManager.ImportMetrics(adMetricsUnit);
}
#endregion
}
#region Getting Sitelinks Data without metrics
if (this.Delivery.Parameters.ContainsKey("AppendSitelinks") && Boolean.Parse(this.Delivery.Parameters["AppendSitelinks"].ToString()))
{
string[] sitelinksRequiredHeaders = new string[1];
sitelinksRequiredHeaders[0] = Const.PlaceholderFeedItemID;
DeliveryFile _sitelinkPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEHOLDER_FEED_ITEM_REPORT]];
var _sitelinkReader = new CsvDynamicReader(_sitelinkPerformanceFile.OpenContents(compression: FileCompression.Gzip), sitelinksRequiredHeaders);
AdMetricsUnit siteLinkMetricsUnit = new AdMetricsUnit();
siteLinkMetricsUnit.Output = currentOutput;
Ad sitelinkAd;
using (_sitelinkReader)
{
this.Mappings.OnFieldRequired = field => _sitelinkReader.Current[field];
//to do : get site link tracker
string[] sitelinkAttr;
while (_sitelinkReader.Read())
{
if (((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Equals(Const.Sitelink_EOF))
{
break;
}
string sitelinkId = string.Format("{0}{1}{2}", _sitelinkReader.Current[Const.PlaceholderFeedItemID], _sitelinkReader.Current[Const.CampaignIdFieldName], _sitelinkReader.Current[Const.AdGroupIdFieldName]);
sitelinkAd = new Ad();
sitelinkAd.OriginalID = sitelinkId;
sitelinkAd.Channel = new Channel()
{
ID = 1
};
sitelinkAd.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"]
};
//Creative
sitelinkAttr = ((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Split(';');
bool legacy = sitelinkAttr.Count() != 4 ? true : false;
string destUrl = string.Empty;
bool destUrlParsingError = false;
//Checking desturl errors ( we would like to insert only sitelinks that contains desturl ).
try
{
destUrl = sitelinkAttr[1];
}
catch (Exception e)
{
Log.Write("Error while trying to pars destination url from attribute field on sitelink report", e);
destUrlParsingError = true;
}
if (!destUrlParsingError)
{
////Setting Tracker for Ad
if (!String.IsNullOrWhiteSpace(sitelinkAttr[1]))
{
sitelinkAd.DestinationUrl = sitelinkAttr[1];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(sitelinkAd);
}
}
sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign()
{
OriginalID = _sitelinkReader.Current[Const.CampaignIdFieldName],
Name = _sitelinkReader.Current[Const.CampaignFieldName],
};
//Insert Adgroup
sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Campaign = (Campaign)sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
Value = _sitelinkReader.Current[Const.AdGroupFieldName],
OriginalID = _sitelinkReader.Current[Const.AdGroupIdFieldName],
// Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])]
};
sitelinkAd.Creatives.Add(new TextCreative {
TextType = TextCreativeType.DisplayUrl, Text = sitelinkAttr[1]
});
if (!legacy) // only in case it doesnt contains legacy siteink
{
sitelinkAd.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Body,
Text = sitelinkAttr[2],
Text2 = sitelinkAttr[3]
});
}
sitelinkAd.Name = "[Sitelink] " + sitelinkAttr[0];
sitelinkAd.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Title,
Text = "[Sitelink] " + sitelinkAttr[0]
});
//Ad Type
//Note: changed to "sitelink" following Amir request
sitelinkAd.ExtraFields[AdType] = (int)(EdgeAdType.Sitelink);
siteLinkMetricsUnit.Ad = sitelinkAd;
//Setting Default Currency as USD following Amir's request from March 2014
siteLinkMetricsUnit.Currency = new Currency
{
Code = "USD"
};
//INSERTING METRICS DATA
siteLinkMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], 0);
ImportManager.ImportMetrics(siteLinkMetricsUnit);
ImportManager.ImportAd(sitelinkAd);
}
}
}
}// end if
#endregion
currentOutput.Checksum = _totals;
this.ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"];
StringBuilder warningsStr = new StringBuilder();
Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>();
Dictionary <string, Ad> ads = new Dictionary <string, Ad>();
Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >();
DeliveryOutput currentOutput = Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
#region Campaigns
List <string> campaignsFiles = filesByType[Consts.FileTypes.Campaigns];
foreach (var campaignFile in campaignsFiles)
{
DeliveryFile campaigns = this.Delivery.Files[campaignFile];
var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*");
using (campaignsReader)
{
while (campaignsReader.Read())
{
Campaign camp = new Campaign()
{
Name = campaignsReader.Current.name,
OriginalID = Convert.ToString(campaignsReader.Current.id),
};
//string campaignStatus = campaignsReader.Current.campaign_status;
//switch (campaignStatus)
//{
// case "ACTIVE":
// camp.Status = ObjectStatus.Active;
// break;
// case "PAUSED":
// camp.Status = ObjectStatus.Paused;
// break;
// case "DELETED":
// camp.Status = ObjectStatus.Deleted;
// break;
//}
campaignsData.Add(camp.OriginalID, camp);
}
}
}
#endregion
this.ReportProgress(0.1);
#region adGroups And Targeting
List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups];
foreach (var adGroup in adGroupsFiles)
{
DeliveryFile adGroups = this.Delivery.Files[adGroup];
var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*");
using (adGroupsReader)
{
while (adGroupsReader.Read())
{
Ad ad = new Ad();
ad.OriginalID = Convert.ToString(adGroupsReader.Current.id);
ad.Segments = new Dictionary <Segment, SegmentObject>();
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = campaignsData[Convert.ToString(adGroupsReader.Current.campaign_id)];
ad.Name = adGroupsReader.Current.name;
ad.Channel = new Channel()
{
ID = 6
};
ad.Account = new Account()
{
ID = this.Delivery.Account.ID,
OriginalID = this.Delivery.Account.OriginalID.ToString()
};
if (Instance.Configuration.Options.ContainsKey("AutoAdGroupSegment") && Instance.Configuration.Options["AutoAdGroupSegment"].ToLower() == "true")
{
string[] delimiter = new string[1];
delimiter[0] = string.Empty;
if (!Instance.Configuration.Options.ContainsKey("AdGroupDelimiter"))
{
Edge.Core.Utilities.Log.Write(string.Format("Facebook{0}", this), Core.Utilities.LogMessageType.Warning);
}
else
{
delimiter[0] = Instance.Configuration.Options["AdGroupDelimiter"];
}
//Note for Ronen - this section if relevant for AD SETS
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
Value = delimiter[0] == string.Empty ? ad.Name : ad.Name.Split(delimiter, StringSplitOptions.None)[0],
OriginalID = delimiter[0] == string.Empty ? (ad.Name + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID) :
(ad.Name.Split(delimiter, StringSplitOptions.None)[0] + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID)
};
}
else
{
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Value = ad.Name,
OriginalID = ad.Name + ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]].OriginalID + ad.Account.ID
};
}
// adgroup targeting
string age_min = string.Empty;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min"))
{
age_min = adGroupsReader.Current.targeting["age_min"];
}
if (!string.IsNullOrEmpty(age_min))
{
AgeTarget ageTarget = new AgeTarget()
{
FromAge = int.Parse(age_min), ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"])
};
ad.Targets.Add(ageTarget);
}
List <object> genders = null;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders"))
{
genders = adGroupsReader.Current.targeting["genders"];
}
if (genders != null)
{
foreach (object gender in genders)
{
GenderTarget genderTarget = new GenderTarget();
if (gender.ToString() == "1")
{
genderTarget.Gender = Gender.Male;
}
else if (gender.ToString() == "2")
{
genderTarget.Gender = Gender.Female;
}
else
{
genderTarget.Gender = Gender.Unspecified;
}
genderTarget.OriginalID = gender.ToString();
ad.Targets.Add(genderTarget);
}
}
if (adGroupsReader.Current.creative_ids != null)
{
foreach (string creative in adGroupsReader.Current.creative_ids)
{
if (!adsBycreatives.ContainsKey(creative))
{
adsBycreatives.Add(creative, new List <Ad>());
}
adsBycreatives[creative].Add(ad);
}
}
ads.Add(ad.OriginalID, ad);
}
}
}
#endregion
#region AdGroupStats start new import session
//GetAdGroupStats
#region for validation
foreach (var measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(measure.Key))
{
currentOutput.Checksum.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP
}
}
}
#endregion
if (filesByType.ContainsKey(Consts.FileTypes.ConversionsStats))
{
List <string> convStatsFiles = filesByType[Consts.FileTypes.ConversionsStats];
foreach (var convStat in convStatsFiles)
{
DeliveryFile conversionStatsFile = this.Delivery.Files[convStat];
//Getting Next conversion file URL
var conversionStatsReader = new JsonDynamicReader(conversionStatsFile.OpenContents(), "$.data[*].*");
using (conversionStatsReader)
{
//Get Stats from conversion file
}
}
}
if (filesByType.ContainsKey(Consts.FileTypes.AdGroupStats))
{
List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats];
foreach (var adGroupStat in adGroupStatsFiles)
{
DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat];
var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*");
using (adGroupStatsReader)
{
while (adGroupStatsReader.Read())
{
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
adMetricsUnit.Output = currentOutput;
adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
Ad tempAd;
if (adGroupStatsReader.Current.adgroup_id != null)
{
if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd))
{
adMetricsUnit.Ad = tempAd;
//adMetricsUnit.PeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
//adMetricsUnit.PeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
// Common and Facebook specific meausures
/* Sets totals for validations */
if (currentOutput.Checksum.ContainsKey(Measure.Common.Clicks))
{
currentOutput.Checksum[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Impressions))
{
currentOutput.Checksum[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Cost))
{
currentOutput.Checksum[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d;
}
/* Sets measures values */
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(adGroupStatsReader.Current.clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueClicks], Convert.ToInt64(adGroupStatsReader.Current.unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(adGroupStatsReader.Current.impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueImpressions], Convert.ToInt64(adGroupStatsReader.Current.unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d);
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Actions], 0);
//adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections));
this.ImportManager.ImportMetrics(adMetricsUnit);
}
else
{
warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID));
}
}
else
{
warningsStr.AppendLine("adGroupStatsReader.Current.id=null");
}
}
this.ReportProgress(0.4);
}
#endregion
this.ReportProgress(0.6);
#region Creatives
List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives];
Dictionary <string, string> usedCreatives = new Dictionary <string, string>();
foreach (string creative in creativeFiles)
{
DeliveryFile creativeFile = Delivery.Files[creative];
var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (adGroupCreativesReader)
{
this.Mappings.OnFieldRequired = field => adGroupCreativesReader.Current[field];
while (adGroupCreativesReader.Read())
{
List <Ad> adsByCreativeID = null;
if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
usedCreatives.Add(adGroupCreativesReader.Current.id, adGroupCreativesReader.Current.id);
adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.id];
}
}
if (adsByCreativeID != null)
{
foreach (Ad ad in adsByCreativeID)
{
// Ronen , please see Naama's file of Ad types
ad.DestinationUrl = adGroupCreativesReader.Current.object_url;
/*Get Data from Mapping E.g Tracker*/
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
ad.Creatives = new List <Creative>();
switch ((string)adGroupCreativesReader.Current.type)
{
//case "8": // deprecated
//case "9": // deprecated
//case "10": // deprecated
//case "16": // deprecated
//case "17": // deprecated
//case "19": // deprecated
case "25":
{
TextCreative sponserStory = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Title,
Text = "Sponsored Story",
};
ad.DestinationUrl = "Sponsored Story";
ad.Creatives.Add(sponserStory);
break;
}
case "27":
{
TextCreative sponserStory = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Title,
Text = "Page Ads for a Page post"
};
ad.DestinationUrl = "Page Ads for a Page post";
ad.Creatives.Add(sponserStory);
break;
}
case "1":
case "2":
case "3":
case "4":
case "12":
{
ImageCreative ic = new ImageCreative()
{
ImageUrl = adGroupCreativesReader.Current.image_url,
OriginalID = adGroupCreativesReader.Current.id
//Name = adGroupCreativesReader.Current.name
};
if (!string.IsNullOrEmpty(ic.ImageUrl))
{
ad.Creatives.Add(ic);
}
TextCreative bc = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Body,
Text = adGroupCreativesReader.Current.body
//Name = adGroupCreativesReader.Current.name
};
if (!string.IsNullOrEmpty(bc.Text))
{
ad.Creatives.Add(bc);
}
//bug creative type =9 story like
TextCreative tc = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Title,
Text = adGroupCreativesReader.Current.title
};
if (!string.IsNullOrEmpty(bc.Text))
{
ad.Creatives.Add(tc);
}
break;
}
default:
{
TextCreative unknown = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.id,
TextType = TextCreativeType.Title,
Text = "UnKnown creative"
};
ad.DestinationUrl = "UnKnown creative";
ad.Creatives.Add(unknown);
break;
}
}
this.ImportManager.ImportAd(ad);
}
}
//TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS)
}
}
#endregion
}
}
}
currentOutput.Status = DeliveryOutputStatus.Imported;
this.ImportManager.EndImport();
if (!string.IsNullOrEmpty(warningsStr.ToString()))
{
Log.Write(warningsStr.ToString(), LogMessageType.Warning);
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
#region Init General
// ...............................
// SETUP
this.Delivery = NewDelivery();
// This is for finding conflicting services
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
OriginalID = this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString()
};
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = 6
};
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("facebook-[{0}]-[{1}]-[{2}]",
this.Instance.AccountID,
this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString(),
this.Delivery.TimePeriodDefinition.ToAbsolute())),
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd,
Account = this.Delivery.Account,
Channel = this.Delivery.Channel
});
// Now that we have a new delivery, start adding values
this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory];
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]))
{
throw new Exception("facebook base url must be configured!");
}
_baseAddress = new Uri(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]);
if (string.IsNullOrEmpty(this.Instance.Configuration.Options["API_Purchases_Field_Name"]))
{
throw new Exception("facebook 'API_Purchases_Field_Name' must be configured!");
}
this.Delivery.Parameters.Add("API_Purchases_Field_Name", this.Instance.Configuration.Options["API_Purchases_Field_Name"]);
#endregion
#region Init Delivery Files
Dictionary <string, string> methodParams = new Dictionary <string, string>();
string methodUrl;
DeliveryFile deliveryFile = new DeliveryFile();
#region adgroupstats
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupStats;
methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TimePeriod.Start.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TimePeriod.End.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.AdgroupStatus, "[%27ARCHIVED%27,%27ACTIVE%27,%27PAUSED%27]");
methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupStats);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.AdGroupStats.ToString()));
this.Delivery.Files.Add(deliveryFile);
#endregion
//this.ReportProgress(0.4);
#region Creating report stats for conversion data
///<summary>
///27.10.2014
///
///Ad Report Stats provides Ads API to fetch statistics at the adaccount level. You can query data per ad account,
///campaigns or ads. One query will always return data for one ad account only.
///The API is very generic and enables you to fetch data for arbitrary time ranges. You can group the data by 1-90 days, monthly or all days.
///You can break down the data by gender, age, placement or country. You can filter by specific values (spend > 100) and sort by most columns.
///By default the API returns data for all ads/campaigns that had delivery during given time range. That means you don't need to set any status
///filters to display ARCHIVED and DELETED adgroups/campaigns, as they are all included as far as they have delivery.
///method=post&async=true
/// data_columns=['campaign_group_id','campaign_id','adgroup_id','actions']
/// time_ranges=[{'day_start':{'day':1,'month':8,'year':2014},'day_stop':{'day':2,'month':8,'year':2014}}]
/// access_token=CAACZAMUPZCAd0BAK5eHzaiDVgvM0tGaP8b2EPpxpzj5upCkKThsfH5eUf8TLGpsmQVHpoKbTahNBE5AcJ3XkVu0hun4Woi7ZCx5lx560FBSmfUkGOOBh7eOVbZBLsAeAfs5ojZBeiXWJkpYNF2OksZCv7qk0OYZCQ6u0xA4TgOQSMQUJRqPM0oSrJg2zJaXTO0ZD&method=post&async=true&actions_group_by=['action_type']
///</summary>
//getting conversion report Id from facebook
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetReportStats);
methodParams.Add("time_ranges", ConvertToFacebookDateTime(TimePeriod.Start.ToDateTime(), datetimeRangeFormat: true));
methodParams.Add("data_columns", "[%27campaign_group_id%27,%27campaign_id%27,%27adgroup_id%27,%27actions%27]");
methodParams.Add("method", "post");
methodParams.Add("async", "true");
methodParams.Add("actions_group_by", "['action_type']");
string downloadStringUrl = string.Format("{0}access_token={1}", GetMethodUrl(methodUrl, methodParams), this.Instance.Configuration.Options[FacebookConfigurationOptions.Auth_AccessToken]);
Int64 reportId;
try
{
using (var webClient = new System.Net.WebClient())
{
string Id = webClient.DownloadString(downloadStringUrl);
reportId = Int64.Parse(Id.Trim('"'));
}
}
catch (Exception e)
{
throw new Exception(string.Format("Error while trying to parse conversion report id , for more details {0}", downloadStringUrl), e);
}
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.ConversionsStats;
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetReportStats);
methodParams.Add("report_run_id", reportId.ToString());
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.ConversionsStats.ToString()));
this.Delivery.Files.Add(deliveryFile);
#endregion
#region adgroup
/*
* Summary
* An ad group contains the data necessary for an ad, such as bid type, bid info,
* targeting data, creative elements, and campaign information. Each ad group is
* associated with a campaign and all ad groups in a campaign have the same daily
* or lifetime budget and schedule.
* */
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroup;
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroups);
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.AdgroupStatus, "[%27ACTIVE%27,%27PAUSED%27,%27CAMPAIGN_PAUSED%27,%27CAMPAIGN_GROUP_PAUSED%27,%27PENDING_REVIEW%27,%27PREAPPROVED%27,%27DISAPPROVED%27,%27PENDING_BILLING_INFO%27,%27ARCHIVED%27]");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.AdGroupFields].ToString());
}
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.AdGroups);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region AdSet- Formally Campaigns
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Campaigns;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.CampaignFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.CampaignFields].ToString());
}
// methodParams.Add("redownload", "true");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaignsAdSets);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.AdSets);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region Campaigns - New Structure
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.CampaignGroups;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.CampaignStatus, "[%27ARCHIVED%27,%27ACTIVE%27,%27PAUSED%27]");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.CampaignGroupsFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.CampaignGroupsFields].ToString());
}
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaignsGroups);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.CampaignGroups);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region Creatives
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Creatives;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.AdGroupCreativeFields].ToString());
}
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupCreatives);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Creatives);
this.Delivery.Files.Add(deliveryFile);
#endregion
#endregion
//this.ReportProgress(0.9);
this.Delivery.Save();
this.ReportProgress(1);
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
currentOutput = Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
// TODO: add checks for delivery state
string[] requiredHeaders;
requiredHeaders = new string[1];
requiredHeaders[0] = WS.CampaignPerformanceReportColumn.CampaignName.ToString();
_campaignsCache = new Dictionary <string, Campaign>();
_adCache = new Dictionary <long, Ad>();
DeliveryFile adReport = this.Delivery.Files[Const.Files.AdReport];
DeliveryFile campaignReport = this.Delivery.Files[Const.Files.CampaignReport];
FileInfo campaignReportFileInfo = campaignReport.GetFileInfo(ArchiveType.Zip);
string[] campaignReportSubFiles = campaignReportFileInfo.GetSubFiles();
var campaignReportReader = new CsvDynamicReader(
campaignReport.OpenContents(subLocation: campaignReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders
);
#region Reading campaigns file
while (campaignReportReader.Read())
{
string accountNameColVal = campaignReportReader.Current[WS.CampaignPerformanceReportColumn.AccountName.ToString()];
if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim().Contains(endOfFileMicrosoftCorporation.Trim())) //end of file
{
break;
}
Campaign campaign = CreateCampaign(campaignReportReader.Current);
_campaignsCache.Add(campaign.Name, campaign);
}
this.ReportProgress(0.2);
#endregion
#region Read the ad report, and build a lookup table for later
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
ImportManager.BeginImport(this.Delivery);
// create the ad report reader
requiredHeaders = new string[1];
requiredHeaders[0] = WS.AdPerformanceReportColumn.AdId.ToString();
string[] adReportSubFiles = adReport.GetFileInfo(ArchiveType.Zip).GetSubFiles();
var adReportReader = new CsvDynamicReader(
adReport.OpenContents(subLocation: adReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders);
//read
using (adReportReader)
{
while (adReportReader.Read())
{
// create the ad
this.Mappings.OnFieldRequired = field => adReportReader.Current[field];
string accountNameColVal = adReportReader.Current[WS.AdPerformanceReportColumn.AccountName.ToString()]; //end of file
if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim() == endOfFileMicrosoftCorporation.Trim())
{
break;
}
Ad ad = CreateAd(adReportReader.Current, ImportManager);
//TRACKER
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
//ADDING CAMPAIGN
ad.Segments[this.ImportManager.
SegmentTypes[Segment.Common.Campaign]] = _campaignsCache[adReportReader.Current[WS.AdPerformanceReportColumn.CampaignName.ToString()]];
//CREATING Adgroup
AdGroup adGroup = new AdGroup()
{
Value = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupName.ToString()],
Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
OriginalID = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupId.ToString()]
};
ad.Segments.Add(ImportManager.SegmentTypes[Segment.Common.AdGroup], adGroup);
//CRAETING AD TYPE
string adTypeKey = Convert.ToString(adReportReader.Current[WS.AdPerformanceReportColumn.AdType.ToString()]);
ad.ExtraFields[AdType] = AdCenterAdTypeDic[adTypeKey];
ImportManager.ImportAd(ad);
_adCache.Add(long.Parse(ad.OriginalID), ad);
}
}
this.ReportProgress(0.7);
#endregion
// ...............................................................
#region Read the keyword report, cross reference it with the ad data, and commit
// Read the keyword report, cross reference it with the ad data, and commit
// The name of the time period column is specified by the initializer, depending on the report
DeliveryFile keywordReport = this.Delivery.Files[Const.Files.KeywordReport];
string timePeriodColumn = keywordReport.Parameters[Const.Parameters.TimePeriodColumnName] as string;
// // create the keyword report reader
requiredHeaders = new string[1];
requiredHeaders[0] = "Keyword";
string[] keywordReportSubFiles = keywordReport.GetFileInfo(ArchiveType.Zip).GetSubFiles();
var keywordReportReader = new CsvDynamicReader(
keywordReport.OpenContents(subLocation: keywordReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders);
//Added by Shay for validation
foreach (KeyValuePair <string, Measure> measure in ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum.Add(measure.Key, 0);
}
}
// read and save in transaction
using (keywordReportReader)
{
while (keywordReportReader.Read())
{
string GregorianDateColVal = keywordReportReader.Current.GregorianDate;
if (GregorianDateColVal.Trim() == string.Empty || GregorianDateColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) //end of file
{
break;
}
// get the unit from the keyword report, and add the missing ad data
AdMetricsUnit unit = CreateMetrics(keywordReportReader.Current, timePeriodColumn, ImportManager);
ImportManager.ImportMetrics(unit);
// Validation information:
currentOutput.Checksum[Measure.Common.Clicks] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Clicks]];
currentOutput.Checksum[Measure.Common.Cost] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Cost]];
currentOutput.Checksum[Measure.Common.Impressions] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Impressions]];
}
ImportManager.EndImport();
ReportProgress(1);
}
#endregion
}
return(ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"];
StringBuilder warningsStr = new StringBuilder();
Dictionary <string, double> _totalsValidation = new Dictionary <string, double>();
Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>();
Dictionary <string, Ad> ads = new Dictionary <string, Ad>();
Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >();
#region Campaigns
List <string> campaignsFiles = filesByType[Consts.FileTypes.Campaigns];
foreach (var campaignFile in campaignsFiles)
{
DeliveryFile campaigns = this.Delivery.Files[campaignFile];
var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*");
using (campaignsReader)
{
while (campaignsReader.Read())
{
Campaign camp = new Campaign()
{
Name = campaignsReader.Current.name,
OriginalID = Convert.ToString(campaignsReader.Current.campaign_id),
Channel = new Channel()
{
ID = 6
},
Account = new Account()
{
ID = this.Delivery.Account.ID,
OriginalID = this.Delivery.Account.OriginalID.ToString()
}
};
long campaignStatus = long.Parse(campaignsReader.Current.campaign_status);
switch (campaignStatus)
{
case 1:
camp.Status = ObjectStatus.Active;
break;
case 2:
camp.Status = ObjectStatus.Paused;
break;
case 3:
camp.Status = ObjectStatus.Deleted;
break;
}
campaignsData.Add(camp.OriginalID, camp);
}
campaigns.History.Add(DeliveryOperation.Imported, Instance.InstanceID);
}
}
#endregion
this.ReportProgress(0.1);
#region adGroups And Targeting
List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups];
foreach (var adGroup in adGroupsFiles)
{
DeliveryFile adGroups = this.Delivery.Files[adGroup];
var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*");
using (adGroupsReader)
{
while (adGroupsReader.Read())
{
Ad ad = new Ad();
ad.OriginalID = Convert.ToString(adGroupsReader.Current.ad_id);
ad.Campaign = campaignsData[Convert.ToString(adGroupsReader.Current.campaign_id)];
ad.Name = adGroupsReader.Current.name;
if (Instance.Configuration.Options.ContainsKey("AutoAdGroupSegment") && Instance.Configuration.Options["AutoAdGroupSegment"].ToLower() == "true")
{
string[] delimiter = new string[1];
delimiter[0] = string.Empty;
if (!Instance.Configuration.Options.ContainsKey("AdGroupDelimiter"))
{
Edge.Core.Utilities.Log.Write(string.Format("Facebook{0}", this), Core.Utilities.LogMessageType.Warning);
}
else
{
delimiter[0] = Instance.Configuration.Options["AdGroupDelimiter"];
}
ad.Segments[Segment.AdGroupSegment] = new SegmentValue();
ad.Segments[Segment.AdGroupSegment].Value = delimiter[0] == string.Empty ? ad.Name : ad.Name.Split(delimiter, StringSplitOptions.None)[0];
ad.Segments[Segment.AdGroupSegment].OriginalID = delimiter[0] == string.Empty ? (ad.Name + ad.Campaign.OriginalID + ad.Campaign.Account.ID) :
(ad.Name.Split(delimiter, StringSplitOptions.None)[0] + ad.Campaign.OriginalID + ad.Campaign.Account.ID);
}
else
{
ad.Segments[Segment.AdGroupSegment] = new SegmentValue();
ad.Segments[Segment.AdGroupSegment].Value = ad.Name;
ad.Segments[Segment.AdGroupSegment].OriginalID = ad.Name + ad.Campaign.OriginalID + ad.Campaign.Account.ID;
}
// adgroup targeting
string age_min = string.Empty;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min"))
{
age_min = adGroupsReader.Current.targeting["age_min"];
}
if (!string.IsNullOrEmpty(age_min))
{
AgeTarget ageTarget = new AgeTarget()
{
FromAge = int.Parse(age_min), ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"])
};
ad.Targets.Add(ageTarget);
}
List <object> genders = null;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders"))
{
genders = adGroupsReader.Current.targeting["genders"];
}
if (genders != null)
{
foreach (object gender in genders)
{
GenderTarget genderTarget = new GenderTarget();
if (gender.ToString() == "1")
{
genderTarget.Gender = Gender.Male;
}
else if (gender.ToString() == "2")
{
genderTarget.Gender = Gender.Female;
}
else
{
genderTarget.Gender = Gender.Unspecified;
}
genderTarget.OriginalID = gender.ToString();
ad.Targets.Add(genderTarget);
}
}
if (adGroupsReader.Current.creative_ids != null)
{
foreach (string creative in adGroupsReader.Current.creative_ids)
{
if (!adsBycreatives.ContainsKey(creative))
{
adsBycreatives.Add(creative, new List <Ad>());
}
adsBycreatives[creative].Add(ad);
}
}
ads.Add(ad.OriginalID, ad);
}
adGroups.History.Add(DeliveryOperation.Imported, Instance.InstanceID);
}
}
#endregion
#region AdGroupStats start new import session
//GetAdGroupStats
using (var session = new AdMetricsImportManager(this.Instance.InstanceID))
{
session.BeginImport(this.Delivery);
#region for validation
foreach (var measure in session.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!_totalsValidation.ContainsKey(measure.Key))
{
_totalsValidation.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP
}
}
}
#endregion
if (filesByType.ContainsKey(Consts.FileTypes.AdGroupStats))
{
List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats];
foreach (var adGroupStat in adGroupStatsFiles)
{
DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat];
var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*");
using (adGroupStatsReader)
{
while (adGroupStatsReader.Read())
{
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
Ad tempAd;
if (adGroupStatsReader.Current.adgroup_id != null)
{
if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd))
{
adMetricsUnit.Ad = tempAd;
adMetricsUnit.PeriodStart = this.Delivery.TargetPeriod.Start.ToDateTime();
adMetricsUnit.PeriodEnd = this.Delivery.TargetPeriod.End.ToDateTime();
// Common and Facebook specific meausures
if (_totalsValidation.ContainsKey(Measure.Common.Clicks))
{
_totalsValidation[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks);
}
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Clicks]] = Convert.ToInt64(adGroupStatsReader.Current.clicks);
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.UniqueClicks]] = Convert.ToInt64(adGroupStatsReader.Current.unique_clicks);
if (_totalsValidation.ContainsKey(Measure.Common.Impressions))
{
_totalsValidation[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions);
}
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Impressions]] = Convert.ToInt64(adGroupStatsReader.Current.impressions);
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.UniqueImpressions]] = Convert.ToInt64(adGroupStatsReader.Current.unique_impressions);
if (_totalsValidation.ContainsKey(Measure.Common.Cost))
{
_totalsValidation[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d;
}
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Cost]] = Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d);
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d);
//adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Actions], double.Parse(adGroupStatsReader.Current.actions));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Actions], 0);
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections));
adMetricsUnit.TargetMatches = new List <Target>();
session.ImportMetrics(adMetricsUnit);
}
else
{
warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID));
}
}
else
{
warningsStr.AppendLine("adGroupStatsReader.Current.id=null");
}
}
adGroupStats.History.Add(DeliveryOperation.Imported, Instance.InstanceID); //TODO: HISTORY WHEN?PROCCESED IS AFTER DATABASE'?
this.ReportProgress(0.4);
}
#endregion
this.ReportProgress(0.6);
#region Creatives
List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives];
Dictionary <string, string> usedCreatives = new Dictionary <string, string>();
foreach (string creative in creativeFiles)
{
DeliveryFile creativeFile = Delivery.Files[creative];
var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (adGroupCreativesReader)
{
while (adGroupCreativesReader.Read())
{
List <Ad> adsByCreativeID = null;
if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.creative_id))
{
if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.creative_id))
{
usedCreatives.Add(adGroupCreativesReader.Current.creative_id, adGroupCreativesReader.Current.creative_id);
adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.creative_id];
}
}
if (adsByCreativeID != null)
{
foreach (Ad ad in adsByCreativeID)
{
ad.DestinationUrl = adGroupCreativesReader.Current.link_url;
if (!string.IsNullOrEmpty(ad.DestinationUrl))
{
SegmentValue tracker = this.AutoSegments.ExtractSegmentValue(Segment.TrackerSegment, ad.DestinationUrl);
if (tracker != null)
{
ad.Segments[Segment.TrackerSegment] = tracker;
}
}
ad.Creatives = new List <Creative>();
switch ((string)adGroupCreativesReader.Current.type)
{
case "8":
case "9":
case "10":
case "16":
case "17":
case "19":
case "25":
{
TextCreative sponserStory = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.creative_id,
TextType = TextCreativeType.Title,
Text = "Sponsored Story"
};
ad.Creatives.Add(sponserStory);
break;
}
case "27":
{
TextCreative sponserStory = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.creative_id,
TextType = TextCreativeType.Title,
Text = "Page Ads for a Page post"
};
ad.Creatives.Add(sponserStory);
break;
}
case "1":
case "2":
case "3":
case "4":
case "12":
{
ImageCreative ic = new ImageCreative()
{
ImageUrl = adGroupCreativesReader.Current.image_url,
OriginalID = adGroupCreativesReader.Current.creative_id
//Name = adGroupCreativesReader.Current.name
};
if (!string.IsNullOrEmpty(ic.ImageUrl))
{
ad.Creatives.Add(ic);
}
TextCreative bc = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.creative_id,
TextType = TextCreativeType.Body,
Text = adGroupCreativesReader.Current.body
//Name = adGroupCreativesReader.Current.name
};
if (!string.IsNullOrEmpty(bc.Text))
{
ad.Creatives.Add(bc);
}
//bug creative type =9 story like
TextCreative tc = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.creative_id,
TextType = TextCreativeType.Title,
Text = adGroupCreativesReader.Current.title
};
if (!string.IsNullOrEmpty(bc.Text))
{
ad.Creatives.Add(tc);
}
break;
}
}
session.ImportAd(ad);
}
}
//TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS)
}
creativeFile.History.Add(DeliveryOperation.Imported, Instance.InstanceID);
}
#endregion
}
}
}
session.HistoryEntryParameters.Add(Edge.Data.Pipeline.Common.Importing.Consts.DeliveryHistoryParameters.ChecksumTotals, _totalsValidation);
session.EndImport();
if (!string.IsNullOrEmpty(warningsStr.ToString()))
{
Log.Write(warningsStr.ToString(), LogMessageType.Warning);
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
#region Init General
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
OriginalID = this.Instance.Configuration.Options["AdCenter.CustomerID"].ToString()
};
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = 14
};
this.Delivery.FileDirectory = Instance.Configuration.Options["DeliveryFilesDir"];
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery FileDirectory must be configured in configuration file (DeliveryFilesDir)");
}
// Copy some options as delivery parameters
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("Microsoft-AdCenter-[{0}]-[{1}]-[{2}]",
this.Instance.AccountID,
this.Instance.Configuration.Options["AdCenter.CustomerID"].ToString(),
this.TimePeriod.ToAbsolute()
)),
Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID, OriginalID = this.Instance.Configuration.Options["AdCenter.CustomerID"]
},
Channel = new Data.Objects.Channel()
{
ID = 14
},
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd
}
);
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
// ...............................
// Now that we have a new delivery, start adding values
this.ReportProgress(0.2);
#endregion
// Wrapper for adCenter API
AdCenterApi adCenterApi = new AdCenterApi(this);
// ................................
// Campaign report
this.Delivery.Files.Add(new DeliveryFile()
{
Name = Const.Files.CampaignReport
});
ReportProgress(0.33);
// ................................
// Ad report
ReportProgress(0.33);
// ................................
// Keyword report
this.Delivery.Files.Add(new DeliveryFile()
{
Name = Const.Files.KeywordReport
});
this.Delivery.Files[Const.Files.KeywordReport].Parameters.Add(Const.Parameters.TimePeriodColumnName, AdCenterApi.GetTimePeriodColumnName(WS.ReportAggregation.Daily));
this.Delivery.Files.Add(new DeliveryFile()
{
Name = Const.Files.AdReport
});
ReportProgress(0.33);
// Save with success
this.Delivery.Save();
return(ServiceOutcome.Success);
}