protected override ServiceOutcome DoPipelineWork()
{
currentOutput = Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
// TODO: add checks for delivery state
string[] requiredHeaders;
requiredHeaders = new string[1];
requiredHeaders[0] = WS.CampaignPerformanceReportColumn.CampaignName.ToString();
_campaignsCache = new Dictionary <string, Campaign>();
_adCache = new Dictionary <long, Ad>();
DeliveryFile adReport = this.Delivery.Files[Const.Files.AdReport];
DeliveryFile campaignReport = this.Delivery.Files[Const.Files.CampaignReport];
FileInfo campaignReportFileInfo = campaignReport.GetFileInfo(ArchiveType.Zip);
string[] campaignReportSubFiles = campaignReportFileInfo.GetSubFiles();
var campaignReportReader = new CsvDynamicReader(
campaignReport.OpenContents(subLocation: campaignReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders
);
#region Reading campaigns file
while (campaignReportReader.Read())
{
string accountNameColVal = campaignReportReader.Current[WS.CampaignPerformanceReportColumn.AccountName.ToString()];
if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim().Contains(endOfFileMicrosoftCorporation.Trim())) //end of file
{
break;
}
Campaign campaign = CreateCampaign(campaignReportReader.Current);
_campaignsCache.Add(campaign.Name, campaign);
}
this.ReportProgress(0.2);
#endregion
#region Read the ad report, and build a lookup table for later
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
ImportManager.BeginImport(this.Delivery);
// create the ad report reader
requiredHeaders = new string[1];
requiredHeaders[0] = WS.AdPerformanceReportColumn.AdId.ToString();
string[] adReportSubFiles = adReport.GetFileInfo(ArchiveType.Zip).GetSubFiles();
var adReportReader = new CsvDynamicReader(
adReport.OpenContents(subLocation: adReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders);
//read
using (adReportReader)
{
while (adReportReader.Read())
{
// create the ad
this.Mappings.OnFieldRequired = field => adReportReader.Current[field];
string accountNameColVal = adReportReader.Current[WS.AdPerformanceReportColumn.AccountName.ToString()]; //end of file
if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim() == endOfFileMicrosoftCorporation.Trim())
{
break;
}
Ad ad = CreateAd(adReportReader.Current, ImportManager);
//TRACKER
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
//ADDING CAMPAIGN
ad.Segments[this.ImportManager.
SegmentTypes[Segment.Common.Campaign]] = _campaignsCache[adReportReader.Current[WS.AdPerformanceReportColumn.CampaignName.ToString()]];
//CREATING Adgroup
AdGroup adGroup = new AdGroup()
{
Value = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupName.ToString()],
Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
OriginalID = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupId.ToString()]
};
ad.Segments.Add(ImportManager.SegmentTypes[Segment.Common.AdGroup], adGroup);
//CRAETING AD TYPE
string adTypeKey = Convert.ToString(adReportReader.Current[WS.AdPerformanceReportColumn.AdType.ToString()]);
ad.ExtraFields[AdType] = AdCenterAdTypeDic[adTypeKey];
ImportManager.ImportAd(ad);
_adCache.Add(long.Parse(ad.OriginalID), ad);
}
}
this.ReportProgress(0.7);
#endregion
// ...............................................................
#region Read the keyword report, cross reference it with the ad data, and commit
// Read the keyword report, cross reference it with the ad data, and commit
// The name of the time period column is specified by the initializer, depending on the report
DeliveryFile keywordReport = this.Delivery.Files[Const.Files.KeywordReport];
string timePeriodColumn = keywordReport.Parameters[Const.Parameters.TimePeriodColumnName] as string;
// // create the keyword report reader
requiredHeaders = new string[1];
requiredHeaders[0] = "Keyword";
string[] keywordReportSubFiles = keywordReport.GetFileInfo(ArchiveType.Zip).GetSubFiles();
var keywordReportReader = new CsvDynamicReader(
keywordReport.OpenContents(subLocation: keywordReportSubFiles[0], archiveType: ArchiveType.Zip),
requiredHeaders);
//Added by Shay for validation
foreach (KeyValuePair <string, Measure> measure in ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
currentOutput.Checksum.Add(measure.Key, 0);
}
}
// read and save in transaction
using (keywordReportReader)
{
while (keywordReportReader.Read())
{
string GregorianDateColVal = keywordReportReader.Current.GregorianDate;
if (GregorianDateColVal.Trim() == string.Empty || GregorianDateColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) //end of file
{
break;
}
// get the unit from the keyword report, and add the missing ad data
AdMetricsUnit unit = CreateMetrics(keywordReportReader.Current, timePeriodColumn, ImportManager);
ImportManager.ImportMetrics(unit);
// Validation information:
currentOutput.Checksum[Measure.Common.Clicks] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Clicks]];
currentOutput.Checksum[Measure.Common.Cost] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Cost]];
currentOutput.Checksum[Measure.Common.Impressions] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Impressions]];
}
ImportManager.EndImport();
ReportProgress(1);
}
#endregion
}
return(ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
bool includeConversionTypes = Boolean.Parse(this.Delivery.Parameters["includeConversionTypes"].ToString());
bool includeDisplaytData = Boolean.Parse(this.Delivery.Parameters["includeDisplaytData"].ToString());
bool ConvertToUSD = Boolean.Parse(this.Delivery.Parameters["ConvertToUSD"].ToString());
//double ConstCurrencyRate = this.Delivery.Parameters.ContainsKey("ConstCurrencyRate") ? Convert.ToDouble(this.Delivery.Parameters["ConstCurrencyRate"]) : 1;
//Status Members
Dictionary <string, ObjectStatus> kwd_Status_Data = new Dictionary <string, ObjectStatus>();
Dictionary <string, ObjectStatus> placement_kwd_Status_Data = new Dictionary <string, ObjectStatus>();
Dictionary <Int64, ObjectStatus> adGroup_Status_Data = new Dictionary <Int64, ObjectStatus>();
Dictionary <Int64, ObjectStatus> ad_Status_Data = new Dictionary <Int64, ObjectStatus>();
Dictionary <Int64, ObjectStatus> campaign_Status_Data = new Dictionary <Int64, ObjectStatus>();
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
string[] requiredHeaders = new string[1];
requiredHeaders[0] = Const.AdPreRequiredHeader;
#region Getting Keywords Data
Dictionary <string, double> _totals = new Dictionary <string, double>();
DeliveryFile _keyWordsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]];
requiredHeaders[0] = Const.AdPreRequiredHeader;
var _keywordsReader = new CsvDynamicReader(_keyWordsFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
_keywordsReader.MatchExactColumns = false;
Dictionary <string, KeywordTarget> _keywordsData = new Dictionary <string, KeywordTarget>();
this.ImportManager.BeginImport(this.Delivery);
using (_keywordsReader)
{
while (_keywordsReader.Read())
{
this.Mappings.OnFieldRequired = field => _keywordsReader.Current[field];
if (_keywordsReader.Current[Const.KeywordIdFieldName] == Const.EOF)
{
break;
}
KeywordPrimaryKey keywordPrimaryKey = new KeywordPrimaryKey()
{
KeywordId = Convert.ToInt64(_keywordsReader.Current[Const.KeywordIdFieldName]),
AdgroupId = Convert.ToInt64(_keywordsReader.Current[Const.AdGroupIdFieldName]),
CampaignId = Convert.ToInt64(_keywordsReader.Current[Const.CampaignIdFieldName])
};
KeywordTarget keyword = new KeywordTarget()
{
OriginalID = _keywordsReader.Current[Const.KeywordIdFieldName],
Keyword = _keywordsReader.Current[Const.KeywordFieldName],
//Status = kwd_Status_Data[keywordPrimaryKey.ToString()]
};
keyword.QualityScore = Convert.ToString(_keywordsReader.Current[Const.QualityScoreFieldName]);
string matchType = _keywordsReader.Current[Const.MatchTypeFieldName];
keyword.MatchType = (KeywordMatchType)Enum.Parse(typeof(KeywordMatchType), matchType, true);
//Setting Tracker for Keyword
if (!String.IsNullOrWhiteSpace(Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName])))
{
keyword.DestinationUrl = Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]);
//setting kwd tracker
//if (((String)(_keywordsReader.Current[Const.DestUrlFieldName])).IndexOf(Const.CreativeIDTrackingValue, StringComparison.OrdinalIgnoreCase) >= 0)
if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"]))
{
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(KeywordTarget)))
{
this.Mappings.Objects[typeof(KeywordTarget)].Apply(keyword);
}
}
}
_keywordsData.Add(keywordPrimaryKey.ToString(), keyword);
}
}
#endregion
Dictionary <string, PlacementTarget> _placementsData = new Dictionary <string, PlacementTarget>();
#region Getting Placements Data
string[] _placementsFileRequiredHeaders = new string[1];
_placementsFileRequiredHeaders[0] = Const.PlacementCriteriaID;
DeliveryFile _PlacementsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]];
var _PlacementsReader = new CsvDynamicReader(_PlacementsFile.OpenContents(compression: FileCompression.Gzip), _placementsFileRequiredHeaders);
using (_PlacementsReader)
{
while (_PlacementsReader.Read())
{
if (_PlacementsReader.Current[Const.PlacementCriteriaID] == Const.EOF)
{
break;
}
//Read data only if managed GDN, otherwise it is an automatic GDN so skip
if (!((String)(_PlacementsReader.Current[Const.PlacementCriteriaID])).Trim().Equals("--"))
{
KeywordPrimaryKey placementPrimaryKey = new KeywordPrimaryKey()
{
KeywordId = Convert.ToInt64(_PlacementsReader.Current[Const.PlacementCriteriaID]),
AdgroupId = Convert.ToInt64(_PlacementsReader.Current[Const.AdGroupIdFieldName]),
CampaignId = Convert.ToInt64(_PlacementsReader.Current[Const.CampaignIdFieldName])
};
PlacementTarget placement = new PlacementTarget()
{
OriginalID = _PlacementsReader.Current[Const.PlacementCriteriaID],
Placement = _PlacementsReader.Current[Const.PlacementFieldName],
PlacementType = PlacementType.Managed,
// Status = placement_kwd_Status_Data[placementPrimaryKey.ToString()]
};
//Setting Tracker for placment
if (!String.IsNullOrWhiteSpace(Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName])))
{
placement.DestinationUrl = Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]);
}
_placementsData.Add(placementPrimaryKey.ToString(), placement);
}
}
}
#endregion
#region Getting Conversions Data
//Get Ads Conversion ( for ex. signup , purchase )
DeliveryFile _conversionFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"];
var _conversionsReader = new CsvDynamicReader(_conversionFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
Dictionary <string, Dictionary <string, long> > importedAdsWithConv = new Dictionary <string, Dictionary <string, long> >();
using (_conversionsReader)
{
while (_conversionsReader.Read())
{
if (_conversionsReader.Current[Const.AdIDFieldName] == Const.EOF) // if end of report
{
break;
}
string conversionKey = String.Format("{0}#{1}", _conversionsReader.Current[Const.AdIDFieldName], _conversionsReader.Current[Const.KeywordIdFieldName]);
Dictionary <string, long> conversionDic = new Dictionary <string, long>();
if (!importedAdsWithConv.TryGetValue(conversionKey, out conversionDic))
{
//ADD conversionKey to importedAdsWithConv
//than add conversion field to importedAdsWithConv : <conversion name , conversion value>
Dictionary <string, long> conversion = new Dictionary <string, long>();
conversion.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]));
importedAdsWithConv.Add(conversionKey, conversion);
}
else // if Key exists
{
// if current add already has current conversion type than add value to the current type
if (!conversionDic.ContainsKey(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])))
{
conversionDic.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]));
}
// else create new conversion type and add the value
else
{
conversionDic[Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])] += Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]);
}
}
}
}
#endregion
#region Getting Ads Data
DeliveryFile _adPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]];
var _adsReader = new CsvDynamicReader(_adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders);
Dictionary <string, Ad> importedAds = new Dictionary <string, Ad>();
//session.Begin(false);
//this.ImportManager.BeginImport(this.Delivery);
DeliveryOutput currentOutput = Delivery.Outputs.First();
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
_totals.Add(measure.Key, 0);
}
}
using (_adsReader)
{
this.Mappings.OnFieldRequired = field => _adsReader.Current[field];
while (_adsReader.Read())
{
string currencyCode = ((string)(_adsReader.Current.Currency)).ToUpper();
// Adding totals line for validation (checksum)
if (_adsReader.Current[Const.AdIDFieldName] == Const.EOF)
{
foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures)
{
if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
continue;
}
switch (measure.Key)
{
case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_adsReader.Current.Clicks); break;
case Measure.Common.Cost: _totals[Measure.Common.Cost] =
ConvertToUSD ? this.ConvertToUSD(this.Delivery.Parameters["CurrencyCode"].ToString().ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000;
break;
case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_adsReader.Current.Impressions); break;
}
}
break;
}
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
adMetricsUnit.Output = currentOutput;
Ad ad;
#region Try Get SearchKWD
/***********************************************************************************/
//Creating kwd primary key
KeywordPrimaryKey kwdKey = new KeywordPrimaryKey()
{
AdgroupId = Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName]),
KeywordId = Convert.ToInt64(_adsReader.Current[Const.KeywordIdFieldName]),
CampaignId = Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])
};
//Check if keyword file contains this kwdkey and not a GDN Keyword
String[] GdnKwdIds = this.Delivery.Parameters["KeywordContentId"].ToString().Split(',');
bool IsSearchKwd = false;
KeywordTarget kwd = null;
PlacementTarget placement = null;
if (!GdnKwdIds.Contains(kwdKey.KeywordId.ToString()) && _keywordsData.ContainsKey(kwdKey.ToString()))
{
kwd = new KeywordTarget();
try
{
kwd = _keywordsData[kwdKey.ToString()];
}
catch (Exception)
{
//Creating KWD with OriginalID , since the KWD doesnt exists in KWD report.
kwd = new KeywordTarget {
OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName])
};
}
IsSearchKwd = true;
}
else
{
placement = new PlacementTarget();
try
{
placement = _placementsData[kwdKey.ToString()];
}
catch (Exception)
{
placement.OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]);
placement.PlacementType = PlacementType.Automatic;
placement.Placement = Const.AutoDisplayNetworkName;
}
}
/***********************************************************************************/
#endregion
string adId = _adsReader.Current[Const.AdIDFieldName];
if (!importedAds.ContainsKey(adId))
{
ad = new Ad();
ad.OriginalID = adId;
ad.Channel = new Channel()
{
ID = 1
};
ad.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"]
};
// ad.Status = ad_Status_Data[Convert.ToInt64(adId)];
#region Ad Type
/****************************************************************/
string adTypeColumnValue = Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]);
string devicePreferenceColumnValue = Convert.ToString(_adsReader.Current[Const.AdDevicePreferenceFieldName]);
if (!GoogleAdTypeDic.ContainsKey(adTypeColumnValue))
{
continue;
}
string adTypeEdgeValue = GoogleAdTypeDic[adTypeColumnValue].ToString();
//EdgeAdType atv = (EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);
//is mobile ad ?
if (devicePreferenceColumnValue.Equals(Const.AdDevicePreferenceMobileFieldValue))
{
string mobileValue = string.Format("Mobile {0}", Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]));
//Check if this mobile value exists on dictionary
if (GoogleAdTypeDic.ContainsKey(mobileValue))
{
adTypeEdgeValue = GoogleAdTypeDic[mobileValue].ToString();
}
else
{
adTypeEdgeValue = GoogleAdTypeDic[Const.AdTypeValues.Mobile_ad].ToString();
}
}
ad.ExtraFields[AdType] = (int)(EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);;
/****************************************************************/
#endregion
//Creative
ad.Creatives.Add(new TextCreative {
TextType = TextCreativeType.DisplayUrl, Text = _adsReader.Current[Const.DisplayURLFieldName]
});
#region Ad Tracker segment
/******************************************************/
////Setting Tracker for Ad
if (!String.IsNullOrWhiteSpace(_adsReader.Current[Const.DestUrlFieldName]))
{
ad.DestinationUrl = _adsReader.Current[Const.DestUrlFieldName];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
}
//if Ad doesnt contains tracker than check for kwd tracker
if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"]))
{
if (kwd != null && kwd.Segments != null && kwd.Segments.ContainsKey(this.ImportManager.SegmentTypes[Segment.Common.Tracker]))
{
if (kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] != null && kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value != null)
{
SegmentObject tracker = kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]];
//if value contains ADID than replace ADID with AD original id
tracker.Value = tracker.Value.Replace("ADID", ad.OriginalID);
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] = tracker;
}
}
}
/******************************************************/
#endregion
#region Campaign
/****************************************************************/
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign()
{
OriginalID = _adsReader.Current[Const.CampaignIdFieldName],
Name = _adsReader.Current[Const.CampaignFieldName],
//Status = campaign_Status_Data[Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])]
};
/****************************************************************/
#endregion
#region Image
/****************************************************************/
//Image Type > Create Image
if (String.Equals(Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]), "Image ad"))
{
string adNameField = _adsReader.Current[Const.AdFieldName];
string[] imageParams = adNameField.Trim().Split(new Char[] { ':', ';' }); // Ad name: 468_60_Test7options_Romanian.swf; 468 x 60
ad.Name = imageParams[1].Trim();
ad.Creatives.Add(new ImageCreative()
{
ImageUrl = imageParams[1].Trim(),
ImageSize = imageParams[2].Trim()
});
}
/****************************************************************/
#endregion
#region Text od Display
/****************************************************************/
else //Text ad or Display ad
{
ad.Name = _adsReader.Current[Const.AdFieldName];
ad.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Title,
Text = _adsReader.Current.Ad,
});
ad.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Body,
Text = _adsReader.Current["Description line 1"],
Text2 = _adsReader.Current["Description line 2"]
});
}
/****************************************************************/
#endregion
#region Adgroup
/****************************************************************/
//Insert Adgroup
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
Value = _adsReader.Current[Const.AdGroupFieldName],
OriginalID = _adsReader.Current[Const.AdGroupIdFieldName],
// Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])]
};
/****************************************************************/
#endregion
#region Network
/****************************************************************/
//Insert Network Type Display Network / Search Network
//string networkType = Convert.ToString(_adsReader.Current[Const.NetworkFieldName]);
//if (networkType.Equals(Const.GoogleSearchNetwork))
// networkType = Const.SystemSearchNetwork;
//else if (networkType.Equals(Const.GoogleDisplayNetwork))
// networkType = Const.SystemDisplayNetwork;
//ad.ExtraFields[NetworkType] = networkType;
/****************************************************************/
#endregion
importedAds.Add(adId, ad);
this.ImportManager.ImportAd(ad);
}
else
{
ad = importedAds[adId];
}
adMetricsUnit.Ad = ad;
//INSERTING SEARCH KEYWORD INTO METRICS
if (IsSearchKwd & kwd != null)
{
adMetricsUnit.TargetDimensions = new List <Target>();
adMetricsUnit.TargetDimensions.Add(kwd);
}
//INSERTING GDN KEYWORD INTO METRICS
else if (placement != null)
{
//INSERTING KEYWORD INTO METRICS
adMetricsUnit.TargetDimensions = new List <Target>();
adMetricsUnit.TargetDimensions.Add(placement);
}
//INSERTING METRICS DATA
adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_adsReader.Current.Clicks));
//Currencies Conversion Support
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost],
ConvertToUSD ? this.ConvertToUSD(currencyCode.ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000);
if (ConvertToUSD)
{
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.CostBeforeConversion], (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000);
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.USDConversionRate], Convert.ToDouble(this.ImportManager.CurrencyRates[currencyCode].RateValue));
}
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_adsReader.Current.Impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], Convert.ToDouble(_adsReader.Current[Const.AvgPositionFieldName]));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionOnePerClickFieldName]));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionManyPerClickFieldName]));
//Inserting conversion values
string conversionKey = String.Format("{0}#{1}", ad.OriginalID, _adsReader.Current[Const.KeywordIdFieldName]);
Dictionary <string, long> conversionDic = new Dictionary <string, long>();
if (importedAdsWithConv.TryGetValue(conversionKey, out conversionDic))
{
foreach (var pair in conversionDic)
{
if (GoogleMeasuresDic.ContainsKey(pair.Key))
{
adMetricsUnit.MeasureValues[this.ImportManager.Measures[GoogleMeasuresDic[pair.Key]]] = pair.Value;
}
}
}
adMetricsUnit.Currency = new Currency
{
Code = Convert.ToString(_adsReader.Current.Currency)
};
this.ImportManager.ImportMetrics(adMetricsUnit);
}
#endregion
}
#region Getting Sitelinks Data without metrics
if (this.Delivery.Parameters.ContainsKey("AppendSitelinks") && Boolean.Parse(this.Delivery.Parameters["AppendSitelinks"].ToString()))
{
string[] sitelinksRequiredHeaders = new string[1];
sitelinksRequiredHeaders[0] = Const.PlaceholderFeedItemID;
DeliveryFile _sitelinkPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEHOLDER_FEED_ITEM_REPORT]];
var _sitelinkReader = new CsvDynamicReader(_sitelinkPerformanceFile.OpenContents(compression: FileCompression.Gzip), sitelinksRequiredHeaders);
AdMetricsUnit siteLinkMetricsUnit = new AdMetricsUnit();
siteLinkMetricsUnit.Output = currentOutput;
Ad sitelinkAd;
using (_sitelinkReader)
{
this.Mappings.OnFieldRequired = field => _sitelinkReader.Current[field];
//to do : get site link tracker
string[] sitelinkAttr;
while (_sitelinkReader.Read())
{
if (((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Equals(Const.Sitelink_EOF))
{
break;
}
string sitelinkId = string.Format("{0}{1}{2}", _sitelinkReader.Current[Const.PlaceholderFeedItemID], _sitelinkReader.Current[Const.CampaignIdFieldName], _sitelinkReader.Current[Const.AdGroupIdFieldName]);
sitelinkAd = new Ad();
sitelinkAd.OriginalID = sitelinkId;
sitelinkAd.Channel = new Channel()
{
ID = 1
};
sitelinkAd.Account = new Account {
ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"]
};
//Creative
sitelinkAttr = ((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Split(';');
bool legacy = sitelinkAttr.Count() != 4 ? true : false;
string destUrl = string.Empty;
bool destUrlParsingError = false;
//Checking desturl errors ( we would like to insert only sitelinks that contains desturl ).
try
{
destUrl = sitelinkAttr[1];
}
catch (Exception e)
{
Log.Write("Error while trying to pars destination url from attribute field on sitelink report", e);
destUrlParsingError = true;
}
if (!destUrlParsingError)
{
////Setting Tracker for Ad
if (!String.IsNullOrWhiteSpace(sitelinkAttr[1]))
{
sitelinkAd.DestinationUrl = sitelinkAttr[1];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(sitelinkAd);
}
}
sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign()
{
OriginalID = _sitelinkReader.Current[Const.CampaignIdFieldName],
Name = _sitelinkReader.Current[Const.CampaignFieldName],
};
//Insert Adgroup
sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup()
{
Campaign = (Campaign)sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]],
Value = _sitelinkReader.Current[Const.AdGroupFieldName],
OriginalID = _sitelinkReader.Current[Const.AdGroupIdFieldName],
// Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])]
};
sitelinkAd.Creatives.Add(new TextCreative {
TextType = TextCreativeType.DisplayUrl, Text = sitelinkAttr[1]
});
if (!legacy) // only in case it doesnt contains legacy siteink
{
sitelinkAd.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Body,
Text = sitelinkAttr[2],
Text2 = sitelinkAttr[3]
});
}
sitelinkAd.Name = "[Sitelink] " + sitelinkAttr[0];
sitelinkAd.Creatives.Add(new TextCreative
{
TextType = TextCreativeType.Title,
Text = "[Sitelink] " + sitelinkAttr[0]
});
//Ad Type
//Note: changed to "sitelink" following Amir request
sitelinkAd.ExtraFields[AdType] = (int)(EdgeAdType.Sitelink);
siteLinkMetricsUnit.Ad = sitelinkAd;
//Setting Default Currency as USD following Amir's request from March 2014
siteLinkMetricsUnit.Currency = new Currency
{
Code = "USD"
};
//INSERTING METRICS DATA
siteLinkMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], 0);
siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], 0);
ImportManager.ImportMetrics(siteLinkMetricsUnit);
ImportManager.ImportAd(sitelinkAd);
}
}
}
}// end if
#endregion
currentOutput.Checksum = _totals;
this.ImportManager.EndImport();
}
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"];
StringBuilder warningsStr = new StringBuilder();
Dictionary <string, double> _totalsValidation = new Dictionary <string, double>();
Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>();
Dictionary <string, Ad> ads = new Dictionary <string, Ad>();
Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >();
#region Campaigns
List <string> campaignsFiles = filesByType[Consts.FileTypes.Campaigns];
foreach (var campaignFile in campaignsFiles)
{
DeliveryFile campaigns = this.Delivery.Files[campaignFile];
var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*");
using (campaignsReader)
{
while (campaignsReader.Read())
{
Campaign camp = new Campaign()
{
Name = campaignsReader.Current.name,
OriginalID = Convert.ToString(campaignsReader.Current.campaign_id),
Channel = new Channel()
{
ID = 6
},
Account = new Account()
{
ID = this.Delivery.Account.ID,
OriginalID = this.Delivery.Account.OriginalID.ToString()
}
};
long campaignStatus = long.Parse(campaignsReader.Current.campaign_status);
switch (campaignStatus)
{
case 1:
camp.Status = ObjectStatus.Active;
break;
case 2:
camp.Status = ObjectStatus.Paused;
break;
case 3:
camp.Status = ObjectStatus.Deleted;
break;
}
campaignsData.Add(camp.OriginalID, camp);
}
campaigns.History.Add(DeliveryOperation.Imported, Instance.InstanceID);
}
}
#endregion
this.ReportProgress(0.1);
#region adGroups And Targeting
List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups];
foreach (var adGroup in adGroupsFiles)
{
DeliveryFile adGroups = this.Delivery.Files[adGroup];
var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*");
using (adGroupsReader)
{
while (adGroupsReader.Read())
{
Ad ad = new Ad();
ad.OriginalID = Convert.ToString(adGroupsReader.Current.ad_id);
ad.Campaign = campaignsData[Convert.ToString(adGroupsReader.Current.campaign_id)];
ad.Name = adGroupsReader.Current.name;
if (Instance.Configuration.Options.ContainsKey("AutoAdGroupSegment") && Instance.Configuration.Options["AutoAdGroupSegment"].ToLower() == "true")
{
string[] delimiter = new string[1];
delimiter[0] = string.Empty;
if (!Instance.Configuration.Options.ContainsKey("AdGroupDelimiter"))
{
Edge.Core.Utilities.Log.Write(string.Format("Facebook{0}", this), Core.Utilities.LogMessageType.Warning);
}
else
{
delimiter[0] = Instance.Configuration.Options["AdGroupDelimiter"];
}
ad.Segments[Segment.AdGroupSegment] = new SegmentValue();
ad.Segments[Segment.AdGroupSegment].Value = delimiter[0] == string.Empty ? ad.Name : ad.Name.Split(delimiter, StringSplitOptions.None)[0];
ad.Segments[Segment.AdGroupSegment].OriginalID = delimiter[0] == string.Empty ? (ad.Name + ad.Campaign.OriginalID + ad.Campaign.Account.ID) :
(ad.Name.Split(delimiter, StringSplitOptions.None)[0] + ad.Campaign.OriginalID + ad.Campaign.Account.ID);
}
else
{
ad.Segments[Segment.AdGroupSegment] = new SegmentValue();
ad.Segments[Segment.AdGroupSegment].Value = ad.Name;
ad.Segments[Segment.AdGroupSegment].OriginalID = ad.Name + ad.Campaign.OriginalID + ad.Campaign.Account.ID;
}
// adgroup targeting
string age_min = string.Empty;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min"))
{
age_min = adGroupsReader.Current.targeting["age_min"];
}
if (!string.IsNullOrEmpty(age_min))
{
AgeTarget ageTarget = new AgeTarget()
{
FromAge = int.Parse(age_min), ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"])
};
ad.Targets.Add(ageTarget);
}
List <object> genders = null;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders"))
{
genders = adGroupsReader.Current.targeting["genders"];
}
if (genders != null)
{
foreach (object gender in genders)
{
GenderTarget genderTarget = new GenderTarget();
if (gender.ToString() == "1")
{
genderTarget.Gender = Gender.Male;
}
else if (gender.ToString() == "2")
{
genderTarget.Gender = Gender.Female;
}
else
{
genderTarget.Gender = Gender.Unspecified;
}
genderTarget.OriginalID = gender.ToString();
ad.Targets.Add(genderTarget);
}
}
if (adGroupsReader.Current.creative_ids != null)
{
foreach (string creative in adGroupsReader.Current.creative_ids)
{
if (!adsBycreatives.ContainsKey(creative))
{
adsBycreatives.Add(creative, new List <Ad>());
}
adsBycreatives[creative].Add(ad);
}
}
ads.Add(ad.OriginalID, ad);
}
adGroups.History.Add(DeliveryOperation.Imported, Instance.InstanceID);
}
}
#endregion
#region AdGroupStats start new import session
//GetAdGroupStats
using (var session = new AdMetricsImportManager(this.Instance.InstanceID))
{
session.BeginImport(this.Delivery);
#region for validation
foreach (var measure in session.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!_totalsValidation.ContainsKey(measure.Key))
{
_totalsValidation.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP
}
}
}
#endregion
if (filesByType.ContainsKey(Consts.FileTypes.AdGroupStats))
{
List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats];
foreach (var adGroupStat in adGroupStatsFiles)
{
DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat];
var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*");
using (adGroupStatsReader)
{
while (adGroupStatsReader.Read())
{
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
Ad tempAd;
if (adGroupStatsReader.Current.adgroup_id != null)
{
if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd))
{
adMetricsUnit.Ad = tempAd;
adMetricsUnit.PeriodStart = this.Delivery.TargetPeriod.Start.ToDateTime();
adMetricsUnit.PeriodEnd = this.Delivery.TargetPeriod.End.ToDateTime();
// Common and Facebook specific meausures
if (_totalsValidation.ContainsKey(Measure.Common.Clicks))
{
_totalsValidation[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks);
}
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Clicks]] = Convert.ToInt64(adGroupStatsReader.Current.clicks);
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.UniqueClicks]] = Convert.ToInt64(adGroupStatsReader.Current.unique_clicks);
if (_totalsValidation.ContainsKey(Measure.Common.Impressions))
{
_totalsValidation[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions);
}
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Impressions]] = Convert.ToInt64(adGroupStatsReader.Current.impressions);
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.UniqueImpressions]] = Convert.ToInt64(adGroupStatsReader.Current.unique_impressions);
if (_totalsValidation.ContainsKey(Measure.Common.Cost))
{
_totalsValidation[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d;
}
adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Cost]] = Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d);
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d);
//adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Actions], double.Parse(adGroupStatsReader.Current.actions));
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Actions], 0);
adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections));
adMetricsUnit.TargetMatches = new List <Target>();
session.ImportMetrics(adMetricsUnit);
}
else
{
warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID));
}
}
else
{
warningsStr.AppendLine("adGroupStatsReader.Current.id=null");
}
}
adGroupStats.History.Add(DeliveryOperation.Imported, Instance.InstanceID); //TODO: HISTORY WHEN?PROCCESED IS AFTER DATABASE'?
this.ReportProgress(0.4);
}
#endregion
this.ReportProgress(0.6);
#region Creatives
List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives];
Dictionary <string, string> usedCreatives = new Dictionary <string, string>();
foreach (string creative in creativeFiles)
{
DeliveryFile creativeFile = Delivery.Files[creative];
var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (adGroupCreativesReader)
{
while (adGroupCreativesReader.Read())
{
List <Ad> adsByCreativeID = null;
if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.creative_id))
{
if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.creative_id))
{
usedCreatives.Add(adGroupCreativesReader.Current.creative_id, adGroupCreativesReader.Current.creative_id);
adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.creative_id];
}
}
if (adsByCreativeID != null)
{
foreach (Ad ad in adsByCreativeID)
{
ad.DestinationUrl = adGroupCreativesReader.Current.link_url;
if (!string.IsNullOrEmpty(ad.DestinationUrl))
{
SegmentValue tracker = this.AutoSegments.ExtractSegmentValue(Segment.TrackerSegment, ad.DestinationUrl);
if (tracker != null)
{
ad.Segments[Segment.TrackerSegment] = tracker;
}
}
ad.Creatives = new List <Creative>();
switch ((string)adGroupCreativesReader.Current.type)
{
case "8":
case "9":
case "10":
case "16":
case "17":
case "19":
case "25":
{
TextCreative sponserStory = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.creative_id,
TextType = TextCreativeType.Title,
Text = "Sponsored Story"
};
ad.Creatives.Add(sponserStory);
break;
}
case "27":
{
TextCreative sponserStory = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.creative_id,
TextType = TextCreativeType.Title,
Text = "Page Ads for a Page post"
};
ad.Creatives.Add(sponserStory);
break;
}
case "1":
case "2":
case "3":
case "4":
case "12":
{
ImageCreative ic = new ImageCreative()
{
ImageUrl = adGroupCreativesReader.Current.image_url,
OriginalID = adGroupCreativesReader.Current.creative_id
//Name = adGroupCreativesReader.Current.name
};
if (!string.IsNullOrEmpty(ic.ImageUrl))
{
ad.Creatives.Add(ic);
}
TextCreative bc = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.creative_id,
TextType = TextCreativeType.Body,
Text = adGroupCreativesReader.Current.body
//Name = adGroupCreativesReader.Current.name
};
if (!string.IsNullOrEmpty(bc.Text))
{
ad.Creatives.Add(bc);
}
//bug creative type =9 story like
TextCreative tc = new TextCreative()
{
OriginalID = adGroupCreativesReader.Current.creative_id,
TextType = TextCreativeType.Title,
Text = adGroupCreativesReader.Current.title
};
if (!string.IsNullOrEmpty(bc.Text))
{
ad.Creatives.Add(tc);
}
break;
}
}
session.ImportAd(ad);
}
}
//TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS)
}
creativeFile.History.Add(DeliveryOperation.Imported, Instance.InstanceID);
}
#endregion
}
}
}
session.HistoryEntryParameters.Add(Edge.Data.Pipeline.Common.Importing.Consts.DeliveryHistoryParameters.ChecksumTotals, _totalsValidation);
session.EndImport();
if (!string.IsNullOrEmpty(warningsStr.ToString()))
{
Log.Write(warningsStr.ToString(), LogMessageType.Warning);
}
}
return(Core.Services.ServiceOutcome.Success);
}