private void LoadPlacements(DeliveryFile file, string[] headers, FileCompression compression = FileCompression.Gzip) { if (file == null) { throw new ArgumentException("Placement delivery file does not exist"); } _placementsCache.Clear(); using (var placementsReader = new CsvDynamicReader(file.OpenContents(compression: compression), headers)) { Mappings.OnFieldRequired = fieldName => placementsReader.Current[fieldName]; placementsReader.MatchExactColumns = false; while (placementsReader.Read()) { if (placementsReader.Current[AdWordsConst.PlacementIdFieldName] == AdWordsConst.EOF) { break; } var placementPrimaryKey = new KeywordPrimaryKey { KeywordId = Convert.ToInt64(placementsReader.Current[AdWordsConst.PlacementIdFieldName]), AdgroupId = Convert.ToInt64(placementsReader.Current[AdWordsConst.AdGroupIdFieldName]), CampaignId = Convert.ToInt64(placementsReader.Current[AdWordsConst.CampaignIdFieldName]) }; var placement = new PlacementTarget(); PlacementMappings.Apply(placement); _placementsCache.Add(placementPrimaryKey.ToString(), placement); } } }
protected override MetricsUnit GetSampleMetrics() { var headers = new[] { AdWordsConst.AdPreRequiredHeader }; // load sample keywords var file = new DeliveryFile { Location = Configuration.Parameters.Get <string>("KeywordSampleFile") }; LoadKeywords(file, headers, FileCompression.None); file = new DeliveryFile { Location = Configuration.Parameters.Get <string>("PlacementSampleFile") }; LoadPlacements(file, new[] { AdWordsConst.AutoPlacRequiredHeader }, FileCompression.None); // load ad using (_adsReader = new CsvDynamicReader(Configuration.Parameters.Get <string>("AdSampleFile"), headers)) { Mappings.OnFieldRequired = fieldName => _adsReader.Current[fieldName]; if (_adsReader.Read()) { CurrentMetricsUnit = new MetricsUnit { GetEdgeField = GetEdgeField, Output = new DeliveryOutput() }; MetricsMappings.Apply(CurrentMetricsUnit); return(CurrentMetricsUnit); } } throw new ConfigurationErrorsException(String.Format("Failed to read sample metrics from file: {0}", Configuration.Parameters.Get <string>("AdSampleFile"))); }
private void LoadKeywords(DeliveryFile file, string[] headers, FileCompression compression = FileCompression.Gzip) { if (file == null) throw new ArgumentException("Keywords delivery file does not exist"); _keywordsCache.Clear(); using (var keywordsReader = new CsvDynamicReader(file.OpenContents(compression: compression), headers)) { Mappings.OnFieldRequired = fieldName => keywordsReader.Current[fieldName]; keywordsReader.MatchExactColumns = false; while (keywordsReader.Read()) { if (keywordsReader.Current[AdWordsConst.KeywordIdFieldName] == AdWordsConst.EOF) break; var keywordPrimaryKey = new KeywordPrimaryKey { KeywordId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.KeywordIdFieldName]), AdgroupId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.AdGroupIdFieldName]), CampaignId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.CampaignIdFieldName]) }; var keyword = new KeywordTarget(); KeywordMappings.Apply(keyword); _keywordsCache.Add(keywordPrimaryKey.ToString(), keyword); } } }
private void LoadConversions(DeliveryFile file, string[] headers) { if (file == null) { throw new ArgumentException("Ad conversions delivery file does not exist"); } _importedAdsWithConv.Clear(); using (var conversionsReader = new CsvDynamicReader(file.OpenContents(compression: FileCompression.Gzip), headers)) { while (conversionsReader.Read()) { if (conversionsReader.Current[AdWordsConst.AdIDFieldName] == AdWordsConst.EOF) { break; // if end of report } string conversionKey = String.Format("{0}#{1}#{2}", conversionsReader.Current[AdWordsConst.AdIDFieldName], conversionsReader.Current[AdWordsConst.KeywordIdFieldName], conversionsReader.Current[AdWordsConst.DateFieldName]); var trackingPurpose = Convert.ToString(conversionsReader.Current[AdWordsConst.ConversionTrackingPurposeFieldName]); var manyPerClick = Convert.ToInt64(conversionsReader.Current[AdWordsConst.ConversionManyPerClickFieldName]); if (!_importedAdsWithConv.ContainsKey(conversionKey)) { // add new conversion key with new dictionary of tracking purpose and clicks _importedAdsWithConv.Add(conversionKey, new Dictionary <string, long> { { trackingPurpose, manyPerClick } }); } else // if conversion key exists { if (_importedAdsWithConv[conversionKey].ContainsKey(trackingPurpose)) { // if purpose exists --> sum to existing value _importedAdsWithConv[conversionKey][trackingPurpose] += manyPerClick; } else { // create new entry for new tracking purpose _importedAdsWithConv[conversionKey].Add(trackingPurpose, manyPerClick); } } } } }
protected override Core.Services.ServiceOutcome DoPipelineWork() { bool includeConversionTypes = Boolean.Parse(this.Delivery.Parameters["includeConversionTypes"].ToString()); bool includeDisplaytData = Boolean.Parse(this.Delivery.Parameters["includeDisplaytData"].ToString()); bool ConvertToUSD = Boolean.Parse(this.Delivery.Parameters["ConvertToUSD"].ToString()); //double ConstCurrencyRate = this.Delivery.Parameters.ContainsKey("ConstCurrencyRate") ? Convert.ToDouble(this.Delivery.Parameters["ConstCurrencyRate"]) : 1; //Status Members Dictionary <string, ObjectStatus> kwd_Status_Data = new Dictionary <string, ObjectStatus>(); Dictionary <string, ObjectStatus> placement_kwd_Status_Data = new Dictionary <string, ObjectStatus>(); Dictionary <Int64, ObjectStatus> adGroup_Status_Data = new Dictionary <Int64, ObjectStatus>(); Dictionary <Int64, ObjectStatus> ad_Status_Data = new Dictionary <Int64, ObjectStatus>(); Dictionary <Int64, ObjectStatus> campaign_Status_Data = new Dictionary <Int64, ObjectStatus>(); using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { string[] requiredHeaders = new string[1]; requiredHeaders[0] = Const.AdPreRequiredHeader; #region Getting Keywords Data Dictionary <string, double> _totals = new Dictionary <string, double>(); DeliveryFile _keyWordsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]]; requiredHeaders[0] = Const.AdPreRequiredHeader; var _keywordsReader = new CsvDynamicReader(_keyWordsFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); _keywordsReader.MatchExactColumns = false; Dictionary <string, KeywordTarget> _keywordsData = new Dictionary <string, KeywordTarget>(); this.ImportManager.BeginImport(this.Delivery); using (_keywordsReader) { while (_keywordsReader.Read()) { this.Mappings.OnFieldRequired = field => _keywordsReader.Current[field]; if (_keywordsReader.Current[Const.KeywordIdFieldName] == Const.EOF) { break; } KeywordPrimaryKey keywordPrimaryKey = new KeywordPrimaryKey() { KeywordId = Convert.ToInt64(_keywordsReader.Current[Const.KeywordIdFieldName]), AdgroupId = Convert.ToInt64(_keywordsReader.Current[Const.AdGroupIdFieldName]), CampaignId = Convert.ToInt64(_keywordsReader.Current[Const.CampaignIdFieldName]) }; KeywordTarget keyword = new KeywordTarget() { OriginalID = _keywordsReader.Current[Const.KeywordIdFieldName], Keyword = _keywordsReader.Current[Const.KeywordFieldName], //Status = kwd_Status_Data[keywordPrimaryKey.ToString()] }; keyword.QualityScore = Convert.ToString(_keywordsReader.Current[Const.QualityScoreFieldName]); string matchType = _keywordsReader.Current[Const.MatchTypeFieldName]; keyword.MatchType = (KeywordMatchType)Enum.Parse(typeof(KeywordMatchType), matchType, true); //Setting Tracker for Keyword if (!String.IsNullOrWhiteSpace(Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]))) { keyword.DestinationUrl = Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]); //setting kwd tracker //if (((String)(_keywordsReader.Current[Const.DestUrlFieldName])).IndexOf(Const.CreativeIDTrackingValue, StringComparison.OrdinalIgnoreCase) >= 0) if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"])) { if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(KeywordTarget))) { this.Mappings.Objects[typeof(KeywordTarget)].Apply(keyword); } } } _keywordsData.Add(keywordPrimaryKey.ToString(), keyword); } } #endregion Dictionary <string, PlacementTarget> _placementsData = new Dictionary <string, PlacementTarget>(); #region Getting Placements Data string[] _placementsFileRequiredHeaders = new string[1]; _placementsFileRequiredHeaders[0] = Const.PlacementCriteriaID; DeliveryFile _PlacementsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]]; var _PlacementsReader = new CsvDynamicReader(_PlacementsFile.OpenContents(compression: FileCompression.Gzip), _placementsFileRequiredHeaders); using (_PlacementsReader) { while (_PlacementsReader.Read()) { if (_PlacementsReader.Current[Const.PlacementCriteriaID] == Const.EOF) { break; } //Read data only if managed GDN, otherwise it is an automatic GDN so skip if (!((String)(_PlacementsReader.Current[Const.PlacementCriteriaID])).Trim().Equals("--")) { KeywordPrimaryKey placementPrimaryKey = new KeywordPrimaryKey() { KeywordId = Convert.ToInt64(_PlacementsReader.Current[Const.PlacementCriteriaID]), AdgroupId = Convert.ToInt64(_PlacementsReader.Current[Const.AdGroupIdFieldName]), CampaignId = Convert.ToInt64(_PlacementsReader.Current[Const.CampaignIdFieldName]) }; PlacementTarget placement = new PlacementTarget() { OriginalID = _PlacementsReader.Current[Const.PlacementCriteriaID], Placement = _PlacementsReader.Current[Const.PlacementFieldName], PlacementType = PlacementType.Managed, // Status = placement_kwd_Status_Data[placementPrimaryKey.ToString()] }; //Setting Tracker for placment if (!String.IsNullOrWhiteSpace(Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]))) { placement.DestinationUrl = Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]); } _placementsData.Add(placementPrimaryKey.ToString(), placement); } } } #endregion #region Getting Conversions Data //Get Ads Conversion ( for ex. signup , purchase ) DeliveryFile _conversionFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"]; var _conversionsReader = new CsvDynamicReader(_conversionFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); Dictionary <string, Dictionary <string, long> > importedAdsWithConv = new Dictionary <string, Dictionary <string, long> >(); using (_conversionsReader) { while (_conversionsReader.Read()) { if (_conversionsReader.Current[Const.AdIDFieldName] == Const.EOF) // if end of report { break; } string conversionKey = String.Format("{0}#{1}", _conversionsReader.Current[Const.AdIDFieldName], _conversionsReader.Current[Const.KeywordIdFieldName]); Dictionary <string, long> conversionDic = new Dictionary <string, long>(); if (!importedAdsWithConv.TryGetValue(conversionKey, out conversionDic)) { //ADD conversionKey to importedAdsWithConv //than add conversion field to importedAdsWithConv : <conversion name , conversion value> Dictionary <string, long> conversion = new Dictionary <string, long>(); conversion.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName])); importedAdsWithConv.Add(conversionKey, conversion); } else // if Key exists { // if current add already has current conversion type than add value to the current type if (!conversionDic.ContainsKey(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]))) { conversionDic.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName])); } // else create new conversion type and add the value else { conversionDic[Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])] += Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]); } } } } #endregion #region Getting Ads Data DeliveryFile _adPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]]; var _adsReader = new CsvDynamicReader(_adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); Dictionary <string, Ad> importedAds = new Dictionary <string, Ad>(); //session.Begin(false); //this.ImportManager.BeginImport(this.Delivery); DeliveryOutput currentOutput = Delivery.Outputs.First(); foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { _totals.Add(measure.Key, 0); } } using (_adsReader) { this.Mappings.OnFieldRequired = field => _adsReader.Current[field]; while (_adsReader.Read()) { string currencyCode = ((string)(_adsReader.Current.Currency)).ToUpper(); // Adding totals line for validation (checksum) if (_adsReader.Current[Const.AdIDFieldName] == Const.EOF) { foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { continue; } switch (measure.Key) { case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_adsReader.Current.Clicks); break; case Measure.Common.Cost: _totals[Measure.Common.Cost] = ConvertToUSD ? this.ConvertToUSD(this.Delivery.Parameters["CurrencyCode"].ToString().ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000; break; case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_adsReader.Current.Impressions); break; } } break; } AdMetricsUnit adMetricsUnit = new AdMetricsUnit(); adMetricsUnit.Output = currentOutput; Ad ad; #region Try Get SearchKWD /***********************************************************************************/ //Creating kwd primary key KeywordPrimaryKey kwdKey = new KeywordPrimaryKey() { AdgroupId = Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName]), KeywordId = Convert.ToInt64(_adsReader.Current[Const.KeywordIdFieldName]), CampaignId = Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName]) }; //Check if keyword file contains this kwdkey and not a GDN Keyword String[] GdnKwdIds = this.Delivery.Parameters["KeywordContentId"].ToString().Split(','); bool IsSearchKwd = false; KeywordTarget kwd = null; PlacementTarget placement = null; if (!GdnKwdIds.Contains(kwdKey.KeywordId.ToString()) && _keywordsData.ContainsKey(kwdKey.ToString())) { kwd = new KeywordTarget(); try { kwd = _keywordsData[kwdKey.ToString()]; } catch (Exception) { //Creating KWD with OriginalID , since the KWD doesnt exists in KWD report. kwd = new KeywordTarget { OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]) }; } IsSearchKwd = true; } else { placement = new PlacementTarget(); try { placement = _placementsData[kwdKey.ToString()]; } catch (Exception) { placement.OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]); placement.PlacementType = PlacementType.Automatic; placement.Placement = Const.AutoDisplayNetworkName; } } /***********************************************************************************/ #endregion string adId = _adsReader.Current[Const.AdIDFieldName]; if (!importedAds.ContainsKey(adId)) { ad = new Ad(); ad.OriginalID = adId; ad.Channel = new Channel() { ID = 1 }; ad.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"] }; // ad.Status = ad_Status_Data[Convert.ToInt64(adId)]; #region Ad Type /****************************************************************/ string adTypeColumnValue = Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]); string devicePreferenceColumnValue = Convert.ToString(_adsReader.Current[Const.AdDevicePreferenceFieldName]); if (!GoogleAdTypeDic.ContainsKey(adTypeColumnValue)) { continue; } string adTypeEdgeValue = GoogleAdTypeDic[adTypeColumnValue].ToString(); //EdgeAdType atv = (EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true); //is mobile ad ? if (devicePreferenceColumnValue.Equals(Const.AdDevicePreferenceMobileFieldValue)) { string mobileValue = string.Format("Mobile {0}", Convert.ToString(_adsReader.Current[Const.AdTypeFieldName])); //Check if this mobile value exists on dictionary if (GoogleAdTypeDic.ContainsKey(mobileValue)) { adTypeEdgeValue = GoogleAdTypeDic[mobileValue].ToString(); } else { adTypeEdgeValue = GoogleAdTypeDic[Const.AdTypeValues.Mobile_ad].ToString(); } } ad.ExtraFields[AdType] = (int)(EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);; /****************************************************************/ #endregion //Creative ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.DisplayUrl, Text = _adsReader.Current[Const.DisplayURLFieldName] }); #region Ad Tracker segment /******************************************************/ ////Setting Tracker for Ad if (!String.IsNullOrWhiteSpace(_adsReader.Current[Const.DestUrlFieldName])) { ad.DestinationUrl = _adsReader.Current[Const.DestUrlFieldName]; if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } } //if Ad doesnt contains tracker than check for kwd tracker if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"])) { if (kwd != null && kwd.Segments != null && kwd.Segments.ContainsKey(this.ImportManager.SegmentTypes[Segment.Common.Tracker])) { if (kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] != null && kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value != null) { SegmentObject tracker = kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]]; //if value contains ADID than replace ADID with AD original id tracker.Value = tracker.Value.Replace("ADID", ad.OriginalID); ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] = tracker; } } } /******************************************************/ #endregion #region Campaign /****************************************************************/ ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign() { OriginalID = _adsReader.Current[Const.CampaignIdFieldName], Name = _adsReader.Current[Const.CampaignFieldName], //Status = campaign_Status_Data[Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])] }; /****************************************************************/ #endregion #region Image /****************************************************************/ //Image Type > Create Image if (String.Equals(Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]), "Image ad")) { string adNameField = _adsReader.Current[Const.AdFieldName]; string[] imageParams = adNameField.Trim().Split(new Char[] { ':', ';' }); // Ad name: 468_60_Test7options_Romanian.swf; 468 x 60 ad.Name = imageParams[1].Trim(); ad.Creatives.Add(new ImageCreative() { ImageUrl = imageParams[1].Trim(), ImageSize = imageParams[2].Trim() }); } /****************************************************************/ #endregion #region Text od Display /****************************************************************/ else //Text ad or Display ad { ad.Name = _adsReader.Current[Const.AdFieldName]; ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.Title, Text = _adsReader.Current.Ad, }); ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.Body, Text = _adsReader.Current["Description line 1"], Text2 = _adsReader.Current["Description line 2"] }); } /****************************************************************/ #endregion #region Adgroup /****************************************************************/ //Insert Adgroup ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup() { Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], Value = _adsReader.Current[Const.AdGroupFieldName], OriginalID = _adsReader.Current[Const.AdGroupIdFieldName], // Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])] }; /****************************************************************/ #endregion #region Network /****************************************************************/ //Insert Network Type Display Network / Search Network //string networkType = Convert.ToString(_adsReader.Current[Const.NetworkFieldName]); //if (networkType.Equals(Const.GoogleSearchNetwork)) // networkType = Const.SystemSearchNetwork; //else if (networkType.Equals(Const.GoogleDisplayNetwork)) // networkType = Const.SystemDisplayNetwork; //ad.ExtraFields[NetworkType] = networkType; /****************************************************************/ #endregion importedAds.Add(adId, ad); this.ImportManager.ImportAd(ad); } else { ad = importedAds[adId]; } adMetricsUnit.Ad = ad; //INSERTING SEARCH KEYWORD INTO METRICS if (IsSearchKwd & kwd != null) { adMetricsUnit.TargetDimensions = new List <Target>(); adMetricsUnit.TargetDimensions.Add(kwd); } //INSERTING GDN KEYWORD INTO METRICS else if (placement != null) { //INSERTING KEYWORD INTO METRICS adMetricsUnit.TargetDimensions = new List <Target>(); adMetricsUnit.TargetDimensions.Add(placement); } //INSERTING METRICS DATA adMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_adsReader.Current.Clicks)); //Currencies Conversion Support adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], ConvertToUSD ? this.ConvertToUSD(currencyCode.ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000); if (ConvertToUSD) { adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.CostBeforeConversion], (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.USDConversionRate], Convert.ToDouble(this.ImportManager.CurrencyRates[currencyCode].RateValue)); } adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_adsReader.Current.Impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], Convert.ToDouble(_adsReader.Current[Const.AvgPositionFieldName])); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionOnePerClickFieldName])); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionManyPerClickFieldName])); //Inserting conversion values string conversionKey = String.Format("{0}#{1}", ad.OriginalID, _adsReader.Current[Const.KeywordIdFieldName]); Dictionary <string, long> conversionDic = new Dictionary <string, long>(); if (importedAdsWithConv.TryGetValue(conversionKey, out conversionDic)) { foreach (var pair in conversionDic) { if (GoogleMeasuresDic.ContainsKey(pair.Key)) { adMetricsUnit.MeasureValues[this.ImportManager.Measures[GoogleMeasuresDic[pair.Key]]] = pair.Value; } } } adMetricsUnit.Currency = new Currency { Code = Convert.ToString(_adsReader.Current.Currency) }; this.ImportManager.ImportMetrics(adMetricsUnit); } #endregion } #region Getting Sitelinks Data without metrics if (this.Delivery.Parameters.ContainsKey("AppendSitelinks") && Boolean.Parse(this.Delivery.Parameters["AppendSitelinks"].ToString())) { string[] sitelinksRequiredHeaders = new string[1]; sitelinksRequiredHeaders[0] = Const.PlaceholderFeedItemID; DeliveryFile _sitelinkPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEHOLDER_FEED_ITEM_REPORT]]; var _sitelinkReader = new CsvDynamicReader(_sitelinkPerformanceFile.OpenContents(compression: FileCompression.Gzip), sitelinksRequiredHeaders); AdMetricsUnit siteLinkMetricsUnit = new AdMetricsUnit(); siteLinkMetricsUnit.Output = currentOutput; Ad sitelinkAd; using (_sitelinkReader) { this.Mappings.OnFieldRequired = field => _sitelinkReader.Current[field]; //to do : get site link tracker string[] sitelinkAttr; while (_sitelinkReader.Read()) { if (((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Equals(Const.Sitelink_EOF)) { break; } string sitelinkId = string.Format("{0}{1}{2}", _sitelinkReader.Current[Const.PlaceholderFeedItemID], _sitelinkReader.Current[Const.CampaignIdFieldName], _sitelinkReader.Current[Const.AdGroupIdFieldName]); sitelinkAd = new Ad(); sitelinkAd.OriginalID = sitelinkId; sitelinkAd.Channel = new Channel() { ID = 1 }; sitelinkAd.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"] }; //Creative sitelinkAttr = ((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Split(';'); bool legacy = sitelinkAttr.Count() != 4 ? true : false; string destUrl = string.Empty; bool destUrlParsingError = false; //Checking desturl errors ( we would like to insert only sitelinks that contains desturl ). try { destUrl = sitelinkAttr[1]; } catch (Exception e) { Log.Write("Error while trying to pars destination url from attribute field on sitelink report", e); destUrlParsingError = true; } if (!destUrlParsingError) { ////Setting Tracker for Ad if (!String.IsNullOrWhiteSpace(sitelinkAttr[1])) { sitelinkAd.DestinationUrl = sitelinkAttr[1]; if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(sitelinkAd); } } sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign() { OriginalID = _sitelinkReader.Current[Const.CampaignIdFieldName], Name = _sitelinkReader.Current[Const.CampaignFieldName], }; //Insert Adgroup sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup() { Campaign = (Campaign)sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], Value = _sitelinkReader.Current[Const.AdGroupFieldName], OriginalID = _sitelinkReader.Current[Const.AdGroupIdFieldName], // Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])] }; sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.DisplayUrl, Text = sitelinkAttr[1] }); if (!legacy) // only in case it doesnt contains legacy siteink { sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.Body, Text = sitelinkAttr[2], Text2 = sitelinkAttr[3] }); } sitelinkAd.Name = "[Sitelink] " + sitelinkAttr[0]; sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.Title, Text = "[Sitelink] " + sitelinkAttr[0] }); //Ad Type //Note: changed to "sitelink" following Amir request sitelinkAd.ExtraFields[AdType] = (int)(EdgeAdType.Sitelink); siteLinkMetricsUnit.Ad = sitelinkAd; //Setting Default Currency as USD following Amir's request from March 2014 siteLinkMetricsUnit.Currency = new Currency { Code = "USD" }; //INSERTING METRICS DATA siteLinkMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], 0); ImportManager.ImportMetrics(siteLinkMetricsUnit); ImportManager.ImportAd(sitelinkAd); } } } }// end if #endregion currentOutput.Checksum = _totals; this.ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { string[] requiredHeaders = new string[] { Const.AutoPlacRequiredHeader }; DeliveryOutput currentOutput = Delivery.Outputs.First(); //Open Auto Plac file DeliveryFile _autoPlacFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AUTOMATIC_PLACEMENTS_PERFORMANCE_REPORT]]; var _autoPlacReader = new CsvDynamicReader(_autoPlacFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); using (this.ImportManager = new GenericMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { Dictionary <string, double> _totals = new Dictionary <string, double>(); this.ImportManager.BeginImport(this.Delivery); //Intializing totals for validation foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { _totals.Add(measure.Key, 0); } } using (_autoPlacReader) { this.Mappings.OnFieldRequired = field => _autoPlacReader.Current[field]; while (_autoPlacReader.Read()) { #region Setting Totals /*==================================================================================================================*/ // If end of file if (_autoPlacReader.Current[Const.CampaignIdFieldName] == Const.EOF) { //Setting totals for validation from totals line in adowrds file foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { continue; } switch (measure.Key) { case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_autoPlacReader.Current.Clicks); break; case Measure.Common.Cost: _totals[Measure.Common.Cost] = (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000; break; case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_autoPlacReader.Current.Impressions); break; } } break; } /*==================================================================================================================*/ #endregion GenericMetricsUnit autoPlacMetricsUnit = new GenericMetricsUnit(); autoPlacMetricsUnit.Output = currentOutput; autoPlacMetricsUnit.Channel = new Channel() { ID = 1 }; autoPlacMetricsUnit.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_autoPlacFile.Parameters["AdwordsClientID"] }; autoPlacMetricsUnit.SegmentDimensions = new Dictionary <Segment, SegmentObject>(); //ADDING CAMPAIGN Campaign campaign = new Campaign() { OriginalID = _autoPlacReader.Current[Const.CampaignIdFieldName], Name = _autoPlacReader.Current[Const.CampaignFieldName], Status = ObjectStatusDic[((string)_autoPlacReader.Current[Const.CampaignStatusFieldName]).ToUpper()] }; autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.Campaign], campaign); //ADDING ADGROUP AdGroup adgroup = new AdGroup() { Campaign = campaign, Value = _autoPlacReader.Current[Const.AdGroupFieldName], OriginalID = _autoPlacReader.Current[Const.AdGroupIdFieldName] }; autoPlacMetricsUnit.SegmentDimensions.Add(this.ImportManager.SegmentTypes[Segment.Common.AdGroup], adgroup); //INSERTING METRICS DATA autoPlacMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_autoPlacReader.Current.Clicks)); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], (Convert.ToDouble(_autoPlacReader.Current.Cost)) / 1000000); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_autoPlacReader.Current.Impressions)); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionOnePerClickFieldName])); autoPlacMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_autoPlacReader.Current[Const.ConversionManyPerClickFieldName])); //CREATING PLACEMENT autoPlacMetricsUnit.TargetDimensions = new List <Target>(); PlacementTarget placement = new PlacementTarget() { Placement = _autoPlacReader.Current[Const.DomainFieldName], PlacementType = PlacementType.Automatic // Add status !!! }; autoPlacMetricsUnit.TargetDimensions.Add(placement); //SETTING TIME PERIOD autoPlacMetricsUnit.TimePeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime(); autoPlacMetricsUnit.TimePeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime(); this.ImportManager.ImportMetrics(autoPlacMetricsUnit); } } this.Delivery.Outputs.First().Checksum = _totals; this.ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { Log("Starting Google.AdWords.ProcessorService", LogMessageType.Debug); InitMappings(); Mappings.OnMappingApplied = SetEdgeType; if (!Mappings.Objects.TryGetValue(typeof(KeywordTarget), out KeywordMappings)) { throw new MappingConfigurationException("Missing mapping definition for KeywordTarget.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(PlacementTarget), out PlacementMappings)) { throw new MappingConfigurationException("Missing mapping definition for PlacementTarget.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings)) { throw new MappingConfigurationException("Missing mapping definition for MetricsUnit.", "Object"); } if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings)) { throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object"); } using (ImportManager = new MetricsDeliveryManager(InstanceID, EdgeTypes, new MetricsDeliveryManagerOptions())) //{ // MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, // MeasureOptionsOperator = OptionsOperator.Not, // SegmentOptions = Data.Objects.SegmentOptions.All, // SegmentOptionsOperator = OptionsOperator.And //})) { ImportManager.BeginImport(Delivery, GetSampleMetrics()); Log("Objects and Metrics tables are created", LogMessageType.Debug); Progress = 0.1; var requiredHeaders = new[] { AdWordsConst.AdPreRequiredHeader }; // Getting Keywords Data Log("Start loading keywords", LogMessageType.Debug); LoadKeywords(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]], requiredHeaders); Log("Finished loading keywords", LogMessageType.Debug); Progress = 0.3; // Getting Placements Data Log("Start loading placements", LogMessageType.Debug); LoadPlacements(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]], requiredHeaders); Log("Finished loading placements", LogMessageType.Debug); // Getting Conversions Data ( for ex. signup , purchase ) Log("Start loading conversions", LogMessageType.Debug); LoadConversions(Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"], requiredHeaders); Log("Finished loading conversions", LogMessageType.Debug); Progress = 0.4; #region Getting Ads Data and Import Metrics Log("Start loading Ads", LogMessageType.Debug); var adPerformanceFile = Delivery.Files[GoogleStaticReportsNamesUtill.ReportNames[global::Google.Api.Ads.AdWords.v201302.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]]; _adsReader = new CsvDynamicReader(adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); Mappings.OnFieldRequired = field => _adsReader.Current[field]; _importedAds.Clear(); using (_adsReader) { Mappings.OnFieldRequired = fieldName => _adsReader.Current[fieldName]; while (_adsReader.Read() && _adsReader.Current[AdWordsConst.AdIDFieldName] != AdWordsConst.EOF) { ProcessMetrics(); // add Ad if it is not exists yet if (!_importedAds.ContainsKey(CurrentMetricsUnit.Ad.OriginalID)) { _importedAds.Add(CurrentMetricsUnit.Ad.OriginalID, CurrentMetricsUnit.Ad); } } //Progress = 0.8; //Log("Start importing objects", LogMessageType.Debug); ImportManager.EndImport(); //Log("Finished importing objects", LogMessageType.Debug); } #endregion } return(ServiceOutcome.Success); }
protected override ServiceOutcome DoPipelineWork() { currentOutput = Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); // TODO: add checks for delivery state string[] requiredHeaders; requiredHeaders = new string[1]; requiredHeaders[0] = WS.CampaignPerformanceReportColumn.CampaignName.ToString(); _campaignsCache = new Dictionary <string, Campaign>(); _adCache = new Dictionary <long, Ad>(); DeliveryFile adReport = this.Delivery.Files[Const.Files.AdReport]; DeliveryFile campaignReport = this.Delivery.Files[Const.Files.CampaignReport]; FileInfo campaignReportFileInfo = campaignReport.GetFileInfo(ArchiveType.Zip); string[] campaignReportSubFiles = campaignReportFileInfo.GetSubFiles(); var campaignReportReader = new CsvDynamicReader( campaignReport.OpenContents(subLocation: campaignReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders ); #region Reading campaigns file while (campaignReportReader.Read()) { string accountNameColVal = campaignReportReader.Current[WS.CampaignPerformanceReportColumn.AccountName.ToString()]; if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim().Contains(endOfFileMicrosoftCorporation.Trim())) //end of file { break; } Campaign campaign = CreateCampaign(campaignReportReader.Current); _campaignsCache.Add(campaign.Name, campaign); } this.ReportProgress(0.2); #endregion #region Read the ad report, and build a lookup table for later using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { ImportManager.BeginImport(this.Delivery); // create the ad report reader requiredHeaders = new string[1]; requiredHeaders[0] = WS.AdPerformanceReportColumn.AdId.ToString(); string[] adReportSubFiles = adReport.GetFileInfo(ArchiveType.Zip).GetSubFiles(); var adReportReader = new CsvDynamicReader( adReport.OpenContents(subLocation: adReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders); //read using (adReportReader) { while (adReportReader.Read()) { // create the ad this.Mappings.OnFieldRequired = field => adReportReader.Current[field]; string accountNameColVal = adReportReader.Current[WS.AdPerformanceReportColumn.AccountName.ToString()]; //end of file if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) { break; } Ad ad = CreateAd(adReportReader.Current, ImportManager); //TRACKER if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } //ADDING CAMPAIGN ad.Segments[this.ImportManager. SegmentTypes[Segment.Common.Campaign]] = _campaignsCache[adReportReader.Current[WS.AdPerformanceReportColumn.CampaignName.ToString()]]; //CREATING Adgroup AdGroup adGroup = new AdGroup() { Value = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupName.ToString()], Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], OriginalID = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupId.ToString()] }; ad.Segments.Add(ImportManager.SegmentTypes[Segment.Common.AdGroup], adGroup); //CRAETING AD TYPE string adTypeKey = Convert.ToString(adReportReader.Current[WS.AdPerformanceReportColumn.AdType.ToString()]); ad.ExtraFields[AdType] = AdCenterAdTypeDic[adTypeKey]; ImportManager.ImportAd(ad); _adCache.Add(long.Parse(ad.OriginalID), ad); } } this.ReportProgress(0.7); #endregion // ............................................................... #region Read the keyword report, cross reference it with the ad data, and commit // Read the keyword report, cross reference it with the ad data, and commit // The name of the time period column is specified by the initializer, depending on the report DeliveryFile keywordReport = this.Delivery.Files[Const.Files.KeywordReport]; string timePeriodColumn = keywordReport.Parameters[Const.Parameters.TimePeriodColumnName] as string; // // create the keyword report reader requiredHeaders = new string[1]; requiredHeaders[0] = "Keyword"; string[] keywordReportSubFiles = keywordReport.GetFileInfo(ArchiveType.Zip).GetSubFiles(); var keywordReportReader = new CsvDynamicReader( keywordReport.OpenContents(subLocation: keywordReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders); //Added by Shay for validation foreach (KeyValuePair <string, Measure> measure in ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum.Add(measure.Key, 0); } } // read and save in transaction using (keywordReportReader) { while (keywordReportReader.Read()) { string GregorianDateColVal = keywordReportReader.Current.GregorianDate; if (GregorianDateColVal.Trim() == string.Empty || GregorianDateColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) //end of file { break; } // get the unit from the keyword report, and add the missing ad data AdMetricsUnit unit = CreateMetrics(keywordReportReader.Current, timePeriodColumn, ImportManager); ImportManager.ImportMetrics(unit); // Validation information: currentOutput.Checksum[Measure.Common.Clicks] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Clicks]]; currentOutput.Checksum[Measure.Common.Cost] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Cost]]; currentOutput.Checksum[Measure.Common.Impressions] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Impressions]]; } ImportManager.EndImport(); ReportProgress(1); } #endregion } return(ServiceOutcome.Success); }