protected override ServiceOutcome DoPipelineWork() { currentOutput = Delivery.Outputs.First(); currentOutput.Checksum = new Dictionary <string, double>(); // TODO: add checks for delivery state string[] requiredHeaders; requiredHeaders = new string[1]; requiredHeaders[0] = WS.CampaignPerformanceReportColumn.CampaignName.ToString(); _campaignsCache = new Dictionary <string, Campaign>(); _adCache = new Dictionary <long, Ad>(); DeliveryFile adReport = this.Delivery.Files[Const.Files.AdReport]; DeliveryFile campaignReport = this.Delivery.Files[Const.Files.CampaignReport]; FileInfo campaignReportFileInfo = campaignReport.GetFileInfo(ArchiveType.Zip); string[] campaignReportSubFiles = campaignReportFileInfo.GetSubFiles(); var campaignReportReader = new CsvDynamicReader( campaignReport.OpenContents(subLocation: campaignReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders ); #region Reading campaigns file while (campaignReportReader.Read()) { string accountNameColVal = campaignReportReader.Current[WS.CampaignPerformanceReportColumn.AccountName.ToString()]; if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim().Contains(endOfFileMicrosoftCorporation.Trim())) //end of file { break; } Campaign campaign = CreateCampaign(campaignReportReader.Current); _campaignsCache.Add(campaign.Name, campaign); } this.ReportProgress(0.2); #endregion #region Read the ad report, and build a lookup table for later using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { ImportManager.BeginImport(this.Delivery); // create the ad report reader requiredHeaders = new string[1]; requiredHeaders[0] = WS.AdPerformanceReportColumn.AdId.ToString(); string[] adReportSubFiles = adReport.GetFileInfo(ArchiveType.Zip).GetSubFiles(); var adReportReader = new CsvDynamicReader( adReport.OpenContents(subLocation: adReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders); //read using (adReportReader) { while (adReportReader.Read()) { // create the ad this.Mappings.OnFieldRequired = field => adReportReader.Current[field]; string accountNameColVal = adReportReader.Current[WS.AdPerformanceReportColumn.AccountName.ToString()]; //end of file if (accountNameColVal.Trim() == string.Empty || accountNameColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) { break; } Ad ad = CreateAd(adReportReader.Current, ImportManager); //TRACKER if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } //ADDING CAMPAIGN ad.Segments[this.ImportManager. SegmentTypes[Segment.Common.Campaign]] = _campaignsCache[adReportReader.Current[WS.AdPerformanceReportColumn.CampaignName.ToString()]]; //CREATING Adgroup AdGroup adGroup = new AdGroup() { Value = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupName.ToString()], Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], OriginalID = adReportReader.Current[WS.AdPerformanceReportColumn.AdGroupId.ToString()] }; ad.Segments.Add(ImportManager.SegmentTypes[Segment.Common.AdGroup], adGroup); //CRAETING AD TYPE string adTypeKey = Convert.ToString(adReportReader.Current[WS.AdPerformanceReportColumn.AdType.ToString()]); ad.ExtraFields[AdType] = AdCenterAdTypeDic[adTypeKey]; ImportManager.ImportAd(ad); _adCache.Add(long.Parse(ad.OriginalID), ad); } } this.ReportProgress(0.7); #endregion // ............................................................... #region Read the keyword report, cross reference it with the ad data, and commit // Read the keyword report, cross reference it with the ad data, and commit // The name of the time period column is specified by the initializer, depending on the report DeliveryFile keywordReport = this.Delivery.Files[Const.Files.KeywordReport]; string timePeriodColumn = keywordReport.Parameters[Const.Parameters.TimePeriodColumnName] as string; // // create the keyword report reader requiredHeaders = new string[1]; requiredHeaders[0] = "Keyword"; string[] keywordReportSubFiles = keywordReport.GetFileInfo(ArchiveType.Zip).GetSubFiles(); var keywordReportReader = new CsvDynamicReader( keywordReport.OpenContents(subLocation: keywordReportSubFiles[0], archiveType: ArchiveType.Zip), requiredHeaders); //Added by Shay for validation foreach (KeyValuePair <string, Measure> measure in ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { currentOutput.Checksum.Add(measure.Key, 0); } } // read and save in transaction using (keywordReportReader) { while (keywordReportReader.Read()) { string GregorianDateColVal = keywordReportReader.Current.GregorianDate; if (GregorianDateColVal.Trim() == string.Empty || GregorianDateColVal.Trim() == endOfFileMicrosoftCorporation.Trim()) //end of file { break; } // get the unit from the keyword report, and add the missing ad data AdMetricsUnit unit = CreateMetrics(keywordReportReader.Current, timePeriodColumn, ImportManager); ImportManager.ImportMetrics(unit); // Validation information: currentOutput.Checksum[Measure.Common.Clicks] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Clicks]]; currentOutput.Checksum[Measure.Common.Cost] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Cost]]; currentOutput.Checksum[Measure.Common.Impressions] += unit.MeasureValues[ImportManager.Measures[Measure.Common.Impressions]]; } ImportManager.EndImport(); ReportProgress(1); } #endregion } return(ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { bool includeConversionTypes = Boolean.Parse(this.Delivery.Parameters["includeConversionTypes"].ToString()); bool includeDisplaytData = Boolean.Parse(this.Delivery.Parameters["includeDisplaytData"].ToString()); bool ConvertToUSD = Boolean.Parse(this.Delivery.Parameters["ConvertToUSD"].ToString()); //double ConstCurrencyRate = this.Delivery.Parameters.ContainsKey("ConstCurrencyRate") ? Convert.ToDouble(this.Delivery.Parameters["ConstCurrencyRate"]) : 1; //Status Members Dictionary <string, ObjectStatus> kwd_Status_Data = new Dictionary <string, ObjectStatus>(); Dictionary <string, ObjectStatus> placement_kwd_Status_Data = new Dictionary <string, ObjectStatus>(); Dictionary <Int64, ObjectStatus> adGroup_Status_Data = new Dictionary <Int64, ObjectStatus>(); Dictionary <Int64, ObjectStatus> ad_Status_Data = new Dictionary <Int64, ObjectStatus>(); Dictionary <Int64, ObjectStatus> campaign_Status_Data = new Dictionary <Int64, ObjectStatus>(); using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions() { MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice, MeasureOptionsOperator = OptionsOperator.Not, SegmentOptions = Data.Objects.SegmentOptions.All, SegmentOptionsOperator = OptionsOperator.And })) { string[] requiredHeaders = new string[1]; requiredHeaders[0] = Const.AdPreRequiredHeader; #region Getting Keywords Data Dictionary <string, double> _totals = new Dictionary <string, double>(); DeliveryFile _keyWordsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.KEYWORDS_PERFORMANCE_REPORT]]; requiredHeaders[0] = Const.AdPreRequiredHeader; var _keywordsReader = new CsvDynamicReader(_keyWordsFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); _keywordsReader.MatchExactColumns = false; Dictionary <string, KeywordTarget> _keywordsData = new Dictionary <string, KeywordTarget>(); this.ImportManager.BeginImport(this.Delivery); using (_keywordsReader) { while (_keywordsReader.Read()) { this.Mappings.OnFieldRequired = field => _keywordsReader.Current[field]; if (_keywordsReader.Current[Const.KeywordIdFieldName] == Const.EOF) { break; } KeywordPrimaryKey keywordPrimaryKey = new KeywordPrimaryKey() { KeywordId = Convert.ToInt64(_keywordsReader.Current[Const.KeywordIdFieldName]), AdgroupId = Convert.ToInt64(_keywordsReader.Current[Const.AdGroupIdFieldName]), CampaignId = Convert.ToInt64(_keywordsReader.Current[Const.CampaignIdFieldName]) }; KeywordTarget keyword = new KeywordTarget() { OriginalID = _keywordsReader.Current[Const.KeywordIdFieldName], Keyword = _keywordsReader.Current[Const.KeywordFieldName], //Status = kwd_Status_Data[keywordPrimaryKey.ToString()] }; keyword.QualityScore = Convert.ToString(_keywordsReader.Current[Const.QualityScoreFieldName]); string matchType = _keywordsReader.Current[Const.MatchTypeFieldName]; keyword.MatchType = (KeywordMatchType)Enum.Parse(typeof(KeywordMatchType), matchType, true); //Setting Tracker for Keyword if (!String.IsNullOrWhiteSpace(Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]))) { keyword.DestinationUrl = Convert.ToString(_keywordsReader.Current[Const.DestUrlFieldName]); //setting kwd tracker //if (((String)(_keywordsReader.Current[Const.DestUrlFieldName])).IndexOf(Const.CreativeIDTrackingValue, StringComparison.OrdinalIgnoreCase) >= 0) if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"])) { if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(KeywordTarget))) { this.Mappings.Objects[typeof(KeywordTarget)].Apply(keyword); } } } _keywordsData.Add(keywordPrimaryKey.ToString(), keyword); } } #endregion Dictionary <string, PlacementTarget> _placementsData = new Dictionary <string, PlacementTarget>(); #region Getting Placements Data string[] _placementsFileRequiredHeaders = new string[1]; _placementsFileRequiredHeaders[0] = Const.PlacementCriteriaID; DeliveryFile _PlacementsFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEMENT_PERFORMANCE_REPORT]]; var _PlacementsReader = new CsvDynamicReader(_PlacementsFile.OpenContents(compression: FileCompression.Gzip), _placementsFileRequiredHeaders); using (_PlacementsReader) { while (_PlacementsReader.Read()) { if (_PlacementsReader.Current[Const.PlacementCriteriaID] == Const.EOF) { break; } //Read data only if managed GDN, otherwise it is an automatic GDN so skip if (!((String)(_PlacementsReader.Current[Const.PlacementCriteriaID])).Trim().Equals("--")) { KeywordPrimaryKey placementPrimaryKey = new KeywordPrimaryKey() { KeywordId = Convert.ToInt64(_PlacementsReader.Current[Const.PlacementCriteriaID]), AdgroupId = Convert.ToInt64(_PlacementsReader.Current[Const.AdGroupIdFieldName]), CampaignId = Convert.ToInt64(_PlacementsReader.Current[Const.CampaignIdFieldName]) }; PlacementTarget placement = new PlacementTarget() { OriginalID = _PlacementsReader.Current[Const.PlacementCriteriaID], Placement = _PlacementsReader.Current[Const.PlacementFieldName], PlacementType = PlacementType.Managed, // Status = placement_kwd_Status_Data[placementPrimaryKey.ToString()] }; //Setting Tracker for placment if (!String.IsNullOrWhiteSpace(Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]))) { placement.DestinationUrl = Convert.ToString(_PlacementsReader.Current[Const.DestUrlFieldName]); } _placementsData.Add(placementPrimaryKey.ToString(), placement); } } } #endregion #region Getting Conversions Data //Get Ads Conversion ( for ex. signup , purchase ) DeliveryFile _conversionFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT] + "_Conv"]; var _conversionsReader = new CsvDynamicReader(_conversionFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); Dictionary <string, Dictionary <string, long> > importedAdsWithConv = new Dictionary <string, Dictionary <string, long> >(); using (_conversionsReader) { while (_conversionsReader.Read()) { if (_conversionsReader.Current[Const.AdIDFieldName] == Const.EOF) // if end of report { break; } string conversionKey = String.Format("{0}#{1}", _conversionsReader.Current[Const.AdIDFieldName], _conversionsReader.Current[Const.KeywordIdFieldName]); Dictionary <string, long> conversionDic = new Dictionary <string, long>(); if (!importedAdsWithConv.TryGetValue(conversionKey, out conversionDic)) { //ADD conversionKey to importedAdsWithConv //than add conversion field to importedAdsWithConv : <conversion name , conversion value> Dictionary <string, long> conversion = new Dictionary <string, long>(); conversion.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName])); importedAdsWithConv.Add(conversionKey, conversion); } else // if Key exists { // if current add already has current conversion type than add value to the current type if (!conversionDic.ContainsKey(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]))) { conversionDic.Add(Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName]), Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName])); } // else create new conversion type and add the value else { conversionDic[Convert.ToString(_conversionsReader.Current[Const.ConversionTrackingPurposeFieldName])] += Convert.ToInt64(_conversionsReader.Current[Const.ConversionManyPerClickFieldName]); } } } } #endregion #region Getting Ads Data DeliveryFile _adPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.AD_PERFORMANCE_REPORT]]; var _adsReader = new CsvDynamicReader(_adPerformanceFile.OpenContents(compression: FileCompression.Gzip), requiredHeaders); Dictionary <string, Ad> importedAds = new Dictionary <string, Ad>(); //session.Begin(false); //this.ImportManager.BeginImport(this.Delivery); DeliveryOutput currentOutput = Delivery.Outputs.First(); foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { _totals.Add(measure.Key, 0); } } using (_adsReader) { this.Mappings.OnFieldRequired = field => _adsReader.Current[field]; while (_adsReader.Read()) { string currencyCode = ((string)(_adsReader.Current.Currency)).ToUpper(); // Adding totals line for validation (checksum) if (_adsReader.Current[Const.AdIDFieldName] == Const.EOF) { foreach (KeyValuePair <string, Measure> measure in this.ImportManager.Measures) { if (!measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { continue; } switch (measure.Key) { case Measure.Common.Clicks: _totals[Measure.Common.Clicks] = Convert.ToInt64(_adsReader.Current.Clicks); break; case Measure.Common.Cost: _totals[Measure.Common.Cost] = ConvertToUSD ? this.ConvertToUSD(this.Delivery.Parameters["CurrencyCode"].ToString().ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000; break; case Measure.Common.Impressions: _totals[Measure.Common.Impressions] = Convert.ToInt64(_adsReader.Current.Impressions); break; } } break; } AdMetricsUnit adMetricsUnit = new AdMetricsUnit(); adMetricsUnit.Output = currentOutput; Ad ad; #region Try Get SearchKWD /***********************************************************************************/ //Creating kwd primary key KeywordPrimaryKey kwdKey = new KeywordPrimaryKey() { AdgroupId = Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName]), KeywordId = Convert.ToInt64(_adsReader.Current[Const.KeywordIdFieldName]), CampaignId = Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName]) }; //Check if keyword file contains this kwdkey and not a GDN Keyword String[] GdnKwdIds = this.Delivery.Parameters["KeywordContentId"].ToString().Split(','); bool IsSearchKwd = false; KeywordTarget kwd = null; PlacementTarget placement = null; if (!GdnKwdIds.Contains(kwdKey.KeywordId.ToString()) && _keywordsData.ContainsKey(kwdKey.ToString())) { kwd = new KeywordTarget(); try { kwd = _keywordsData[kwdKey.ToString()]; } catch (Exception) { //Creating KWD with OriginalID , since the KWD doesnt exists in KWD report. kwd = new KeywordTarget { OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]) }; } IsSearchKwd = true; } else { placement = new PlacementTarget(); try { placement = _placementsData[kwdKey.ToString()]; } catch (Exception) { placement.OriginalID = Convert.ToString(_adsReader.Current[Const.KeywordIdFieldName]); placement.PlacementType = PlacementType.Automatic; placement.Placement = Const.AutoDisplayNetworkName; } } /***********************************************************************************/ #endregion string adId = _adsReader.Current[Const.AdIDFieldName]; if (!importedAds.ContainsKey(adId)) { ad = new Ad(); ad.OriginalID = adId; ad.Channel = new Channel() { ID = 1 }; ad.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"] }; // ad.Status = ad_Status_Data[Convert.ToInt64(adId)]; #region Ad Type /****************************************************************/ string adTypeColumnValue = Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]); string devicePreferenceColumnValue = Convert.ToString(_adsReader.Current[Const.AdDevicePreferenceFieldName]); if (!GoogleAdTypeDic.ContainsKey(adTypeColumnValue)) { continue; } string adTypeEdgeValue = GoogleAdTypeDic[adTypeColumnValue].ToString(); //EdgeAdType atv = (EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true); //is mobile ad ? if (devicePreferenceColumnValue.Equals(Const.AdDevicePreferenceMobileFieldValue)) { string mobileValue = string.Format("Mobile {0}", Convert.ToString(_adsReader.Current[Const.AdTypeFieldName])); //Check if this mobile value exists on dictionary if (GoogleAdTypeDic.ContainsKey(mobileValue)) { adTypeEdgeValue = GoogleAdTypeDic[mobileValue].ToString(); } else { adTypeEdgeValue = GoogleAdTypeDic[Const.AdTypeValues.Mobile_ad].ToString(); } } ad.ExtraFields[AdType] = (int)(EdgeAdType)Enum.Parse(typeof(EdgeAdType), adTypeEdgeValue, true);; /****************************************************************/ #endregion //Creative ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.DisplayUrl, Text = _adsReader.Current[Const.DisplayURLFieldName] }); #region Ad Tracker segment /******************************************************/ ////Setting Tracker for Ad if (!String.IsNullOrWhiteSpace(_adsReader.Current[Const.DestUrlFieldName])) { ad.DestinationUrl = _adsReader.Current[Const.DestUrlFieldName]; if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(ad); } } //if Ad doesnt contains tracker than check for kwd tracker if (Convert.ToBoolean(this.Delivery.Parameters["UseKwdTrackerAsAdTracker"])) { if (kwd != null && kwd.Segments != null && kwd.Segments.ContainsKey(this.ImportManager.SegmentTypes[Segment.Common.Tracker])) { if (kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] != null && kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value != null) { SegmentObject tracker = kwd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]]; //if value contains ADID than replace ADID with AD original id tracker.Value = tracker.Value.Replace("ADID", ad.OriginalID); ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]] = tracker; } } } /******************************************************/ #endregion #region Campaign /****************************************************************/ ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign() { OriginalID = _adsReader.Current[Const.CampaignIdFieldName], Name = _adsReader.Current[Const.CampaignFieldName], //Status = campaign_Status_Data[Convert.ToInt64(_adsReader.Current[Const.CampaignIdFieldName])] }; /****************************************************************/ #endregion #region Image /****************************************************************/ //Image Type > Create Image if (String.Equals(Convert.ToString(_adsReader.Current[Const.AdTypeFieldName]), "Image ad")) { string adNameField = _adsReader.Current[Const.AdFieldName]; string[] imageParams = adNameField.Trim().Split(new Char[] { ':', ';' }); // Ad name: 468_60_Test7options_Romanian.swf; 468 x 60 ad.Name = imageParams[1].Trim(); ad.Creatives.Add(new ImageCreative() { ImageUrl = imageParams[1].Trim(), ImageSize = imageParams[2].Trim() }); } /****************************************************************/ #endregion #region Text od Display /****************************************************************/ else //Text ad or Display ad { ad.Name = _adsReader.Current[Const.AdFieldName]; ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.Title, Text = _adsReader.Current.Ad, }); ad.Creatives.Add(new TextCreative { TextType = TextCreativeType.Body, Text = _adsReader.Current["Description line 1"], Text2 = _adsReader.Current["Description line 2"] }); } /****************************************************************/ #endregion #region Adgroup /****************************************************************/ //Insert Adgroup ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup() { Campaign = (Campaign)ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], Value = _adsReader.Current[Const.AdGroupFieldName], OriginalID = _adsReader.Current[Const.AdGroupIdFieldName], // Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])] }; /****************************************************************/ #endregion #region Network /****************************************************************/ //Insert Network Type Display Network / Search Network //string networkType = Convert.ToString(_adsReader.Current[Const.NetworkFieldName]); //if (networkType.Equals(Const.GoogleSearchNetwork)) // networkType = Const.SystemSearchNetwork; //else if (networkType.Equals(Const.GoogleDisplayNetwork)) // networkType = Const.SystemDisplayNetwork; //ad.ExtraFields[NetworkType] = networkType; /****************************************************************/ #endregion importedAds.Add(adId, ad); this.ImportManager.ImportAd(ad); } else { ad = importedAds[adId]; } adMetricsUnit.Ad = ad; //INSERTING SEARCH KEYWORD INTO METRICS if (IsSearchKwd & kwd != null) { adMetricsUnit.TargetDimensions = new List <Target>(); adMetricsUnit.TargetDimensions.Add(kwd); } //INSERTING GDN KEYWORD INTO METRICS else if (placement != null) { //INSERTING KEYWORD INTO METRICS adMetricsUnit.TargetDimensions = new List <Target>(); adMetricsUnit.TargetDimensions.Add(placement); } //INSERTING METRICS DATA adMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(_adsReader.Current.Clicks)); //Currencies Conversion Support adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], ConvertToUSD ? this.ConvertToUSD(currencyCode.ToUpper(), (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000) : (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000); if (ConvertToUSD) { adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.CostBeforeConversion], (Convert.ToDouble(_adsReader.Current.Cost)) / 1000000); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.USDConversionRate], Convert.ToDouble(this.ImportManager.CurrencyRates[currencyCode].RateValue)); } adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(_adsReader.Current.Impressions)); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], Convert.ToDouble(_adsReader.Current[Const.AvgPositionFieldName])); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionOnePerClickFieldName])); adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], Convert.ToDouble(_adsReader.Current[Const.ConversionManyPerClickFieldName])); //Inserting conversion values string conversionKey = String.Format("{0}#{1}", ad.OriginalID, _adsReader.Current[Const.KeywordIdFieldName]); Dictionary <string, long> conversionDic = new Dictionary <string, long>(); if (importedAdsWithConv.TryGetValue(conversionKey, out conversionDic)) { foreach (var pair in conversionDic) { if (GoogleMeasuresDic.ContainsKey(pair.Key)) { adMetricsUnit.MeasureValues[this.ImportManager.Measures[GoogleMeasuresDic[pair.Key]]] = pair.Value; } } } adMetricsUnit.Currency = new Currency { Code = Convert.ToString(_adsReader.Current.Currency) }; this.ImportManager.ImportMetrics(adMetricsUnit); } #endregion } #region Getting Sitelinks Data without metrics if (this.Delivery.Parameters.ContainsKey("AppendSitelinks") && Boolean.Parse(this.Delivery.Parameters["AppendSitelinks"].ToString())) { string[] sitelinksRequiredHeaders = new string[1]; sitelinksRequiredHeaders[0] = Const.PlaceholderFeedItemID; DeliveryFile _sitelinkPerformanceFile = this.Delivery.Files[GoogleStaticReportsNamesUtill._reportNames[GA.ReportDefinitionReportType.PLACEHOLDER_FEED_ITEM_REPORT]]; var _sitelinkReader = new CsvDynamicReader(_sitelinkPerformanceFile.OpenContents(compression: FileCompression.Gzip), sitelinksRequiredHeaders); AdMetricsUnit siteLinkMetricsUnit = new AdMetricsUnit(); siteLinkMetricsUnit.Output = currentOutput; Ad sitelinkAd; using (_sitelinkReader) { this.Mappings.OnFieldRequired = field => _sitelinkReader.Current[field]; //to do : get site link tracker string[] sitelinkAttr; while (_sitelinkReader.Read()) { if (((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Equals(Const.Sitelink_EOF)) { break; } string sitelinkId = string.Format("{0}{1}{2}", _sitelinkReader.Current[Const.PlaceholderFeedItemID], _sitelinkReader.Current[Const.CampaignIdFieldName], _sitelinkReader.Current[Const.AdGroupIdFieldName]); sitelinkAd = new Ad(); sitelinkAd.OriginalID = sitelinkId; sitelinkAd.Channel = new Channel() { ID = 1 }; sitelinkAd.Account = new Account { ID = this.Delivery.Account.ID, OriginalID = (String)_adPerformanceFile.Parameters["AdwordsClientID"] }; //Creative sitelinkAttr = ((String)_sitelinkReader.Current[Const.SiteLinkAttributeValuesHeader]).Split(';'); bool legacy = sitelinkAttr.Count() != 4 ? true : false; string destUrl = string.Empty; bool destUrlParsingError = false; //Checking desturl errors ( we would like to insert only sitelinks that contains desturl ). try { destUrl = sitelinkAttr[1]; } catch (Exception e) { Log.Write("Error while trying to pars destination url from attribute field on sitelink report", e); destUrlParsingError = true; } if (!destUrlParsingError) { ////Setting Tracker for Ad if (!String.IsNullOrWhiteSpace(sitelinkAttr[1])) { sitelinkAd.DestinationUrl = sitelinkAttr[1]; if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad))) { this.Mappings.Objects[typeof(Ad)].Apply(sitelinkAd); } } sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] = new Campaign() { OriginalID = _sitelinkReader.Current[Const.CampaignIdFieldName], Name = _sitelinkReader.Current[Const.CampaignFieldName], }; //Insert Adgroup sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] = new AdGroup() { Campaign = (Campaign)sitelinkAd.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]], Value = _sitelinkReader.Current[Const.AdGroupFieldName], OriginalID = _sitelinkReader.Current[Const.AdGroupIdFieldName], // Status = adGroup_Status_Data[Convert.ToInt64(_adsReader.Current[Const.AdGroupIdFieldName])] }; sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.DisplayUrl, Text = sitelinkAttr[1] }); if (!legacy) // only in case it doesnt contains legacy siteink { sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.Body, Text = sitelinkAttr[2], Text2 = sitelinkAttr[3] }); } sitelinkAd.Name = "[Sitelink] " + sitelinkAttr[0]; sitelinkAd.Creatives.Add(new TextCreative { TextType = TextCreativeType.Title, Text = "[Sitelink] " + sitelinkAttr[0] }); //Ad Type //Note: changed to "sitelink" following Amir request sitelinkAd.ExtraFields[AdType] = (int)(EdgeAdType.Sitelink); siteLinkMetricsUnit.Ad = sitelinkAd; //Setting Default Currency as USD following Amir's request from March 2014 siteLinkMetricsUnit.Currency = new Currency { Code = "USD" }; //INSERTING METRICS DATA siteLinkMetricsUnit.MeasureValues = new Dictionary <Measure, double>(); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.AveragePosition], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionOnePerClickFieldName]], 0); siteLinkMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[GoogleMeasuresDic[Const.ConversionManyPerClickFieldName]], 0); ImportManager.ImportMetrics(siteLinkMetricsUnit); ImportManager.ImportAd(sitelinkAd); } } } }// end if #endregion currentOutput.Checksum = _totals; this.ImportManager.EndImport(); } return(Core.Services.ServiceOutcome.Success); }
protected override Core.Services.ServiceOutcome DoPipelineWork() { Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"]; StringBuilder warningsStr = new StringBuilder(); Dictionary <string, double> _totalsValidation = new Dictionary <string, double>(); Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>(); Dictionary <string, Ad> ads = new Dictionary <string, Ad>(); Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >(); #region Campaigns List <string> campaignsFiles = filesByType[Consts.FileTypes.Campaigns]; foreach (var campaignFile in campaignsFiles) { DeliveryFile campaigns = this.Delivery.Files[campaignFile]; var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*"); using (campaignsReader) { while (campaignsReader.Read()) { Campaign camp = new Campaign() { Name = campaignsReader.Current.name, OriginalID = Convert.ToString(campaignsReader.Current.campaign_id), Channel = new Channel() { ID = 6 }, Account = new Account() { ID = this.Delivery.Account.ID, OriginalID = this.Delivery.Account.OriginalID.ToString() } }; long campaignStatus = long.Parse(campaignsReader.Current.campaign_status); switch (campaignStatus) { case 1: camp.Status = ObjectStatus.Active; break; case 2: camp.Status = ObjectStatus.Paused; break; case 3: camp.Status = ObjectStatus.Deleted; break; } campaignsData.Add(camp.OriginalID, camp); } campaigns.History.Add(DeliveryOperation.Imported, Instance.InstanceID); } } #endregion this.ReportProgress(0.1); #region adGroups And Targeting List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups]; foreach (var adGroup in adGroupsFiles) { DeliveryFile adGroups = this.Delivery.Files[adGroup]; var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*"); using (adGroupsReader) { while (adGroupsReader.Read()) { Ad ad = new Ad(); ad.OriginalID = Convert.ToString(adGroupsReader.Current.ad_id); ad.Campaign = campaignsData[Convert.ToString(adGroupsReader.Current.campaign_id)]; ad.Name = adGroupsReader.Current.name; if (Instance.Configuration.Options.ContainsKey("AutoAdGroupSegment") && Instance.Configuration.Options["AutoAdGroupSegment"].ToLower() == "true") { string[] delimiter = new string[1]; delimiter[0] = string.Empty; if (!Instance.Configuration.Options.ContainsKey("AdGroupDelimiter")) { Edge.Core.Utilities.Log.Write(string.Format("Facebook{0}", this), Core.Utilities.LogMessageType.Warning); } else { delimiter[0] = Instance.Configuration.Options["AdGroupDelimiter"]; } ad.Segments[Segment.AdGroupSegment] = new SegmentValue(); ad.Segments[Segment.AdGroupSegment].Value = delimiter[0] == string.Empty ? ad.Name : ad.Name.Split(delimiter, StringSplitOptions.None)[0]; ad.Segments[Segment.AdGroupSegment].OriginalID = delimiter[0] == string.Empty ? (ad.Name + ad.Campaign.OriginalID + ad.Campaign.Account.ID) : (ad.Name.Split(delimiter, StringSplitOptions.None)[0] + ad.Campaign.OriginalID + ad.Campaign.Account.ID); } else { ad.Segments[Segment.AdGroupSegment] = new SegmentValue(); ad.Segments[Segment.AdGroupSegment].Value = ad.Name; ad.Segments[Segment.AdGroupSegment].OriginalID = ad.Name + ad.Campaign.OriginalID + ad.Campaign.Account.ID; } // adgroup targeting string age_min = string.Empty; if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min")) { age_min = adGroupsReader.Current.targeting["age_min"]; } if (!string.IsNullOrEmpty(age_min)) { AgeTarget ageTarget = new AgeTarget() { FromAge = int.Parse(age_min), ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"]) }; ad.Targets.Add(ageTarget); } List <object> genders = null; if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders")) { genders = adGroupsReader.Current.targeting["genders"]; } if (genders != null) { foreach (object gender in genders) { GenderTarget genderTarget = new GenderTarget(); if (gender.ToString() == "1") { genderTarget.Gender = Gender.Male; } else if (gender.ToString() == "2") { genderTarget.Gender = Gender.Female; } else { genderTarget.Gender = Gender.Unspecified; } genderTarget.OriginalID = gender.ToString(); ad.Targets.Add(genderTarget); } } if (adGroupsReader.Current.creative_ids != null) { foreach (string creative in adGroupsReader.Current.creative_ids) { if (!adsBycreatives.ContainsKey(creative)) { adsBycreatives.Add(creative, new List <Ad>()); } adsBycreatives[creative].Add(ad); } } ads.Add(ad.OriginalID, ad); } adGroups.History.Add(DeliveryOperation.Imported, Instance.InstanceID); } } #endregion #region AdGroupStats start new import session //GetAdGroupStats using (var session = new AdMetricsImportManager(this.Instance.InstanceID)) { session.BeginImport(this.Delivery); #region for validation foreach (var measure in session.Measures) { if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired)) { if (!_totalsValidation.ContainsKey(measure.Key)) { _totalsValidation.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP } } } #endregion if (filesByType.ContainsKey(Consts.FileTypes.AdGroupStats)) { List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats]; foreach (var adGroupStat in adGroupStatsFiles) { DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat]; var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*"); using (adGroupStatsReader) { while (adGroupStatsReader.Read()) { AdMetricsUnit adMetricsUnit = new AdMetricsUnit(); Ad tempAd; if (adGroupStatsReader.Current.adgroup_id != null) { if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd)) { adMetricsUnit.Ad = tempAd; adMetricsUnit.PeriodStart = this.Delivery.TargetPeriod.Start.ToDateTime(); adMetricsUnit.PeriodEnd = this.Delivery.TargetPeriod.End.ToDateTime(); // Common and Facebook specific meausures if (_totalsValidation.ContainsKey(Measure.Common.Clicks)) { _totalsValidation[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks); } adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Clicks]] = Convert.ToInt64(adGroupStatsReader.Current.clicks); adMetricsUnit.MeasureValues[session.Measures[Measure.Common.UniqueClicks]] = Convert.ToInt64(adGroupStatsReader.Current.unique_clicks); if (_totalsValidation.ContainsKey(Measure.Common.Impressions)) { _totalsValidation[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions); } adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Impressions]] = Convert.ToInt64(adGroupStatsReader.Current.impressions); adMetricsUnit.MeasureValues[session.Measures[Measure.Common.UniqueImpressions]] = Convert.ToInt64(adGroupStatsReader.Current.unique_impressions); if (_totalsValidation.ContainsKey(Measure.Common.Cost)) { _totalsValidation[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d; } adMetricsUnit.MeasureValues[session.Measures[Measure.Common.Cost]] = Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d); adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions)); adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions)); adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks)); adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks)); adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d); //adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Actions], double.Parse(adGroupStatsReader.Current.actions)); adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Actions], 0); adMetricsUnit.MeasureValues.Add(session.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections)); adMetricsUnit.TargetMatches = new List <Target>(); session.ImportMetrics(adMetricsUnit); } else { warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID)); } } else { warningsStr.AppendLine("adGroupStatsReader.Current.id=null"); } } adGroupStats.History.Add(DeliveryOperation.Imported, Instance.InstanceID); //TODO: HISTORY WHEN?PROCCESED IS AFTER DATABASE'? this.ReportProgress(0.4); } #endregion this.ReportProgress(0.6); #region Creatives List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives]; Dictionary <string, string> usedCreatives = new Dictionary <string, string>(); foreach (string creative in creativeFiles) { DeliveryFile creativeFile = Delivery.Files[creative]; var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*"); using (adGroupCreativesReader) { while (adGroupCreativesReader.Read()) { List <Ad> adsByCreativeID = null; if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.creative_id)) { if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.creative_id)) { usedCreatives.Add(adGroupCreativesReader.Current.creative_id, adGroupCreativesReader.Current.creative_id); adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.creative_id]; } } if (adsByCreativeID != null) { foreach (Ad ad in adsByCreativeID) { ad.DestinationUrl = adGroupCreativesReader.Current.link_url; if (!string.IsNullOrEmpty(ad.DestinationUrl)) { SegmentValue tracker = this.AutoSegments.ExtractSegmentValue(Segment.TrackerSegment, ad.DestinationUrl); if (tracker != null) { ad.Segments[Segment.TrackerSegment] = tracker; } } ad.Creatives = new List <Creative>(); switch ((string)adGroupCreativesReader.Current.type) { case "8": case "9": case "10": case "16": case "17": case "19": case "25": { TextCreative sponserStory = new TextCreative() { OriginalID = adGroupCreativesReader.Current.creative_id, TextType = TextCreativeType.Title, Text = "Sponsored Story" }; ad.Creatives.Add(sponserStory); break; } case "27": { TextCreative sponserStory = new TextCreative() { OriginalID = adGroupCreativesReader.Current.creative_id, TextType = TextCreativeType.Title, Text = "Page Ads for a Page post" }; ad.Creatives.Add(sponserStory); break; } case "1": case "2": case "3": case "4": case "12": { ImageCreative ic = new ImageCreative() { ImageUrl = adGroupCreativesReader.Current.image_url, OriginalID = adGroupCreativesReader.Current.creative_id //Name = adGroupCreativesReader.Current.name }; if (!string.IsNullOrEmpty(ic.ImageUrl)) { ad.Creatives.Add(ic); } TextCreative bc = new TextCreative() { OriginalID = adGroupCreativesReader.Current.creative_id, TextType = TextCreativeType.Body, Text = adGroupCreativesReader.Current.body //Name = adGroupCreativesReader.Current.name }; if (!string.IsNullOrEmpty(bc.Text)) { ad.Creatives.Add(bc); } //bug creative type =9 story like TextCreative tc = new TextCreative() { OriginalID = adGroupCreativesReader.Current.creative_id, TextType = TextCreativeType.Title, Text = adGroupCreativesReader.Current.title }; if (!string.IsNullOrEmpty(bc.Text)) { ad.Creatives.Add(tc); } break; } } session.ImportAd(ad); } } //TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS) } creativeFile.History.Add(DeliveryOperation.Imported, Instance.InstanceID); } #endregion } } } session.HistoryEntryParameters.Add(Edge.Data.Pipeline.Common.Importing.Consts.DeliveryHistoryParameters.ChecksumTotals, _totalsValidation); session.EndImport(); if (!string.IsNullOrEmpty(warningsStr.ToString())) { Log.Write(warningsStr.ToString(), LogMessageType.Warning); } } return(Core.Services.ServiceOutcome.Success); }