protected override MetricsUnit GetSampleMetrics()
{
var headers = new[] { AdWordsConst.AdPreRequiredHeader };
// load sample keywords
var file = new DeliveryFile {
Location = Configuration.Parameters.Get <string>("KeywordSampleFile")
};
LoadKeywords(file, headers, FileCompression.None);
file = new DeliveryFile {
Location = Configuration.Parameters.Get <string>("PlacementSampleFile")
};
LoadPlacements(file, new[] { AdWordsConst.AutoPlacRequiredHeader }, FileCompression.None);
// load ad
using (_adsReader = new CsvDynamicReader(Configuration.Parameters.Get <string>("AdSampleFile"), headers))
{
Mappings.OnFieldRequired = fieldName => _adsReader.Current[fieldName];
if (_adsReader.Read())
{
CurrentMetricsUnit = new MetricsUnit {
GetEdgeField = GetEdgeField, Output = new DeliveryOutput()
};
MetricsMappings.Apply(CurrentMetricsUnit);
return(CurrentMetricsUnit);
}
}
throw new ConfigurationErrorsException(String.Format("Failed to read sample metrics from file: {0}", Configuration.Parameters.Get <string>("AdSampleFile")));
}
private void LoadKeywords(DeliveryFile file, string[] headers, FileCompression compression = FileCompression.Gzip)
{
if (file == null)
throw new ArgumentException("Keywords delivery file does not exist");
_keywordsCache.Clear();
using (var keywordsReader = new CsvDynamicReader(file.OpenContents(compression: compression), headers))
{
Mappings.OnFieldRequired = fieldName => keywordsReader.Current[fieldName];
keywordsReader.MatchExactColumns = false;
while (keywordsReader.Read())
{
if (keywordsReader.Current[AdWordsConst.KeywordIdFieldName] == AdWordsConst.EOF)
break;
var keywordPrimaryKey = new KeywordPrimaryKey
{
KeywordId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.KeywordIdFieldName]),
AdgroupId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.AdGroupIdFieldName]),
CampaignId = Convert.ToInt64(keywordsReader.Current[AdWordsConst.CampaignIdFieldName])
};
var keyword = new KeywordTarget();
KeywordMappings.Apply(keyword);
_keywordsCache.Add(keywordPrimaryKey.ToString(), keyword);
}
}
}
private void CreateNextFiles(DeliveryFile file, int counter, List <DeliveryFile> list)
{
string next = GetNextUrl(file);
if (!string.IsNullOrEmpty(next))
{
var url = FixUrlChars(next);
var name = file.Name.Substring(0, file.Name.IndexOf('-'));
var nameFormat = "{0}-{1}.json";
var fileName = string.Format(nameFormat, name, counter);
var fileType = file.Parameters[Consts.DeliveryFileParameters.FileType].ToString();
var f = CreateDeliveryFile(fileName, url, fileType);
if (!filesByType.ContainsKey((Consts.FileTypes)f.Parameters[Consts.DeliveryFileParameters.FileType]))
{
filesByType.Add((Consts.FileTypes)f.Parameters[Consts.DeliveryFileParameters.FileType],
new List <string>());
}
filesByType[(Consts.FileTypes)f.Parameters[Consts.DeliveryFileParameters.FileType]].Add(f.Name);
list.Add(f);
}
}
private void LoadCreatives(DeliveryFile file)
{
using (var reader = new JsonDynamicReader(file.OpenContents(), "$.data[*].*"))
{
Mappings.OnFieldRequired = fieldName =>
{
try
{
return(reader.Current[fieldName]);
}
catch (Exception ex)
{
Log(String.Format("Failed to read field '{0}' for creative id = {1} in file {2}, ex: {3}", fieldName, reader.Current.creative_id, file.Name, ex.Message), LogMessageType.Error);
return(null);
}
};
while (reader.Read())
{
var creative = new CompositeCreative();
CreativeMappings.Apply(creative);
_creativeCache.Add(reader.Current.creative_id, creative);
}
}
}
private void LoadPlacements(DeliveryFile file, string[] headers, FileCompression compression = FileCompression.Gzip)
{
if (file == null)
{
throw new ArgumentException("Placement delivery file does not exist");
}
_placementsCache.Clear();
using (var placementsReader = new CsvDynamicReader(file.OpenContents(compression: compression), headers))
{
Mappings.OnFieldRequired = fieldName => placementsReader.Current[fieldName];
placementsReader.MatchExactColumns = false;
while (placementsReader.Read())
{
if (placementsReader.Current[AdWordsConst.PlacementIdFieldName] == AdWordsConst.EOF)
{
break;
}
var placementPrimaryKey = new KeywordPrimaryKey
{
KeywordId = Convert.ToInt64(placementsReader.Current[AdWordsConst.PlacementIdFieldName]),
AdgroupId = Convert.ToInt64(placementsReader.Current[AdWordsConst.AdGroupIdFieldName]),
CampaignId = Convert.ToInt64(placementsReader.Current[AdWordsConst.CampaignIdFieldName])
};
var placement = new PlacementTarget();
PlacementMappings.Apply(placement);
_placementsCache.Add(placementPrimaryKey.ToString(), placement);
}
}
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
if (this.Delivery == null)
{
int accountID = this.Configuration.Parameters.Get <int>("AccountID", emptyIsError: false, defaultValue: -1);
int channelID = this.Configuration.Parameters.Get <int>("ChannelID", emptyIsError: false, defaultValue: -1);
this.Delivery = NewDelivery();
this.Delivery.TimePeriodDefinition = this.Configuration.TimePeriod.Value;
this.Delivery.Account = accountID != -1 ? new Account()
{
ID = accountID
} : null;
this.Delivery.FileDirectory = this.Configuration.Parameters.Get <string>(Const.DeliveryServiceConfigurationOptions.FileDirectory);
if (channelID != -1)
{
this.Delivery.Channel = new Channel()
{
ID = channelID
}
}
;
}
DeliveryFile deliveryFile = new DeliveryFile()
{
Name = this.Configuration.Parameters.Get <string>(Const.DeliveryServiceConfigurationOptions.DeliveryFileName, false) ?? "File",
SourceUrl = this.Configuration.Parameters.Get <string>(Const.DeliveryServiceConfigurationOptions.SourceUrl, false)
};
this.Delivery.Files.Add(deliveryFile);
WebRequest request = FileWebRequest.Create(this.Configuration.Parameters.Get <string>("SourceUrl"));
if (request is FtpWebRequest)
{
// FTP Configuration
this.Delivery.Parameters["UsePassive"] = this.Configuration.Parameters.Get <bool>("UsePassive", emptyIsError: false, defaultValue: true);
this.Delivery.Parameters["UseBinary"] = this.Configuration.Parameters.Get <bool>("UseBinary");
this.Delivery.Parameters["SourceUrl"] = this.Configuration.Parameters.Get <string>("SourceUrl");
this.Delivery.Parameters["UserID"] = this.Configuration.Parameters.Get <string>("UserID");
this.Delivery.Parameters["Password"] = Core.Utilities.Encryptor.Dec(this.Configuration.Parameters.Get <string>("Password"));
this.Delivery.Parameters["DirectoryWatcherLocation"] = this.Configuration.Parameters.Get <string>("DirectoryWatcherLocation", false);
deliveryFile.FileSignature = string.Format("{0}-{1}-{2}", this.Configuration.Parameters.Get <object>("DeliveryFileName"),
this.Configuration.Parameters.Get <object>("FileModifyDate"),
this.Configuration.Parameters.Get <object>("FileSize"));
deliveryFile.Parameters.Add("Size", this.Configuration.Parameters.Get <string>("Size", emptyIsError: false));
}
this.Delivery.Save();
return(Core.Services.ServiceOutcome.Success);
}
}
IEnumerable <string> IRetriever.RetrieveData(DeliveryFile file, AdWordsUser user)
{
var offset = 0;
var sb = new StringBuilder();
// 1st row is a header (field list)
var outputLines = new List <string> {
file.Parameters["ReportFields"].ToString()
};
var service = user.GetService(AdWordsService.v201302.CampaignCriterionService) as CampaignCriterionService;
if (service == null)
{
throw new Exception("Failed to create CampaignCriterionService");
}
var selector = new Selector
{
fields = file.Parameters["ReportFields"].ToString().Split(','),
paging = new Paging {
numberResults = PAGE_SIZE
},
};
if (!String.IsNullOrEmpty(file.Parameters["ReportFilter"].ToString()))
{
selector.predicates = new[] { new Predicate {
field = "CriteriaType", @operator = PredicateOperator.IN, values = file.Parameters["ReportFilter"].ToString().Split('|')
} }
}
;
//var query = String.Format("SELECT {0} WHERE CriteriaType IN ('LOCATION', 'LANGUAGE') LIMIT {{0}}, {1} ", file.Parameters["ReportFields"], PAGE_SIZE);
CampaignCriterionPage page;
do
{
// get next page
selector.paging.startIndex = offset;
page = service.get(selector);
if (page != null && page.entries != null)
{
// for each criterion create row with all fields
foreach (var campaignCriterion in page.entries)
{
sb.Clear();
foreach (var field in selector.fields)
{
sb.AppendFormat("{0}{1}", sb.Length > 0 ? "," : "", GetFieldValue(campaignCriterion, field));
}
outputLines.Add(sb.ToString());
}
}
offset += PAGE_SIZE;
} while (page != null && offset < page.totalNumEntries);
return(outputLines);
}
private void DownloadFile(DeliveryFile file, AdWordsUser user, string postData)
{
WebRequest request = CreateAdwordsReportRequest(file, user, postData);
FileDownloadOperation operation = file.Download(request);
operation.RequestBody = postData;
_batchDownloadOperation.Add(operation);
}
private DeliveryFile CreateDeliveryFile(string name, string url, string type)
{
var f = new DeliveryFile();
f.Name = name;
f.Parameters.Add(Consts.DeliveryFileParameters.Url, string.Format("{0}", url));
f.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, (long)Consts.FileSubType.New);
f.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), type));
return(f);
}
protected override ServiceOutcome DoPipelineWork()
{
this.Delivery = this.NewDelivery(); // setup delivery
this.Delivery.FileDirectory = Instance.Configuration.Options[Edge.Data.Pipeline.Services.Const.DeliveryServiceConfigurationOptions.FileDirectory];
this.Delivery.Account = new Account()
{
ID = 0
};
string fileName = string.Empty;
//FileName
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["FileName"]))
{
throw new Exception("Missing Configuration Param , FileName");
}
else
{
fileName = this.Instance.Configuration.Options["FileName"];
}
DeliveryFile _file = new DeliveryFile()
{
Name = fileName
};
_file.SourceUrl = Instance.Configuration.Options["SourceUrl"];
_file.Parameters.Add("XML.IsAttribute", Instance.Configuration.Options["XML.IsAttribute"]);
_file.Parameters.Add("XML.Path", Instance.Configuration.Options["XML.Path"]);
this.Delivery.Files.Add(_file);
//Set Output
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("[{0}]-[{1}]",
this.TimePeriod.ToAbsolute(),
Instance.Configuration.Options["SourceUrl"]
)),
Account = new Data.Objects.Account()
{
ID = 0
},
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd
});
this.Delivery.Save();
return(Core.Services.ServiceOutcome.Success);
}
private FileInfo AddDataFilesToBatch(BatchDownloadOperation batch, DeliveryFile file)
{
if (file.Parameters[Consts.DeliveryFileParameters.FileSubType].Equals((long)Consts.FileSubType.Data))
{
FileDownloadOperation fileDownloadOperation = file.Download(CreateRequest(file.Parameters[Consts.DeliveryFileParameters.Url].ToString()));
batch.Add(fileDownloadOperation);
return(fileDownloadOperation.FileInfo);
}
return(null);
}
private WebRequest CreateAdwordsReportRequest(DeliveryFile file, AdWordsUser user, string postBody)
{
WebRequest request = HttpWebRequest.Create(file.SourceUrl);
request.Timeout = 100000;
if (!string.IsNullOrEmpty(postBody))
{
request.Method = "POST";
}
if (!string.IsNullOrEmpty((user.Config as AdWordsAppConfig).ClientCustomerId))
{
request.Headers.Add("clientCustomerId: " + (user.Config as AdWordsAppConfig).ClientCustomerId);
}
request.ContentType = "application/x-www-form-urlencoded";
if ((user.Config as AdWordsAppConfig).EnableGzipCompression)
{
(request as HttpWebRequest).AutomaticDecompression = DecompressionMethods.GZip
| DecompressionMethods.Deflate;
}
else
{
(request as HttpWebRequest).AutomaticDecompression = DecompressionMethods.None;
}
if (user.OAuthProvider != null)
{
request.Headers["Authorization"] = user.OAuthProvider.GetAuthHeader();
}
else
{
throw new Exception("AdWordsErrorMessages.OAuthProviderCannotBeNull");
}
//if ((user.Config as AdWordsAppConfig).AuthorizationMethod == AdWordsAuthorizationMethod.ClientLogin)
//{
// string authToken = AdwordsUtill.GetAuthToken(user);
// (user.Config as AdWordsAppConfig).AuthToken = authToken;
// request.Headers["Authorization"] = "GoogleLogin auth=" + authToken;
//}
// request.Headers.Add("returnMoneyInMicros: true");
request.Headers.Add("developerToken: " + (user.Config as AdWordsAppConfig).DeveloperToken);
//Try to unmark the following comment in case of api error
//The client library will use only apiMode = true.
request.Headers.Add("apiMode", "true");
return(request);
}
protected void LoadConfiguration()
{
var checksumThreshold = Configuration.Parameters.Get <string>(Consts.ConfigurationOptions.ChecksumTheshold);
_importManagerOptions = new MetricsDeliveryManagerOptions
{
SqlTransformCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlTransformCommand),
SqlStageCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlStageCommand),
SqlRollbackCommand = Configuration.Parameters.Get <string>(Consts.AppSettings.SqlRollbackCommand),
ChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold)
};
_deliveryFile = Delivery.Files[Configuration.DeliveryFileName];
if (_deliveryFile == null)
{
throw new Exception(String.Format("Could not find delivery file '{0}' in the delivery.", Configuration.DeliveryFileName));
}
if (!Enum.TryParse(Configuration.Compression, out _compression))
{
throw new ConfigurationErrorsException(String.Format("Invalid compression type '{0}'.", Configuration.Compression));
}
if (Configuration.Parameters.ContainsKey("EOF"))
{
_eofIndication = Configuration.Parameters.Get <string>("EOF");
}
if (Configuration.Parameters.ContainsKey("EOF_FieldName"))
{
_eofFieldName = Configuration.Parameters.Get <string>("EOF_FieldName");
}
// Create format processor from configuration
var readerAdapterType = Type.GetType(Configuration.ReaderAdapterType, true);
ReaderAdapter = (ReaderAdapter)Activator.CreateInstance(readerAdapterType);
Mappings.OnFieldRequired = ReaderAdapter.GetField;
Mappings.OnMappingApplied = SetEdgeType;
if (!Mappings.Objects.TryGetValue(typeof(MetricsUnit), out MetricsMappings))
{
throw new MappingConfigurationException("Missing mapping definition for GenericMetricsUnit.", "Object");
}
if (!Mappings.Objects.TryGetValue(typeof(Signature), out SignatureMappings))
{
throw new MappingConfigurationException("Missing mapping definition for Signature.", "Object");
}
}
private void LoadCampaigns(DeliveryFile file)
{
using (var reader = new JsonDynamicReader(file.OpenContents(), "$.data[*].*"))
{
Mappings.OnFieldRequired = fieldName => reader.Current[fieldName];
while (reader.Read())
{
var campaign = new Campaign();
CampaignMappings.Apply(campaign);
_campaignCache.Add(campaign.OriginalID, campaign);
}
}
}
private void DownloadFile(DeliveryFile file)
{
FtpWebRequest request = (FtpWebRequest)FtpWebRequest.Create(file.SourceUrl);
request.UseBinary = true;
request.Credentials = new NetworkCredential
(
this.Delivery.Parameters["UserID"].ToString(),
this.Delivery.Parameters["Password"].ToString()
);
request.Method = WebRequestMethods.Ftp.DownloadFile;
request.UsePassive = true;
_batchDownloadOperation.Add(file.Download(request, Convert.ToInt64(file.Parameters["Size"])));
}
private WebRequest CreateAdwordsReportRequest(DeliveryFile file, AdWordsUser user, string postBody)
{
var config = user.Config as AdWordsAppConfig;
var request = WebRequest.Create(file.SourceUrl) as HttpWebRequest;
if (config == null || request == null)
{
return(null);
}
request.Timeout = 100000;
if (!string.IsNullOrEmpty(postBody))
{
request.Method = "POST";
}
if (!string.IsNullOrEmpty(config.ClientCustomerId))
{
request.Headers.Add("clientCustomerId: " + (user.Config as AdWordsAppConfig).ClientCustomerId);
}
request.ContentType = "application/x-www-form-urlencoded";
if (config.EnableGzipCompression)
{
request.AutomaticDecompression = DecompressionMethods.GZip
| DecompressionMethods.Deflate;
}
else
{
request.AutomaticDecompression = DecompressionMethods.None;
}
if (config.AuthorizationMethod == AdWordsAuthorizationMethod.ClientLogin)
{
string authToken = AdwordsUtill.GetAuthToken(user);
(user.Config as AdWordsAppConfig).AuthToken = authToken;
request.Headers["Authorization"] = "GoogleLogin auth=" + authToken;
}
request.Headers.Add("returnMoneyInMicros: true");
request.Headers.Add("developerToken: " + (user.Config as AdWordsAppConfig).DeveloperToken);
//Try to unmark the following comment in case of api error
//The client library will use only apiMode = true.
//request.Headers.Add("apiMode", "true");
return(request);
}
private string GetNextUrl(DeliveryFile file)
{
string next = string.Empty;
using (StreamReader reader = new StreamReader(file.OpenContents()))
{
string json = reader.ReadToEnd();
dynamic dynamicObject = JsonConvert.DeserializeObject(json);
if (dynamicObject.paging != null)
{
next = dynamicObject.paging.next;
}
}
return(next);
}
private List <DeliveryFile> FetchNext(List <DeliveryFile> fetchFrom, int offset)
{
BatchDownloadOperation nextBatch = new BatchDownloadOperation();
List <DeliveryFile> nextRecordsFiles = new List <DeliveryFile>();
foreach (DeliveryFile ReportFile in fetchFrom)
{
//setting cuurent file has batched and batching next file
ReportFile.Parameters.Add("Batch", true);
string fileName = ReportFile.Name + "-" + offset;
JsonDynamicReader reportReader = new JsonDynamicReader(ReportFile.OpenContents(compression: FileCompression.None), "$.nextRecordsUrl");
string nextRecordPath;
if (reportReader.Read())
{
nextRecordPath = reportReader.Current.nextRecordsUrl;
DeliveryFile nextRecordFile = new DeliveryFile();
nextRecordFile.SourceUrl = ((Token)(ReportFile.Parameters["Token"])).instance_url + nextRecordPath;
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(nextRecordFile.SourceUrl);
request.Headers.Add("Authorization: OAuth " + ((Token)(ReportFile.Parameters["Token"])).access_token);
//check if response contains more than one file
FileDownloadOperation fileDownloadOperation = nextRecordFile.Download(request);
nextBatch.Add(fileDownloadOperation);
nextRecordsFiles.Add(nextRecordFile);
}
}
if (nextRecordsFiles.Count > 0)
{
nextBatch.Start();
nextBatch.Wait();
nextBatch.EnsureSuccess();
foreach (DeliveryFile file in FetchNext(nextRecordsFiles, offset))
{
this.Delivery.Files.Add(file);
}
}
return(nextRecordsFiles);
}
private void LoadConversions(DeliveryFile file, string[] headers)
{
if (file == null)
{
throw new ArgumentException("Ad conversions delivery file does not exist");
}
_importedAdsWithConv.Clear();
using (var conversionsReader = new CsvDynamicReader(file.OpenContents(compression: FileCompression.Gzip), headers))
{
while (conversionsReader.Read())
{
if (conversionsReader.Current[AdWordsConst.AdIDFieldName] == AdWordsConst.EOF)
{
break; // if end of report
}
string conversionKey = String.Format("{0}#{1}#{2}", conversionsReader.Current[AdWordsConst.AdIDFieldName],
conversionsReader.Current[AdWordsConst.KeywordIdFieldName],
conversionsReader.Current[AdWordsConst.DateFieldName]);
var trackingPurpose = Convert.ToString(conversionsReader.Current[AdWordsConst.ConversionTrackingPurposeFieldName]);
var manyPerClick = Convert.ToInt64(conversionsReader.Current[AdWordsConst.ConversionManyPerClickFieldName]);
if (!_importedAdsWithConv.ContainsKey(conversionKey))
{
// add new conversion key with new dictionary of tracking purpose and clicks
_importedAdsWithConv.Add(conversionKey, new Dictionary <string, long> {
{ trackingPurpose, manyPerClick }
});
}
else // if conversion key exists
{
if (_importedAdsWithConv[conversionKey].ContainsKey(trackingPurpose))
{
// if purpose exists --> sum to existing value
_importedAdsWithConv[conversionKey][trackingPurpose] += manyPerClick;
}
else
{
// create new entry for new tracking purpose
_importedAdsWithConv[conversionKey].Add(trackingPurpose, manyPerClick);
}
}
}
}
}
private bool ValidateReport(DeliveryFile file, AdWordsUser user, string postData, out string ErrorMsg)
{
ErrorMsg = string.Empty;
WebRequest request = CreateAdwordsReportRequest(file, user, postData);
request.Proxy = null;
request.Headers.Add("validateOnly: true"); // Add Validate
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(postData);
}
//Try to ger response from server
WebResponse response = null;
try
{
response = request.GetResponse();
}
catch (WebException we)
{
response = we.Response;
byte[] preview = ConvertStreamToByteArray(response.GetResponseStream(), MAX_ERROR_LENGTH);
string previewString = ConvertPreviewBytesToString(preview);
if (!string.IsNullOrEmpty(previewString))
{
//BUG fix due to adwords new version changes
//if (Regex.IsMatch(previewString, REPORT_ERROR_REGEX))
if ((previewString.Contains(typeof(GA201406.AuthorizationError).Name)) || (previewString.Contains(typeof(GA201406.AuthenticationError).Name)))
{
ErrorMsg = previewString;
return(false);
}
else
{
throw new Exception(previewString);
}
}
}
return(true);
}
private void LoadAds(DeliveryFile file)
{
using (var reader = new JsonDynamicReader(file.OpenContents(), "$.data[*].*"))
{
Mappings.OnFieldRequired = fieldName =>
{
var parts = fieldName.Split('@');
if (parts.Length > 1)
{
// value from the dictionary by key
var fieldValue = reader.Current[parts[1]].ContainsKey(parts[0]) ?
reader.Current[parts[1]][parts[0]] is List <object>?
String.Join(",", reader.Current[parts[1]][parts[0]]) :
reader.Current[parts[1]][parts[0]] : null;
return(fieldValue);
}
// 1st item in the list
if (reader.Current[fieldName] is IList <object> )
{
return((reader.Current[fieldName] as IList <object>)[0]);
}
// just a string value
return(reader.Current[fieldName]);
};
while (reader.Read())
{
var ad = new Ad();
AdMappings.Apply(ad);
_adCache.Add(ad.OriginalID, ad);
}
}
}
private void DownloadFile(DeliveryFile file)
{
WebRequest request = FileWebRequest.Create(file.SourceUrl);
/* FTP */
if (request.GetType().Equals(typeof(FtpWebRequest)))
{
FtpWebRequest ftpRequest = (FtpWebRequest)request;
ftpRequest.UseBinary = true;
ftpRequest.Credentials = new NetworkCredential
(
this.Delivery.Parameters["UserID"].ToString(),
this.Delivery.Parameters["Password"].ToString()
);
ftpRequest.Method = WebRequestMethods.Ftp.DownloadFile;
ftpRequest.UsePassive = true;
_batch.Add(file.Download(request, Convert.ToInt64(file.Parameters["Size"])));
}
/*OTHER*/
else
{
_batch.Add(file.Download(request));
}
}
private void CreateRequests()
{
List <ManualResetEvent> manualEvents = new List <ManualResetEvent>();
_waitHandle = new AutoResetEvent(false);
#region ADREPORT
DeliveryFile adReportFile = this.Delivery.Files[Const.Files.AdReport];
WS.ReportRequest adReportRequest;
adReportRequest = _adCenterApi.NewAdPerformanceReportRequest
(
WS.AdPerformanceReportColumn.AccountName, //required field
WS.AdPerformanceReportColumn.AccountNumber, // not necessary for this service version
WS.AdPerformanceReportColumn.CampaignName, //required field
WS.AdPerformanceReportColumn.TimePeriod, //required field if aggregation time!=Summary
//WS.AdPerformanceReportColumn.CampaignId, // why did MS leave this out? stupid f***s
WS.AdPerformanceReportColumn.AdGroupName,
WS.AdPerformanceReportColumn.AdGroupId,
WS.AdPerformanceReportColumn.AdId,
WS.AdPerformanceReportColumn.AdTitle,
WS.AdPerformanceReportColumn.AdDescription,
WS.AdPerformanceReportColumn.DestinationUrl,
WS.AdPerformanceReportColumn.Clicks,
WS.AdPerformanceReportColumn.AdType
);
if (string.IsNullOrEmpty(adReportFile.SourceUrl))
{
ManualResetEvent asyncWait = new ManualResetEvent(false);
manualEvents.Add(asyncWait);
Action getAdReportUrl = () =>
{
string innerFileName;
adReportFile.SourceUrl = _adCenterApi.SubmitReportRequest(adReportRequest, out innerFileName);
adReportFile.Parameters["InnerFileName"] = string.Format(@"\{0}.Csv", innerFileName);
};
getAdReportUrl.BeginInvoke(result =>
{
asyncWait.Set();
},
null);
}
#endregion
#region CAMPAIGNREPORT
DeliveryFile campaignReportFile = this.Delivery.Files[Const.Files.CampaignReport];
WS.ReportRequest campaignReportRequest;
campaignReportRequest = _adCenterApi.NewCampaignPerformanceReportRequest
(
WS.CampaignPerformanceReportColumn.AccountName,
WS.CampaignPerformanceReportColumn.CampaignName,
WS.CampaignPerformanceReportColumn.TimePeriod,
WS.CampaignPerformanceReportColumn.CampaignId,
WS.CampaignPerformanceReportColumn.Status,
WS.CampaignPerformanceReportColumn.Clicks
);
if (string.IsNullOrEmpty(campaignReportFile.SourceUrl))
{
ManualResetEvent asyncWait = new ManualResetEvent(false);
manualEvents.Add(asyncWait);
Action getCampaignReportUrl = () =>
{
string innerFileName;
campaignReportFile.SourceUrl = _adCenterApi.SubmitReportRequest(campaignReportRequest, out innerFileName);
campaignReportFile.Parameters["InnerFileName"] = string.Format(@"\{0}.Csv", innerFileName);
};
getCampaignReportUrl.BeginInvoke(result =>
{
asyncWait.Set();
},
null);
}
#endregion
#region KEYWORDREPORT
DeliveryFile keywordReportFile = this.Delivery.Files[Const.Files.KeywordReport];
WS.ReportRequest keywordReportRequest;
keywordReportRequest = _adCenterApi.NewKeywordPerformanceReportRequest(
WS.KeywordPerformanceReportColumn.TimePeriod, // special column
WS.KeywordPerformanceReportColumn.AdId,
WS.KeywordPerformanceReportColumn.AdGroupId,
WS.KeywordPerformanceReportColumn.CampaignId,
WS.KeywordPerformanceReportColumn.Keyword,
WS.KeywordPerformanceReportColumn.KeywordId,
WS.KeywordPerformanceReportColumn.DestinationUrl,
WS.KeywordPerformanceReportColumn.DeliveredMatchType,
WS.KeywordPerformanceReportColumn.CurrencyCode,
WS.KeywordPerformanceReportColumn.Impressions,
WS.KeywordPerformanceReportColumn.Clicks,
WS.KeywordPerformanceReportColumn.Spend,
WS.KeywordPerformanceReportColumn.AveragePosition,
WS.KeywordPerformanceReportColumn.Conversions,
WS.KeywordPerformanceReportColumn.QualityScore
);
if (string.IsNullOrEmpty(keywordReportFile.SourceUrl))
{
ManualResetEvent asyncWait = new ManualResetEvent(false);
manualEvents.Add(asyncWait);
Action getKeywordReportUrl = () =>
{
string innerFileName;
keywordReportFile.SourceUrl = _adCenterApi.SubmitReportRequest(keywordReportRequest, out innerFileName);
keywordReportFile.Parameters["InnerFileName"] = string.Format(@"\{0}.Csv", innerFileName);
};
getKeywordReportUrl.BeginInvoke(result =>
{
asyncWait.Set();
},
null);
}
#endregion
if (manualEvents.Count > 1)
{
WaitHandle.WaitAll(manualEvents.ToArray());
}
}
protected override ServiceOutcome DoPipelineWork()
{
#region Check config parameters
if (!Configuration.Parameters.ContainsKey("FilterDeleted"))
{
throw new Exception("Missing Configuration Param: FilterDeleted");
}
if (!Configuration.Parameters.ContainsKey("KeywordContentId"))
{
throw new Exception("Missing Configuration Param: KeywordContentId");
}
if (!Configuration.Parameters.ContainsKey("DeveloperToken"))
{
throw new Exception("Missing Configuration Param: DeveloperToken");
}
if (!Configuration.Parameters.ContainsKey("Adwords.MccEmail"))
{
throw new Exception("Missing Configuration Param: Adwords.MccEmail");
}
if (!Configuration.Parameters.ContainsKey("Adwords.ClientID"))
{
throw new Exception("Missing Configuration Param: Adwords.ClientID");
}
if (!Configuration.Parameters.ContainsKey("SubChannelName"))
{
throw new Exception("Missing Configuration Param: SubChannelName");
}
if (!Configuration.TimePeriod.HasValue)
{
throw new Exception("No time period is set for Service");
}
if (!Configuration.Parameters.ContainsKey("Adwords.ReportConfig"))
{
throw new Exception("Missing Configuration Param: Adwords.ReportConfig");
}
#endregion
var accountId = Configuration.Parameters.Get("AccountID", false, -1);
var channelId = Configuration.Parameters.Get("ChannelID", false, -1);
// create Delivery
if (Delivery == null)
{
Delivery = NewDelivery();
Delivery.TimePeriodDefinition = Configuration.TimePeriod.Value;
Delivery.Account = accountId != -1 ? new Account {
ID = accountId
} : null;
Delivery.FileDirectory = Configuration.Parameters.Get <string>(Const.DeliveryServiceConfigurationOptions.FileDirectory);
if (channelId != -1)
{
Delivery.Channel = new Channel {
ID = channelId
}
}
;
}
Progress = 0.3;
#region Must Have Params
Delivery.Parameters["FilterDeleted"] = Configuration.Parameters["FilterDeleted"];
//Delivery.Parameters["IncludeStatus"] = Configuration.Parameters.Get<string>("IncludeStatus");
//Get MCC Paramerters
Delivery.Parameters["DeveloperToken"] = Configuration.Parameters.Get <string>("DeveloperToken");
Delivery.Parameters["MccEmail"] = Configuration.Parameters.Get <string>("Adwords.MccEmail");
Delivery.Parameters["KeywordContentId"] = Configuration.Parameters.Get <string>("KeywordContentId");
//Get Account Client Id's
var adwordsClientIds = Configuration.Parameters.Get <string>("Adwords.ClientID").Split('|');
Delivery.Parameters["AdwordsClientIDs"] = adwordsClientIds;
// Get Reports configuration
var reportConfig = AdwordsReportConfig.Deserialize(Configuration.Parameters.Get <string>("Adwords.ReportConfig"));
#endregion
#region Nice to have params
//Check for includeZeroImpression
if (Configuration.Parameters.ContainsKey("includeZeroImpression"))
{
Delivery.Parameters["includeZeroImpression"] = Configuration.Parameters.Get <string>("includeZeroImpression");
}
else
{
Delivery.Parameters["includeZeroImpression"] = false;
}
//Check for includeConversionTypes
if (Configuration.Parameters.ContainsKey("includeConversionTypes"))
{
Delivery.Parameters["includeConversionTypes"] = Configuration.Parameters.Get <string>("includeConversionTypes");
}
else
{
Delivery.Parameters["includeConversionTypes"] = false; // default
}
//Check for includeDisplaytData
if (Configuration.Parameters.ContainsKey("includeDisplaytData"))
{
Delivery.Parameters["includeDisplaytData"] = Configuration.Parameters.Get <string>("includeDisplaytData");
}
else
{
Delivery.Parameters["includeDisplaytData"] = false; // default
}
#endregion
Progress = 0.5;
// create Delivery files for each Client ID and Report type
foreach (var clientId in adwordsClientIds)
{
foreach (var report in reportConfig.Reports.Where(x => x.Enable))
{
var file = new DeliveryFile {
Name = report.Name
};
file.Parameters.Add("ReportType", report.Type);
file.Parameters.Add("ReportFields", report.GetFieldList());
file.Parameters.Add("AdwordsClientID", clientId);
Delivery.Files.Add(file);
}
}
Progress = 0.8;
Delivery.Save();
return(ServiceOutcome.Success);
}
#endregion
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
Dictionary <Consts.FileTypes, List <string> > filesByType = (Dictionary <Consts.FileTypes, List <string> >)Delivery.Parameters["FilesByType"];
StringBuilder warningsStr = new StringBuilder();
Dictionary <string, Campaign> campaignsData = new Dictionary <string, Campaign>();
Dictionary <string, AdGroup> adGroupsData = new Dictionary <string, AdGroup>();
Dictionary <string, Ad> ads = new Dictionary <string, Ad>();
Dictionary <string, List <Ad> > adsBycreatives = new Dictionary <string, List <Ad> >();
var adStatIds = new Dictionary <string, string>();
var insertedAds = new Dictionary <string, string>();
var storyIds = new Dictionary <string, Dictionary <string, string> >();
var photoIds = new Dictionary <string, Dictionary <string, string> >();
DeliveryOutput currentOutput = Delivery.Outputs.First();
currentOutput.Checksum = new Dictionary <string, double>();
using (this.ImportManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
MeasureOptions = MeasureOptions.IsTarget | MeasureOptions.IsCalculated | MeasureOptions.IsBackOffice,
MeasureOptionsOperator = OptionsOperator.Not,
SegmentOptions = Data.Objects.SegmentOptions.All,
SegmentOptionsOperator = OptionsOperator.And
}))
{
this.ImportManager.BeginImport(this.Delivery);
#region AdSets
if (filesByType.ContainsKey(Consts.FileTypes.CampaignGroups))
{
List <string> campaignsFiles = filesByType[Consts.FileTypes.CampaignGroups];
foreach (var campaignFile in campaignsFiles)
{
DeliveryFile campaigns = this.Delivery.Files[campaignFile];
var campaignsReader = new JsonDynamicReader(campaigns.OpenContents(), "$.data[*].*");
using (campaignsReader)
{
while (campaignsReader.Read())
{
Campaign camp = new Campaign()
{
Name = campaignsReader.Current.name,
OriginalID = Convert.ToString(campaignsReader.Current.id),
};
campaignsData.Add(camp.OriginalID, camp);
}
}
}
}
#endregion
#region AdGroups
if (filesByType.ContainsKey(Consts.FileTypes.AdSets))
{
List <string> adSetList = filesByType[Consts.FileTypes.AdSets];
foreach (var adSet in adSetList)
{
DeliveryFile adSetDF = this.Delivery.Files[adSet];
var adSetReader = new JsonDynamicReader(adSetDF.OpenContents(), "$.data[*].*");
using (adSetReader)
{
while (adSetReader.Read())
{
var adGroupObj = new AdGroup()
{
Value = adSetReader.Current.name,
OriginalID = Convert.ToString(adSetReader.Current.id),
};
if (campaignsData.ContainsKey(adSetReader.Current.campaign_group_id))
{
adGroupObj.Campaign = campaignsData[adSetReader.Current.campaign_group_id];
}
adGroupsData.Add(adGroupObj.OriginalID, adGroupObj);
}
}
}
}
#endregion
#region adGroups And Targeting
//******************************************************************************************************************************************************
if (filesByType.ContainsKey(Consts.FileTypes.AdGroups))
{
List <string> adGroupsFiles = filesByType[Consts.FileTypes.AdGroups];
foreach (var adGroup in adGroupsFiles)
{
#region foreach
DeliveryFile adGroups = this.Delivery.Files[adGroup];
var adGroupsReader = new JsonDynamicReader(FileManager.Open(adGroups.Location), "$.data[*].*");
using (adGroupsReader)
{
while (adGroupsReader.Read())
{
var campaignId = Convert.ToString(adGroupsReader.Current.campaign_id);
if (adGroupsData.ContainsKey(campaignId) && ((AdGroup)adGroupsData[campaignId]).Campaign != null)
{
Ad ad = new Ad();
ad.OriginalID = Convert.ToString(adGroupsReader.Current.id);
ad.Segments = new Dictionary <Segment, SegmentObject>();
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Campaign]] =
((AdGroup)adGroupsData[Convert.ToString(adGroupsReader.Current.campaign_id)]).Campaign;
ad.Name = adGroupsReader.Current.name;
ad.Channel = new Channel()
{
ID = 6
};
ad.Account = new Account()
{
ID = this.Delivery.Account.ID,
OriginalID = this.Delivery.Account.OriginalID.ToString()
};
if (adGroupsData.ContainsKey(adGroupsReader.Current.campaign_id))
{
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.AdGroup]] =
adGroupsData[adGroupsReader.Current.campaign_id];
}
// adgroup targeting
string age_min = string.Empty;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("age_min"))
{
age_min = adGroupsReader.Current.targeting["age_min"];
}
if (!string.IsNullOrEmpty(age_min))
{
AgeTarget ageTarget = new AgeTarget()
{
FromAge = int.Parse(age_min),
ToAge = int.Parse(adGroupsReader.Current.targeting["age_max"])
};
ad.Targets.Add(ageTarget);
}
List <object> genders = null;
if (((Dictionary <string, object>)adGroupsReader.Current.targeting).ContainsKey("genders"))
{
genders = adGroupsReader.Current.targeting["genders"];
}
if (genders != null)
{
foreach (object gender in genders)
{
GenderTarget genderTarget = new GenderTarget();
if (gender.ToString() == "1")
{
genderTarget.Gender = Gender.Male;
}
else if (gender.ToString() == "2")
{
genderTarget.Gender = Gender.Female;
}
else
{
genderTarget.Gender = Gender.Unspecified;
}
genderTarget.OriginalID = gender.ToString();
ad.Targets.Add(genderTarget);
}
}
if (adGroupsReader.Current.creative_ids != null)
{
foreach (string creative in adGroupsReader.Current.creative_ids)
{
if (!adsBycreatives.ContainsKey(creative))
{
adsBycreatives.Add(creative, new List <Ad>());
}
adsBycreatives[creative].Add(ad);
}
}
ads.Add(ad.OriginalID, ad);
}
}
}
#endregion
}
}
//******************************************************************************************************************************************************
#endregion adGroups And Targeting
//GetAdGroupStats
#region for validation
foreach (var measure in this.ImportManager.Measures)
{
if (measure.Value.Options.HasFlag(MeasureOptions.ValidationRequired))
{
if (!currentOutput.Checksum.ContainsKey(measure.Key))
{
currentOutput.Checksum.Add(measure.Key, 0); //TODO : SHOULD BE NULL BUT SINCE CAN'T ADD NULLABLE ...TEMP
}
}
}
//**************************************************************************
#endregion
Dictionary <string, List <Dictionary <string, object> > > conversion_data = new Dictionary <string, List <Dictionary <string, object> > >();
if (filesByType.ContainsKey(Consts.FileTypes.ConversionsStats))
{
#region Conversions
List <string> conversionFiles = filesByType[Consts.FileTypes.ConversionsStats];
foreach (string conversionFile in conversionFiles)
{
DeliveryFile creativeFile = Delivery.Files[conversionFile];
var conversionCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (conversionCreativesReader)
{
this.Mappings.OnFieldRequired = field => conversionCreativesReader.Current[field];
while (conversionCreativesReader.Read())
{
if (((Edge.Data.Pipeline.DynamicDictionaryObject)(conversionCreativesReader.Current)).Values == null)
{
break; // in case of empty data (e.g. data [] )
}
if (!conversion_data.ContainsKey(conversionCreativesReader.Current.adgroup_id))
{
Dictionary <string, string> adConversionData = new Dictionary <string, string>();
//string TotalConversionsManyPerClick, Leads, Signups, Purchases;
List <object> actions = conversionCreativesReader.Current.actions;
if (actions.Count > 0)
{
List <Dictionary <string, object> > action_list = actions.Select(s => (Dictionary <string, object>)s).ToList();
List <Dictionary <string, object> > action_items = action_list.Where(dict => dict.ContainsKey("action_type") && dict.ContainsKey("value") &&
(
dict["action_type"].ToString().Equals("offsite_conversion") ||
dict["action_type"].ToString().Equals("offsite_conversion.lead") ||
dict["action_type"].ToString().Equals("offsite_conversion.registration") ||
dict["action_type"].ToString().Equals("offsite_conversion.add_to_cart") ||
dict["action_type"].ToString().Equals("offsite_conversion.checkout") ||
dict["action_type"].ToString().Equals("offsite_conversion.key_page_view") ||
dict["action_type"].ToString().Equals("offsite_conversion.other")
)
).ToList();
if (action_items.Count > 0)
{
conversion_data.Add(conversionCreativesReader.Current.adgroup_id, action_items);
}
}
}
} //End of While read
} // End of Using reader
} //End Foreach conversion file
#endregion
} //End if contains conversion file
#region AdGroupStats start new import session
List <string> adGroupStatsFiles = filesByType[Consts.FileTypes.AdGroupStats];
foreach (var adGroupStat in adGroupStatsFiles)
{
DeliveryFile adGroupStats = this.Delivery.Files[adGroupStat];
var adGroupStatsReader = new JsonDynamicReader(adGroupStats.OpenContents(), "$.data[*].*");
using (adGroupStatsReader)
{
while (adGroupStatsReader.Read())
{
#region Create Metrics
AdMetricsUnit adMetricsUnit = new AdMetricsUnit();
adMetricsUnit.Output = currentOutput;
adMetricsUnit.MeasureValues = new Dictionary <Measure, double>();
Ad tempAd;
try
{
var x = adGroupStatsReader.Current.adgroup_id;
}
catch (Exception)
{
continue;
}
if (adGroupStatsReader.Current.adgroup_id != null)
{
adStatIds[adGroupStatsReader.Current.adgroup_id] = adGroupStatsReader.Current.adgroup_id;
if (ads.TryGetValue(adGroupStatsReader.Current.adgroup_id, out tempAd))
{
adMetricsUnit.Ad = tempAd;
//adMetricsUnit.PeriodStart = this.Delivery.TimePeriodDefinition.Start.ToDateTime();
//adMetricsUnit.PeriodEnd = this.Delivery.TimePeriodDefinition.End.ToDateTime();
// Common and Facebook specific meausures
/* Sets totals for validations */
if (currentOutput.Checksum.ContainsKey(Measure.Common.Clicks))
{
currentOutput.Checksum[Measure.Common.Clicks] += Convert.ToDouble(adGroupStatsReader.Current.clicks);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Impressions))
{
currentOutput.Checksum[Measure.Common.Impressions] += Convert.ToDouble(adGroupStatsReader.Current.impressions);
}
if (currentOutput.Checksum.ContainsKey(Measure.Common.Cost))
{
currentOutput.Checksum[Measure.Common.Cost] += Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d;
}
/* Sets measures values */
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Clicks], Convert.ToInt64(adGroupStatsReader.Current.clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueClicks], Convert.ToInt64(adGroupStatsReader.Current.unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Impressions], Convert.ToInt64(adGroupStatsReader.Current.impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.UniqueImpressions], Convert.ToInt64(adGroupStatsReader.Current.unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[Measure.Common.Cost], Convert.ToDouble(Convert.ToDouble(adGroupStatsReader.Current.spent) / 100d));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialImpressions], double.Parse(adGroupStatsReader.Current.social_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueImpressions], double.Parse(adGroupStatsReader.Current.social_unique_impressions));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialClicks], double.Parse(adGroupStatsReader.Current.social_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialUniqueClicks], double.Parse(adGroupStatsReader.Current.social_unique_clicks));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.SocialCost], Convert.ToDouble(adGroupStatsReader.Current.social_spent) / 100d);
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Actions], 0);
//adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Connections], double.Parse(adGroupStatsReader.Current.connections));
//Setting conversions from conversion files data
List <Dictionary <string, object> > adgroup_conversion_data;
if (conversion_data.TryGetValue(adGroupStatsReader.Current.adgroup_id, out adgroup_conversion_data))
{
var TotalConversionsManyPerClick = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion") select element["value"];
var Leads = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.lead") select element["value"];
var Signups = from element in adgroup_conversion_data where element["action_type"].Equals("offsite_conversion.registration") select element["value"];
//TO DO : Get field from Configuration
var Purchases = from element in adgroup_conversion_data where element["action_type"].Equals(this.Delivery.Parameters["API_Purchases_Field_Name"]) select element["value"];
//add values to metrics unit
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.TotalConversionsManyPerClick], Convert.ToDouble(TotalConversionsManyPerClick.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Leads], Convert.ToDouble(Leads.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Signups], Convert.ToDouble(Signups.DefaultIfEmpty("0").FirstOrDefault()));
adMetricsUnit.MeasureValues.Add(this.ImportManager.Measures[MeasureNames.Purchases], Convert.ToDouble(Purchases.DefaultIfEmpty("0").FirstOrDefault()));
}
this.ImportManager.ImportMetrics(adMetricsUnit);
}
else
{
warningsStr.AppendLine(string.Format("Ad {0} does not exist in the stats report delivery id: {1}", adGroupStatsReader.Current.id, this.Delivery.DeliveryID));
}
}
else
{
warningsStr.AppendLine("adGroupStatsReader.Current.id=null");
}
#endregion
}
}
}
#endregion AdGroupStats start new import session
#region Creatives
List <string> creativeFiles = filesByType[Consts.FileTypes.Creatives];
Dictionary <string, string> usedCreatives = new Dictionary <string, string>();
foreach (string creative in creativeFiles)
{
DeliveryFile creativeFile = Delivery.Files[creative];
var adGroupCreativesReader = new JsonDynamicReader(creativeFile.OpenContents(), "$.data[*].*");
using (adGroupCreativesReader)
{
//this.Mappings.OnFieldRequired = field => if((field == "object_url" && adGroupCreativesReader.Current[field] != null) || field != "object_url")adGroupCreativesReader.Current[field];
this.Mappings.OnFieldRequired = field => adGroupCreativesReader.Current[field];
while (adGroupCreativesReader.Read())
{
List <Ad> adsByCreativeID = null;
if (adsBycreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
if (!usedCreatives.ContainsKey(adGroupCreativesReader.Current.id))
{
usedCreatives.Add(adGroupCreativesReader.Current.id, adGroupCreativesReader.Current.id);
adsByCreativeID = adsBycreatives[adGroupCreativesReader.Current.id];
}
}
if (adsByCreativeID != null)
{
foreach (Ad ad in adsByCreativeID)
{
if (!adStatIds.ContainsKey(ad.OriginalID))
{
continue;
}
ad.Creatives = new List <Creative>();
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_type))
{
string objectType = adGroupCreativesReader.Current.object_type;
switch (objectType.ToUpper())
{
case "SHARE":
{
#region Ads Type SHARE
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_story_id))
{
Dictionary <string, string> shareCreativeData;
string object_story_id = adGroupCreativesReader.Current.object_story_id;
if (storyIds.ContainsKey(object_story_id))
{
shareCreativeData = storyIds[object_story_id];
}
else
{
var accessToken = this.Instance.Configuration.Options[FacebookConfigurationOptions.Auth_AccessToken];
shareCreativeData = GetShareCreativeData(object_story_id, accessToken);
}
ad.DestinationUrl = shareCreativeData["link"];
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
var trackersReadCommands = from n in this.Mappings.Objects[typeof(Ad)].ReadCommands
where n.Field.Equals("link_url")
select n;
foreach (var command in trackersReadCommands)
{
string trackerValue = ApplyRegex(ad.DestinationUrl, command.RegexPattern);
if (!String.IsNullOrEmpty(trackerValue))
{
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].Value = trackerValue;
ad.Segments[this.ImportManager.SegmentTypes[Segment.Common.Tracker]].OriginalID = trackerValue;
}
}
}
ad.Creatives.Add(GetTextCreative(shareCreativeData["text"], adGroupCreativesReader));
ad.Creatives.Add(GetBodyCreative(shareCreativeData["description"], adGroupCreativesReader));
ad.Creatives.Add(GetImageCreative(shareCreativeData["picture"], adGroupCreativesReader));
}
#endregion
break;
}
case "DOMAIN":
{
#region Ads Type DOMAIN
if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.object_url))
{
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields))
{
ad.DestinationUrl = adGroupCreativesReader.Current.object_url;
}
}
else if (!string.IsNullOrEmpty(adGroupCreativesReader.Current.link_url))
{
ad.DestinationUrl = adGroupCreativesReader.Current.link_url;
}
else
{
ad.DestinationUrl = "UnKnown Url";
}
/*Get Data from Mapping E.g Tracker*/
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
if (adGroupCreativesReader.Current.image_url != null)
{
CreateImageCreatives(ad, adGroupCreativesReader);
}
ad.Creatives.Add(GetTextCreative(adGroupCreativesReader));
ad.Creatives.Add(GetBodyCreative(adGroupCreativesReader));
#endregion
break;
}
default:
{
#region Ads Type not handeled
string adtype = objectType.ToUpper();
ad.DestinationUrl = adtype + " Ad Type";
/*Get Data from Mapping E.g Tracker*/
if (this.Mappings != null && this.Mappings.Objects.ContainsKey(typeof(Ad)))
{
this.Mappings.Objects[typeof(Ad)].Apply(ad);
}
TextCreative unhandeledTextAd_title = new TextCreative()
{
OriginalID = ad.OriginalID,
TextType = TextCreativeType.Title,
Text = adtype + " Ad Type"
};
ad.Creatives.Add(unhandeledTextAd_title);
#endregion
break;
}
}
if (!insertedAds.ContainsKey(ad.OriginalID))
{
insertedAds[ad.OriginalID] = ad.OriginalID;
this.ImportManager.ImportAd(ad);
}
}
}
}
//TODO: REPORT PROGRESS 2 ReportProgress(PROGRESS)
}
}
#endregion
}
// End foreach creative
currentOutput.Status = DeliveryOutputStatus.Imported;
this.ImportManager.EndImport();
if (!string.IsNullOrEmpty(warningsStr.ToString()))
{
Log.Write(warningsStr.ToString(), LogMessageType.Warning);
}
}
return(Core.Services.ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
#region Init General
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
// This is for finding conflicting services
this.Delivery.Signature = Delivery.CreateSignature(String.Format("facebook-[{0}]-[{1}]-[{2}]",
this.Instance.AccountID,
this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString(),
this.TargetPeriod.ToAbsolute()));
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.ImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Common.Importing.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
// ...............................
// Now that we have a new delivery, start adding values
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
OriginalID = this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString()
};
this.Delivery.TargetPeriod = this.TargetPeriod;
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = 6
};
this.Delivery.TargetLocationDirectory = Instance.Configuration.Options["DeliveryFilesDir"];
if (string.IsNullOrEmpty(this.Delivery.TargetLocationDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
// Copy some options as delivery parameters
var configOptionsToCopyToDelivery = new string[] {
FacebookConfigurationOptions.Account_ID,
FacebookConfigurationOptions.Account_Name,
FacebookConfigurationOptions.Auth_ApiKey,
FacebookConfigurationOptions.Auth_AppSecret,
FacebookConfigurationOptions.Auth_SessionKey,
FacebookConfigurationOptions.Auth_SessionSecret,
FacebookConfigurationOptions.Auth_RedirectUri,
FacebookConfigurationOptions.Auth_AuthenticationUrl
};
foreach (string option in configOptionsToCopyToDelivery)
{
this.Delivery.Parameters[option] = this.Instance.Configuration.Options[option];
}
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]))
{
throw new Exception("facebook base url must be configured!");
}
_baseAddress = new Uri(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]);
this.ReportProgress(0.2);
#endregion
#region Init Delivery Files
Dictionary <string, string> methodParams = new Dictionary <string, string>();
string methodUrl;
DeliveryFile deliveryFile = new DeliveryFile();
#region adgroupstats
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupStats;
methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TargetPeriod.Start.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TargetPeriod.End.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupStats);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.AdGroupStats.ToString()));
this.Delivery.Files.Add(deliveryFile);
#endregion
this.ReportProgress(0.4);
#region adgroup
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroup;
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroups);
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.AdGroups);
this.Delivery.Files.Add(deliveryFile);
#endregion
this.ReportProgress(0.6);
#region Campaigns
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Campaigns;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaigns);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Campaigns);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region Creatives
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Creatives;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupCreatives);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Creatives);
this.Delivery.Files.Add(deliveryFile);
#endregion
//#region AdGroupTargeting
//deliveryFile = new DeliveryFile();
//deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupTargeting;
//methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupTargeting);
//deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
//deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Data);
//this.Delivery.Files.Add(deliveryFile);
//#endregion
#endregion
this.ReportProgress(0.9);
this.Delivery.Save();
this.ReportProgress(1);
return(ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
this.Delivery = this.NewDelivery(); // setup delivery
this.Delivery.FileDirectory = Instance.Configuration.Options[Edge.Data.Pipeline.Services.Const.DeliveryServiceConfigurationOptions.FileDirectory];
this.Delivery.Account = new Account()
{
ID = 0
};
string fileName = string.Empty;
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["YahhoApiURL"]))
{
this.Delivery.Parameters.Add("YahhoApiURL", this.Instance.Configuration.Options["YahhoApiURL"]);
}
//FileName
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["FileName"]))
{
throw new Exception("Missing Configuration Param , FileName");
}
else
{
fileName = this.Instance.Configuration.Options["FileName"];
}
//CrossRateSymbols
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["CrossRateSymbols"]))
{
throw new Exception("Missing Configuration Param , CrossRateSymbols");
}
else
{
this.Delivery.Parameters.Add("CrossRateSymbols", this.Instance.Configuration.Options["CrossRateSymbols"]);
}
//UserName
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["UserName"]))
{
throw new Exception("Missing Configuration Param , UserName");
}
else
{
this.Delivery.Parameters.Add("UserName", this.Instance.Configuration.Options["UserName"]);
}
//UserPassword
if (String.IsNullOrEmpty(this.Instance.Configuration.Options["UserPassword"]))
{
throw new Exception("Missing Configuration Param , UserPassword");
}
else
{
this.Delivery.Parameters.Add("UserPassword", this.Instance.Configuration.Options["UserPassword"]);
}
DeliveryFile _file = new DeliveryFile()
{
Name = fileName
};
_file.SourceUrl = Instance.Configuration.Options["SourceUrl"]; // "http://www.xignite.com/xCurrencies.asmx";
_file.Parameters.Add("Content-Type", "text/xml; charset=utf-8");
_file.Parameters.Add("SOAPAction", Instance.Configuration.Options["SOAPAction"]);
_file.Parameters.Add("SoapMethod", Instance.Configuration.Options["SoapMethod"]);
_file.Parameters.Add("Body", GetSoapEnvelope(
this.Instance.Configuration.Options["UserName"].ToString(),
this.Instance.Configuration.Options["UserPassword"].ToString(),
this.Instance.Configuration.Options["SoapMethod"].ToString(),
this.Instance.Configuration.Options["CrossRateSymbols"].ToString(),
string.Empty,
this.Delivery.TimePeriodStart.ToString("MM/dd/yyyy")
));
_file.Parameters.Add("XML.IsAttribute", Instance.Configuration.Options["XML.IsAttribute"]);
_file.Parameters.Add("XML.Path", Instance.Configuration.Options["XML.Path"]);
this.Delivery.Files.Add(_file);
//Set Output
foreach (string crossRate in this.Delivery.Parameters["CrossRateSymbols"].ToString().Split(','))
{
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("[{0}]-[{1}]",
this.TimePeriod.ToAbsolute(),
crossRate
)),
Account = new Data.Objects.Account()
{
ID = 0
},
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd
}
);
}
// Create an import manager that will handle rollback, if necessary
//CurrencyImportManager importManager = new CurrencyImportManager(this.Instance.InstanceID,null);
//TO DO: Add rollback
// will use ConflictBehavior configuration option to abort or rollback if any conflicts occur
//this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
this.Delivery.Save();
return(Core.Services.ServiceOutcome.Success);
}
protected override ServiceOutcome DoPipelineWork()
{
#region Init General
// ...............................
// SETUP
this.Delivery = NewDelivery();
// This is for finding conflicting services
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
OriginalID = this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString()
};
this.Delivery.TimePeriodDefinition = this.TimePeriod;
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = 6
};
this.Delivery.Outputs.Add(new DeliveryOutput()
{
Signature = Delivery.CreateSignature(String.Format("facebook-[{0}]-[{1}]-[{2}]",
this.Instance.AccountID,
this.Instance.Configuration.Options[FacebookConfigurationOptions.Account_ID].ToString(),
this.Delivery.TimePeriodDefinition.ToAbsolute())),
TimePeriodStart = Delivery.TimePeriodStart,
TimePeriodEnd = Delivery.TimePeriodEnd,
Account = this.Delivery.Account,
Channel = this.Delivery.Channel
});
// Now that we have a new delivery, start adding values
this.Delivery.FileDirectory = Instance.Configuration.Options[Const.DeliveryServiceConfigurationOptions.FileDirectory];
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new MetricsImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Metrics.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
if (string.IsNullOrEmpty(this.Delivery.FileDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]))
{
throw new Exception("facebook base url must be configured!");
}
_baseAddress = new Uri(this.Instance.Configuration.Options[FacebookConfigurationOptions.BaseServiceAddress]);
//this.ReportProgress(0.2);
#endregion
#region Init Delivery Files
Dictionary <string, string> methodParams = new Dictionary <string, string>();
string methodUrl;
DeliveryFile deliveryFile = new DeliveryFile();
#region adgroupstats
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupStats;
methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TimePeriod.Start.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TimePeriod.End.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupStats);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.AdGroupStats.ToString()));
this.Delivery.Files.Add(deliveryFile);
#endregion
//this.ReportProgress(0.4);
#region Conversions
//======================================================================================
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.ConversionsStats;
methodParams.Add(Consts.FacebookMethodsParams.StartTime, ConvertToFacebookDateTime(TimePeriod.Start.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.EndTime, ConvertToFacebookDateTime(TimePeriod.End.ToDateTime()));
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
methodParams.Add(Consts.FacebookMethodsParams.StatsMode, "with_delivery");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetConversionStats);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Enum.Parse(typeof(Consts.FileTypes), Consts.FileTypes.ConversionsStats.ToString()));
this.Delivery.Files.Add(deliveryFile);
//======================================================================================
#endregion Conversions
//this.ReportProgress(0.4);
#region adgroup
/*
* Summary
* An ad group contains the data necessary for an ad, such as bid type, bid info,
* targeting data, creative elements, and campaign information. Each ad group is
* associated with a campaign and all ad groups in a campaign have the same daily
* or lifetime budget and schedule.
* */
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.AdGroup;
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroups);
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.AdGroupFields].ToString());
}
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.AdGroups);
this.Delivery.Files.Add(deliveryFile);
#endregion
//this.ReportProgress(0.6);
#region AdSet- Formally Campaigns
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Campaigns;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.CampaignFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.CampaignFields].ToString());
}
// methodParams.Add("redownload", "true");
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaignsAdSets);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Campaigns);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region Campaigns - New Structure
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.CampaignGroups;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.CampaignGroupsFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.CampaignGroupsFields].ToString());
}
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetCampaignsGroups);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.CampaignGroups);
this.Delivery.Files.Add(deliveryFile);
#endregion
#region Creatives
deliveryFile = new DeliveryFile();
deliveryFile.Name = Consts.DeliveryFilesNames.Creatives;
methodParams.Add(Consts.FacebookMethodsParams.IncludeDeleted, "true");
if (Instance.Configuration.Options.ContainsKey(FacebookConfigurationOptions.AdGroupCreativeFields))
{
methodParams.Add(Consts.FacebookMethodsParams.Fields, Instance.Configuration.Options[FacebookConfigurationOptions.AdGroupCreativeFields].ToString());
}
methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupCreatives);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Length);
deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileType, Consts.FileTypes.Creatives);
this.Delivery.Files.Add(deliveryFile);
#endregion
//#region AdGroupTargeting
//deliveryFile = new DeliveryFile();
//deliveryFile.Name = Consts.DeliveryFilesNames.AdGroupTargeting;
//methodUrl = string.Format("act_{0}/{1}", Delivery.Account.OriginalID, Consts.FacebookMethodsNames.GetAdGroupTargeting);
//deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.Url, GetMethodUrl(methodUrl, methodParams));
//deliveryFile.Parameters.Add(Consts.DeliveryFileParameters.FileSubType, Consts.FileSubType.Data);
//this.Delivery.Files.Add(deliveryFile);
#endregion
//this.ReportProgress(0.9);
this.Delivery.Save();
this.ReportProgress(1);
return(ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
string baseAddress;
#region Init General
// ...............................
// SETUP
this.Delivery = this.NewDelivery();
// This is for finding conflicting services
this.Delivery.Signature = Delivery.CreateSignature(String.Format("BackOffice-[{0}]-[{1}]",
this.Instance.AccountID,
this.TargetPeriod.ToAbsolute()));
// Create an import manager that will handle rollback, if necessary
AdMetricsImportManager importManager = new AdMetricsImportManager(this.Instance.InstanceID, new Edge.Data.Pipeline.Common.Importing.ImportManagerOptions()
{
SqlRollbackCommand = Instance.Configuration.Options[Edge.Data.Pipeline.Common.Importing.Consts.AppSettings.SqlRollbackCommand]
});
// Apply the delivery (will use ConflictBehavior configuration option to abort or rollback if any conflicts occur)
this.HandleConflicts(importManager, DeliveryConflictBehavior.Abort);
// ...............................
// Now that we have a new delivery, start adding values
this.Delivery.Account = new Data.Objects.Account()
{
ID = this.Instance.AccountID,
};
this.Delivery.TargetPeriod = this.TargetPeriod;
this.Delivery.Channel = new Data.Objects.Channel()
{
ID = -1
};
this.Delivery.TargetLocationDirectory = Instance.Configuration.Options["DeliveryFilesDir"];
if (string.IsNullOrEmpty(this.Delivery.TargetLocationDirectory))
{
throw new Exception("Delivery.TargetLocationDirectory must be configured in configuration file (DeliveryFilesDir)");
}
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress]))
{
throw new Exception("base url must be configured!");
}
baseAddress = this.Instance.Configuration.Options[BoConfigurationOptions.BaseServiceAddress];
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.UserName]))
{
throw new Exception("base url must be configured!");
}
this.Delivery.Parameters["UserName"] = this.Instance.Configuration.Options[BoConfigurationOptions.UserName];
if (string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.Password]))
{
throw new Exception("base url must be configured!");
}
this.Delivery.Parameters["Password"] = this.Instance.Configuration.Options[BoConfigurationOptions.Password];
int utcOffset = 0;
if (!string.IsNullOrEmpty(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]))
{
utcOffset = int.Parse(this.Instance.Configuration.Options[BoConfigurationOptions.UtcOffset]);
}
this.ReportProgress(0.2);
#endregion
#region DeliveryFile
Dictionary <string, string> UrlParams = new Dictionary <string, string>();
DeliveryFile boFile = new DeliveryFile();
boFile.Parameters[BoConfigurationOptions.BO_XPath_Trackers] = Instance.Configuration.Options[BoConfigurationOptions.BO_XPath_Trackers];
boFile.Name = BoConfigurationOptions.BoFileName;
UrlParams.Add("from", utcOffset == 0 ? Delivery.TargetPeriod.Start.ToDateTime().ToString("yyyy-MM-ddTHH:MMZ") : ConvertToTimeZone(utcOffset, TargetPeriod.Start.ToDateTime()));
UrlParams.Add("to", utcOffset == 0 ? Delivery.TargetPeriod.End.ToDateTime().ToString("yyyy-MM-ddTHH:MMZ") : ConvertToTimeZone(utcOffset, TargetPeriod.Start.ToDateTime()));
UrlParams.Add("vendor", this.Delivery.Parameters["UserName"].ToString());
UrlParams.Add("Password", this.Delivery.Parameters["Password"].ToString());
boFile.SourceUrl = CreateUrl(UrlParams, baseAddress.ToString());
this.Delivery.Files.Add(boFile);
this.Delivery.Save();
#endregion
return(Core.Services.ServiceOutcome.Success);
}
protected override Core.Services.ServiceOutcome DoPipelineWork()
{
// Setup/defaults/configuration/etc.
// ------------------------------------------
string checksumThreshold = Configuration.Parameters.Get <T>(Consts.ConfigurationOptions.ChecksumTheshold);
var importManagerOptions = new MetricsDeliveryManagerOptions()
{
SqlTransformCommand = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlTransformCommand),
SqlStageCommand = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlStageCommand),
SqlRollbackCommand = Configuration.Parameters.Get <T>(Consts.AppSettings.SqlRollbackCommand),
ChecksumThreshold = checksumThreshold == null ? 0.01 : double.Parse(checksumThreshold)
};
string fileName;
if (!this.Configuration.Options.TryGetValue(Const.DeliveryServiceConfigurationOptions.DeliveryFileName, out fileName))
{
throw new ConfigurationException(String.Format("{0} is missing in the service configuration options.", Const.DeliveryServiceConfigurationOptions.DeliveryFileName));
}
DeliveryFile file = this.Delivery.Files[fileName];
if (file == null)
{
throw new Exception(String.Format("Could not find delivery file '{0}' in the delivery.", fileName));
}
FileCompression compression;
string compressionOption;
if (this.Configuration.Options.TryGetValue(Const.DeliveryServiceConfigurationOptions.Compression, out compressionOption))
{
if (!Enum.TryParse <FileCompression>(compressionOption, out compression))
{
throw new ConfigurationException(String.Format("Invalid compression type '{0}'.", compressionOption));
}
}
else
{
compression = FileCompression.None;
}
// Create format processor from configuration
string adapterTypeName = Configuration.GetOption(Consts.ConfigurationOptions.ReaderAdapterType);
Type readerAdapterType = Type.GetType(adapterTypeName, true);
this.ReaderAdapter = (ReaderAdapter)Activator.CreateInstance(readerAdapterType);
this.Mappings.OnFieldRequired = this.ReaderAdapter.GetField;
LoadConfiguration();
// ------------------------------------------
using (var stream = file.OpenContents(compression: compression))
{
using (this.ReaderAdapter)
{
this.ReaderAdapter.Init(stream, Configuration);
using (this.ImportManager = CreateImportManager(InstanceID, importManagerOptions))
{
this.ImportManager.BeginImport(this.Delivery);
bool readSuccess = false;
while (this.ReaderAdapter.Reader.Read())
{
readSuccess = true;
OnRead();
}
if (!readSuccess)
{
Edge.Core.Utilities.Log("Could Not read data from file!, check file mapping and configuration", Core.Utilities.LogMessageType.Warning);
}
this.ImportManager.EndImport();
}
}
}
return(Core.Services.ServiceOutcome.Success);
}
