public static List <ExtractResult> Extract(ExtractRequest request)
{
if (NodeConfigurationSection.Standalone)
{
var result = RuiJiExtractor.Extract(request);
return(result);
}
else
{
var proxyUrl = ProxyManager.Instance.Elect(NodeProxyTypeEnum.FEEDPROXY);
if (string.IsNullOrEmpty(proxyUrl))
{
throw new Exception("no available Extractor proxy servers");
}
proxyUrl = IPHelper.FixLocalUrl(proxyUrl);
var client = new RestClient("http://" + proxyUrl);
var restRequest = new RestRequest("api/ep/extract");
restRequest.Method = Method.POST;
restRequest.JsonSerializer = new NewtonJsonSerializer();
var json = JsonConvert.SerializeObject(request);
restRequest.AddJsonBody(json);
restRequest.Timeout = 15000;
var restResponse = client.Execute(restRequest);
var response = JsonConvert.DeserializeObject <List <ExtractResult> >(restResponse.Content);
return(response);
}
}
private bool DoExtractTaskWithPod(Bot bot, Pod pod)
{
// Bring items to the randomly selected first fitting station
foreach (var oStation in Instance.OutputStations.OrderBy(s => GetOrderValue(s, bot)))
{
// Search for requests matching the items of the pod
List <ExtractRequest> fittingRequests = GetPossibleRequests(pod, oStation, PodSelectionExtractRequestFilteringMode.AssignedOnly);
if (fittingRequests.Any())
{
ExtractRequest oldestRequest = fittingRequests.OrderBy(o => o.Order.TimeStamp).First();
// Simply execute the next task with the pod
EnqueueExtract(
bot, // The bot itself
oStation, // The random station
pod, // Keep the pod
new List <ExtractRequest> {
oldestRequest
}); // The first requests to serve
// Finished search for next task
return(true);
}
}
// No fitting request
return(false);
}
public static void Run()
{
var configuration = new Configuration(Common.MyAppSid, Common.MyAppKey);
var apiInstance = new PagesApi(configuration);
try
{
var fileInfo = new FileInfo
{
FilePath = "WordProcessing/sample-10-pages.docx"
};
var options = new ExtractOptions
{
FileInfo = fileInfo,
OutputPath = "Output/extract-pages-by-range.docx",
StartPageNumber = 1,
EndPageNumber = 10,
RangeMode = PageOptions.RangeModeEnum.EvenPages
};
var request = new ExtractRequest(options);
var response = apiInstance.Extract(request);
Console.WriteLine("Output file path: " + response.Path);
}
catch (Exception e)
{
Console.WriteLine("Exception while calling api: " + e.Message);
}
}
public void ExtractApiTest_Tag()
{
var options = new ExtractOptions
{
FileInfo = TestFiles.Docx.ToFileInfo(),
SearchCriteria = new SearchCriteria
{
TagOptions = new TagOptions
{
ExactTag = new Tag
{
Name = "Created",
Category = "Time"
}
}
}
};
var request = new ExtractRequest(options);
var result = MetadataApi.Extract(request);
Assert.IsNotNull(result);
Assert.IsNotEmpty(result.Properties);
Assert.IsTrue(result.Properties.Any(x => string.Equals(x.Name, "CreateTime")));
}
public void ExtractApiTest_PropertyNameExactPhrase()
{
var options = new ExtractOptions
{
FileInfo = TestFiles.Docx.ToFileInfo(),
SearchCriteria = new SearchCriteria
{
NameOptions = new NameOptions
{
Value = "MimeType",
MatchOptions = new MatchOptions
{
ExactPhrase = true
}
}
}
};
var request = new ExtractRequest(options);
var result = MetadataApi.Extract(request);
Assert.IsNotNull(result);
Assert.IsNotEmpty(result.Properties);
Assert.IsTrue(result.Properties.Any(x => x.Name.Equals("MimeType", StringComparison.OrdinalIgnoreCase)));
}
/// <summary>
/// Finds the best delivery task for the specified pod.
/// Sets <code>bestDeliveryRequest</code> to null if none found, otherwise <code>bestDeliveryRequest</code> and <code>bestTimeForDeliveryRequest</code> are initialized.
/// </summary>
/// <param name="bot">The bot to consider.</param>
/// <param name="pod">Pod to take.</param>
/// <param name="podLocation">Current location of the pod.</param>
/// <param name="bestExtractTask">The best extract task set by this method.</param>
/// <param name="bestTimeForExtract">The time of the best extract set by this method.</param>
void GetBestTaskForPod(Bot bot, Pod pod, Waypoint podLocation,
out ExtractRequest bestExtractTask, out double bestTimeForExtract)
{
bestExtractTask = null;
bestTimeForExtract = 0.0;
bestExtractTask = null;
if (pod == null || podLocation == null)
{
return;
}
bestTimeForExtract = double.PositiveInfinity;
// Check all tasks
foreach (var delivery in Instance.ResourceManager.AvailableAndAssignedExtractRequests)
{
// If it has the item
if (pod.IsContained(delivery.Item))
{
// See how long it would take to get to the output-station
// Choose the worst of delivering or waiting
Waypoint sw = delivery.Station.Waypoint;
double time = Math.Max(Estimators.EstimateTravelTimeEuclid(bot, podLocation, sw), Estimators.EstimateOutputStationWaitTime(bot, sw));
// If it's the best, then use it
if (time < bestTimeForExtract)
{
bestExtractTask = delivery;
bestTimeForExtract = time;
}
}
}
}
public void ExtractApiTest_PropertyNameRegex()
{
var options = new ExtractOptions
{
FileInfo = TestFiles.Docx.ToFileInfo(),
SearchCriteria = new SearchCriteria
{
NameOptions = new NameOptions
{
Value = "^dc:.*",
MatchOptions = new MatchOptions
{
IsRegex = true
}
}
}
};
var request = new ExtractRequest(options);
var result = MetadataApi.Extract(request);
Assert.IsNotNull(result);
Assert.IsNotEmpty(result.Properties);
Assert.IsTrue(result.Properties.Any(x => x.Name.StartsWith("dc:", StringComparison.OrdinalIgnoreCase)));
}
public static void Run()
{
var configuration = new Configuration(Common.MyAppSid, Common.MyAppKey);
var apiInstance = new PagesApi(configuration);
try
{
var fileInfo = new FileInfo
{
FilePath = "WordProcessing/sample-10-pages.docx"
};
var options = new ExtractOptions
{
FileInfo = fileInfo,
OutputPath = "Output/extract-pages-by-numbers.docx",
Pages = new List <int?> {
2, 4, 7
}
};
var request = new ExtractRequest(options);
var response = apiInstance.Extract(request);
Console.WriteLine("Output file path: " + response.Path);
}
catch (Exception e)
{
Console.WriteLine("Exception while calling api: " + e.Message);
}
}
public static ExtractResult Extract(ExtractRequest request)
{
var proxyUrl = ProxyManager.Instance.Elect(ProxyTypeEnum.Extracter);
if (string.IsNullOrEmpty(proxyUrl))
{
throw new Exception("no available extracter proxy servers");
}
var client = new RestClient("http://" + proxyUrl);
var restRequest = new RestRequest("api/ep/extract");
restRequest.Method = Method.POST;
restRequest.JsonSerializer = new NewtonJsonSerializer();
var json = JsonConvert.SerializeObject(request);
restRequest.AddJsonBody(json);
restRequest.Timeout = 15000;
var restResponse = client.Execute(restRequest);
var response = JsonConvert.DeserializeObject <ExtractResult>(restResponse.Content);
return(response);
}
// pagenation is ?page=2&per_page=100
/// <summary>
/// the url is like this: https://github.com/andyliuliming/WALinuxAgent.git
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public List <User> Extract(ExtractRequest extractRequest, List <string> user_logins)
{
GithubFeed githubFee = this.Parse(extractRequest.URL);
List <User> users = new List <User>();
RestApiCaller <User> userInfoCaller = new RestApiCaller <User>(ApiFormats.BaseUri);
// 3. extract the commit info ("login")
int leftUserLength = user_logins.Count - extractRequest.StartIndex;
int numberToRetrieve = leftUserLength > extractRequest.Left ? extractRequest.Left : leftUserLength;
for (int i = 0; i < numberToRetrieve; i++)
{
Console.WriteLine("getting the user's info: " + user_logins[extractRequest.StartIndex + i]);
string urlParameters = string.Format(ApiFormats.UserApi, user_logins[extractRequest.StartIndex + i]);
User user = null;
Retry(
() =>
{
user = userInfoCaller.CallApi("get", extractRequest.AccessToken, urlParameters, null);
},
() =>
{
extractRequest.Left--;
}
);
users.Add(user);
}
extractRequest.StartIndex += numberToRetrieve;
return(users);
}
/// <summary>
/// Retrieve document metadata.
/// </summary>
/// <param name="request">Request. <see cref="ExtractRequest" /></param>
/// <returns><see cref="ExtractResult"/></returns>
public ExtractResult Extract(ExtractRequest request)
{
// verify the required parameter 'options' is set
if (request.options == null)
{
throw new ApiException(400, "Missing required parameter 'options' when calling Extract");
}
// create path and map variables
var resourcePath = this.configuration.GetServerUrl() + "/metadata";
resourcePath = Regex
.Replace(resourcePath, "\\*", string.Empty)
.Replace("&", "&")
.Replace("/?", "?");
var postBody = SerializationHelper.Serialize(request.options); // http body (model) parameter
var response = this.apiInvoker.InvokeApi(
resourcePath,
"POST",
postBody,
null,
null);
if (response != null)
{
return((ExtractResult)SerializationHelper.Deserialize(response, typeof(ExtractResult)));
}
return(null);
}
public async Task <string> LoadDataFromGoogleScholar(ExtractRequest extractRequest)
{
BibTeXclient.BaseAddress = new Uri(ProgramAddresses.GoogleScholarWrapperAddress);
return(await MakeARequest("ExtractData/" + JsonSerializer.Serialize(extractRequest), BibTeXclient));
//return File.ReadAllText("./archivoBibtex.json");
}
public async Task <string> LoadDataFromIEEEXplore(ExtractRequest extractRequest)
{
IEEEXclient.BaseAddress = new Uri(ProgramAddresses.IEEEXploreWrapperAddress);
return(await MakeARequest("ExtractData/" + JsonSerializer.Serialize(extractRequest), IEEEXclient));
//return File.ReadAllText("../IEEEXmini.json");
}
public async Task <string> LoadDataFromDBLP(ExtractRequest extractRequest)
{
DBLPclient.BaseAddress = new Uri(ProgramAddresses.DBLPWrapperAddress);
return(await MakeARequest("ExtractData/" + JsonSerializer.Serialize(extractRequest), DBLPclient));
//return File.ReadAllText("../DBLPreal.json");
}
/// <summary>
/// Adds the request to pick the item to the order. This can be used by other components to see the particular requests' status.
/// </summary>
/// <param name="itemDescription">The SKU to add the request for.</param>
/// <param name="request">The particular request to pick the item.</param>
public void AddRequest(ItemDescription itemDescription, ExtractRequest request)
{
if (!_requests.ContainsKey(itemDescription))
{
_requests[itemDescription] = new HashSet <ExtractRequest>();
}
_requests[itemDescription].Add(request);
}
/// <summary>
/// Revokes a reservation of an item.
/// </summary>
/// <param name="item">The type of the item.</param>
/// <param name="extractRequest">The request which should have been done with the item.</param>
internal void UnregisterItem(ItemDescription item, ExtractRequest extractRequest)
{
_itemDescriptionCountAvailable[item]++;
_extractRequestsRegistered.Remove(extractRequest);
extractRequest.Unassign(this);
// Notify instance
Instance.NotifyPodItemUnreserved(this, item, extractRequest);
}
public static void ExtractDataExtension(SoapClient soapClient, string DataExtensionCustomerKey, string FileName)
{
ExtractRequest er = new ExtractRequest();
er.ID = "bb94a04d-9632-4623-be47-daabc3f588a6";
// Always set an StartDate to the value specified
ExtractParameter epOne = new ExtractParameter();
epOne.Name = "StartDate";
epOne.Value = "1/1/1900 1:00:00 AM";
// Always set an StartDate to the value specified
ExtractParameter epTwo = new ExtractParameter();
epTwo.Name = "EndDate";
epTwo.Value = "1/1/1900 1:00:00 AM";
// Always set an _Async to 0
ExtractParameter epThree = new ExtractParameter();
epThree.Name = "_AsyncID";
epThree.Value = "0";
ExtractParameter epFour = new ExtractParameter();
epFour.Name = "OutputFileName";
epFour.Value = FileName;
ExtractParameter epFive = new ExtractParameter();
epFive.Name = "DECustomerKey";
epFive.Value = DataExtensionCustomerKey;
ExtractParameter epSix = new ExtractParameter();
epSix.Name = "HasColumnHeaders";
epSix.Value = "true";
er.Parameters = new ExtractParameter[] { epOne, epTwo, epThree, epFour, epFive, epSix };
string sRequestId;
string sStatus;
ExtractResult[] results;
sStatus = soapClient.Extract(new ExtractRequest[] { er }, out sRequestId, out results);
Console.WriteLine("Status: " + sStatus);
Console.WriteLine("Request ID: " + sRequestId);
foreach (ExtractResult eresult in results)
{
Console.WriteLine("StatusCode: " + eresult.StatusCode);
Console.WriteLine("ErrorCode: " + eresult.ErrorCode);
Console.WriteLine("StatusMessage: " + eresult.StatusMessage);
}
}
/// <summary>
/// Adds another request to this task on-the-fly.
/// </summary>
/// <param name="request">The request that shall also be completed by this task.</param>
public void AddRequest(ExtractRequest request)
{
Instance.ResourceManager.RemoveExtractRequest(request);
if (!ReservedPod.IsContained(request.Item))
{
throw new InvalidOperationException("Cannot add a request for an item that is not available!");
}
ReservedPod.RegisterItem(request.Item, request);
Requests.Add(request);
request.StatInjected = true;
}
public async Task <string> ExtractData(string request)
{
ExtractRequest extractRequest = GetExtractRequest(request);
if (extractRequest == null)
{
return(null);
}
return(await ExtractDataFromIEEEXploreAPIAsync(extractRequest.InitialYear, extractRequest.FinalYear));
}
public string ExtractData(string request)
{
ExtractRequest extractRequest = GetExtractRequest(request);
if (extractRequest == null)
{
return(null);
}
return(ExtractDataFromXmlFile(extractRequest.InitialYear, extractRequest.FinalYear));
}
public static void Run()
{
var configuration = new Configuration(Common.MyAppSid, Common.MyAppKey);
var apiInstance = new MetadataApi(configuration);
try
{
var fileInfo = new FileInfo
{
FilePath = "documents/input.docx",
StorageName = Common.MyStorage
};
var options = new ExtractOptions
{
FileInfo = fileInfo,
SearchCriteria = new SearchCriteria
{
TagOptions = new TagOptions
{
ExactTag = new Tag
{
Name = "Created",
Category = "Time"
}
}
}
};
var request = new ExtractRequest(options);
var response = apiInstance.Extract(request);
foreach (var property in response.Properties)
{
Console.WriteLine($"Property: {property.Name}. Value: {property.Value}");
if (property.Tags == null)
{
continue;
}
foreach (var tag in property.Tags)
{
Console.WriteLine($"Property tag: {tag.Category} {tag.Name} ");
}
}
}
catch (Exception e)
{
Console.WriteLine("Exception while calling MetadataApi: " + e.Message);
}
}
public void ExtractApiTest_DocumentProtectedException()
{
var testFile = TestFiles.PasswordProtected;
var options = new ExtractOptions
{
FileInfo = testFile.ToFileInfo(),
};
var request = new ExtractRequest(options);
var ex = Assert.Throws <ApiException>(() => { MetadataApi.Extract(request); });
Assert.AreEqual($"The specified file '{testFile.FullName}' is protected.", ex.Message);
}
public void ExtractApiTest_FileNotFound()
{
var testFile = TestFiles.NotExist;
var options = new ExtractOptions
{
FileInfo = testFile.ToFileInfo(),
};
var request = new ExtractRequest(options);
var ex = Assert.Throws <ApiException>(() => { MetadataApi.Extract(request); });
Assert.AreEqual($"Can't find file located at '{testFile.FullName}'.", ex.Message);
}
public static void ExtractDataExtension(SoapClient soapClient, string DataExtensionCustomerKey, string FileName)
{
ExtractRequest er = new ExtractRequest();
er.ID = "bb94a04d-9632-4623-be47-daabc3f588a6";
// Always set an StartDate to the value specified
ExtractParameter epOne = new ExtractParameter();
epOne.Name = "StartDate";
epOne.Value = "1/1/1900 1:00:00 AM";
// Always set an StartDate to the value specified
ExtractParameter epTwo = new ExtractParameter();
epTwo.Name = "EndDate";
epTwo.Value = "1/1/1900 1:00:00 AM";
// Always set an _Async to 0
ExtractParameter epThree = new ExtractParameter();
epThree.Name = "_AsyncID";
epThree.Value = "0";
ExtractParameter epFour = new ExtractParameter();
epFour.Name = "OutputFileName";
epFour.Value = FileName;
ExtractParameter epFive = new ExtractParameter();
epFive.Name = "DECustomerKey";
epFive.Value = DataExtensionCustomerKey;
ExtractParameter epSix = new ExtractParameter();
epSix.Name = "HasColumnHeaders";
epSix.Value = "true";
er.Parameters = new ExtractParameter[] {epOne, epTwo, epThree, epFour, epFive, epSix };
string sRequestId;
string sStatus;
ExtractResult[] results;
sStatus = soapClient.Extract(new ExtractRequest[] { er }, out sRequestId, out results);
Console.WriteLine("Status: " + sStatus);
Console.WriteLine("Request ID: " + sRequestId);
foreach (ExtractResult eresult in results)
{
Console.WriteLine("StatusCode: " + eresult.StatusCode);
Console.WriteLine("ErrorCode: " + eresult.ErrorCode);
Console.WriteLine("StatusMessage: " + eresult.StatusMessage);
}
}
public void ExtractApiTest()
{
var options = new ExtractOptions
{
FileInfo = TestFiles.Docx.ToFileInfo()
};
var request = new ExtractRequest(options);
var result = MetadataApi.Extract(request);
Assert.IsNotNull(result.MetadataTree);
Assert.IsNotEmpty(result.MetadataTree.InnerPackages);
Assert.IsTrue(result.MetadataTree.InnerPackages.Any(x => string.Equals(x.PackageName, "FileFormat")));
}
/// <summary>
/// Requests the station to pick the given item for the given order.
/// </summary>
/// <param name="bot">The bot that requests the pick.</param>
/// <param name="request">The request to handle.</param>
public void RequestItemTake(Bot bot, ExtractRequest request)
{
if (bot.Pod != null && _assignedOrders.Contains(request.Order))
{
// Add the request to the list of requests to handle
_requestsBot.Enqueue(bot);
_requestsExtract.Enqueue(request);
}
else
{
// Something went wrong, refuse to handle the request
request.Abort();
bot.WaitUntil(Instance.Controller.CurrentTime + Instance.RefusedRequestPenaltyTime);
}
}
public async Task <ActionResult> Post([FromBody] ExtractRequest request)
{
try
{
var extractedMediaUrl = await _mediaExtractorService.Extract(request.MediaUrl);
return(Ok(new ExtractResponse
{
MediaUrl = extractedMediaUrl
}));
}
catch (Exception e)
{
return(BadRequest(new { error = e.Message }));
}
}
private async Task <LoadAnswer> ProcessLoadRequest(LoadRequest loadRequest)
{
LoadAnswer loadAnswer = new LoadAnswer();
ExtractRequest extractRequest = new ExtractRequest(loadRequest.InitialYear, loadRequest.FinalYear);
if (loadRequest.LoadFromDBLP)
{
(int numberOfResults, List <string> errorList) = await LoadFromSource(
"DBLP",
loadRequest,
extractRequest,
requestsManager.LoadDataFromDBLP,
dBLPExtractor.ExtractData);
loadAnswer.DBLPNumberOfResults = numberOfResults;
loadAnswer.DBLPErrors = errorList;
}
if (loadRequest.LoadFromIEEEXplore)
{
(int numberOfResults, List <string> errorList) = await LoadFromSource(
"IEEEXplore",
loadRequest,
extractRequest,
requestsManager.LoadDataFromIEEEXplore,
iEEEXploreExtractor.ExtractData);
loadAnswer.IEEEXploreNumberOfResults = numberOfResults;
loadAnswer.IEEEXploreErrors = errorList;
}
if (loadRequest.LoadFromGoogleScholar)
{
(int numberOfResults, List <string> errorList) = await LoadFromSource(
"Google Scholar",
loadRequest,
extractRequest,
requestsManager.LoadDataFromGoogleScholar,
bibTeXExtractor.ExtractData);
loadAnswer.GoogleScholarNumberOfResults = numberOfResults;
loadAnswer.GoogleScholarErrors = errorList;
}
return(loadAnswer);
}
/// <summary>
/// Reserves an item that is going to be picked at a station.
/// </summary>
/// <param name="item">The item that is going to be reserved for picking.</param>
/// <param name="extractRequest">The request for which the item shall be reserved.</param>
internal void RegisterItem(ItemDescription item, ExtractRequest extractRequest)
{
// Init, if not done yet
if (_itemDescriptionCountContained == null)
{
InitPodContentInfo();
}
if (_itemDescriptionCountAvailable[item] <= 0)
{
throw new InvalidOperationException("Cannot reserve an item for picking, if there is none left of the kind!");
}
_itemDescriptionCountAvailable[item]--;
_extractRequestsRegistered.Add(extractRequest);
extractRequest.Assign(this);
// Notify instance
Instance.NotifyPodItemReserved(this, item, extractRequest);
}
public HashSet <string> ExtractUserLogin(ExtractRequest extractRequest)
{
GithubFeed githubFee = this.Parse(extractRequest.URL);
HashSet <string> user_logins = new HashSet <string>();
// 1. first get the branches https://api.github.com/repos/torvalds/linux
// https://api.github.com/repos/apache/hadoop
// 2. get the commits from the branches. https://api.github.com/repos/torvalds/linux/commits?page=3&per_page=100
RestApiCaller <List <CommitDetail> > commitInfoCaller = new RestApiCaller <List <CommitDetail> >(ApiFormats.BaseUri);
string repoBaseUri = string.Format(ApiFormats.CommitRelativePathPattern, githubFee.owner, githubFee.repo);
while (extractRequest.Left > 0)
{
Console.WriteLine("getting the page: " + extractRequest.StartPage);
string urlParameters = repoBaseUri + "?page=" + extractRequest.StartPage + "&per_page=" + extractRequest.PerPage;
List <CommitDetail> pagedDetails = null;
Retry(
() =>
{
pagedDetails = commitInfoCaller.CallApi("get", extractRequest.AccessToken, urlParameters, null);
},
() =>
{
extractRequest.Left--;
}
);
if (pagedDetails == null || pagedDetails.Count == 0)
{
break;
}
else
{
for (int i = 0; i < pagedDetails.Count; i++)
{
if (pagedDetails[i].author != null)
{
user_logins.Add(pagedDetails[i].author.login);
}
}
extractRequest.StartPage++;
}
}
return(user_logins);
}
public object TestRule([FromBody] RuleModel rule, bool debug = false)
{
var request = new Request(rule.Url);
request.Method = rule.Method;
request.RunJS = (rule.RunJS == Status.ON);
if (request.RunJS)
{
request.WaitDom = request.WaitDom;
}
var response = Crawler.Request(request);
if (response != null && response.Data != null)
{
var content = response.Data.ToString();
var block = RuiJiBlockParser.ParserBlock(rule.RuiJiExpression);
var r = new ExtractRequest();
r.Content = content;
r.Blocks = new List <ExtractFeatureBlock> {
new ExtractFeatureBlock(block, rule.Feature)
};
var results = Extractor.Extract(r);
var result = results.OrderByDescending(m => m.Metas.Count).FirstOrDefault();
if (result != null && result.Paging != null && result.Paging.Count > 0 && result.Metas != null && result.Metas.ContainsKey("content"))
{
result = PagingExtractor.MergeContent(new Uri(rule.Url), result, block);
}
if (!debug)
{
CrawlTaskFunc.ClearContent(result);
}
return(result);
}
return(new { });
}
public static void Go()
{
var dict_fiidToAuctionID = new Dictionary<int, int>();
{
string sql = "select * from FIBondAuction";
DataSet ds = Singleton<ConnMngr>.Instance.Execute(DBNames.SQLSERVER_SYMMETRYTS, sql);
foreach (DataRow row in ds.Tables[0].Rows)
if(!dict_fiidToAuctionID.ContainsKey(Convert.ToInt32(row["FIID"])))
dict_fiidToAuctionID.Add(Convert.ToInt32(row["FIID"]), Convert.ToInt32(row["FIBondAuctionID"]));
}
var dict_isinToMissingBond = new Dictionary<string, Bond>();
var rss_identifiers = new List<InstrumentIdentifier>();
foreach (var bond in Singleton<Bonds>.Instance.Where(x => x.SymmetryCode.StartsWith("US912828")))
{
if (!dict_fiidToAuctionID.ContainsKey(bond.FIID))
{
dict_isinToMissingBond.Add(bond.SymmetryCode, bond);
rss_identifiers.Add(new InstrumentIdentifier { IdentifierType = "ISN", IdentifierValue = bond.SymmetryCode, Source="TWB"});
Logger.Debug(string.Format("{0}/{1} is missing auction information", bond.SymmetryCode, bond.DisplayName), typeof(UpdateBondAuctionData));
}
}
System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Ssl3;
if (rss_identifiers.Count() == 0)
return; // nothing to do
try
{
var cred = ServiceHelpers.GetCredentials();
{
using (var dsClient = new ExtractionServiceClient())
{
var validatedInstruments = ServiceHelpers.ValidateInstruments(rss_identifiers.ToArray(), dsClient, ref cred);
{
var request = new ExtractRequest
{
CredentialsHeader = cred,
request = new InstrumentExtractionRequestTermsAndConditions
{
Instruments = validatedInstruments,
OutputFields = Enum.GetNames(typeof(FieldsToReturn))
.Select(x => typeof(FieldsToReturn).GetField(x.ToString()))
.Select(x => x.GetCustomAttribute<RSSFieldAttribute>().FieldName).ToArray()
}
};
var response = dsClient.Extract(request);
var pivoted = ExtractionUtility.PivotExtractionResult(response.ExtractResult.Columns, response.ExtractResult.RowCount, response.ExtractResult.Messages);
StringBuilder b = new StringBuilder();
b.AppendLine(ServiceHelpers.ToCsvLine(pivoted.ColumnNames.Concat(new [] {"FIID"})));
foreach (var row in pivoted.Rows)
{
string isin = row[(int)FieldsToReturn.InstrumentID] as string;
b.AppendLine(ServiceHelpers.ToCsvLine(row.Concat(new string[] { dict_isinToMissingBond[isin].FIID.ToString() })));
}
var path = @"e:\temp\missingAuctions.csv";
FileInfo i = new FileInfo(path);
StreamWriter writer = i.CreateText();
writer.Write(b.ToString());
writer.Close();
}
}
}
}
catch (Exception ex_)
{
Logger.Error("blast", typeof(UpdateCTBondData), ex_);
}
}
public static void Go()
{
var ctList = SI.Strategy.CTDs.Generator.Go();
var listOfMissing = new List<MonthYear>();
foreach (var v in ctList)
{
if (v.CT2 == null || v.CT3 == null || v.CT5 == null || v.CT7 == null || v.CT10 == null || v.CT30 == null)
if (!listOfMissing.Contains(v.RefundingMonth))
{
Logger.Debug(string.Format("Missing bonds for {0}", v.RefundingMonth), typeof(UpdateCTBondData));
listOfMissing.Add(v.RefundingMonth);
}
}
System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Ssl3;
try
{
var cred = ServiceHelpers.GetCredentials();
using (var dc = SymmetryDataSource.ReadOnly())
{
using (var dsClient = new ExtractionServiceClient())
{
var isins = new Dictionary<string, ValidatedInstrument>();
foreach (var v in listOfMissing)
{
var request = new SearchInstrumentsRequest
{
CredentialsHeader = cred,
request = new InstrumentSearchRequestGovCorp
{
IdentifierType = "ISN",
IdentifierSearchValue = "US9128*",
PreferredIdentifierType = "ISN",
IsGovernmentGroup = true,
IssueDatePrimaryValue = new DateTime(v.Year, v.Month, 3),
IssueDateSecondaryValue = new DateTime(v.Year, v.Month, 25),
IssueDateComparisonOperator = "FROM",
}
};
var response = dsClient.SearchInstruments(request);
Array.ForEach(response.SearchInstrumentsResult.Instruments.Select(x => x.IdentifierValue).Distinct().ToArray(), (x) => Logger.Debug(x, typeof(UpdateCTBondData)));
foreach (var validatedInstrument in response.SearchInstrumentsResult.Instruments)
{
var isin = validatedInstrument.IdentifierValue;
if (isins.ContainsKey(isin))
{
if (validatedInstrument.Source.Equals("TWB"))
isins[isin] = validatedInstrument;
continue;
}
var fi = FIHelpers.GetFIBySymmetryCode(isin, dc, Symmetry.Core.ThrowBehavior.DontThrow);
if (fi == null)
isins.Add(isin, validatedInstrument);
}
}
if (isins.Count == 0)
return;
{
var request = new ExtractRequest
{
CredentialsHeader = cred,
request = new InstrumentExtractionRequestTermsAndConditions
{
Instruments = isins.ToList().Select(x => x.Value).ToArray(),
OutputFields = Enum.GetNames(typeof(FieldsToReturn))
.Select(x => typeof(FieldsToReturn).GetField(x.ToString()))
.Select(x => x.GetCustomAttribute<RSSFieldAttribute>().FieldName).ToArray()
}
};
var response = dsClient.Extract(request);
var pivoted = ExtractionUtility.PivotExtractionResult(response.ExtractResult.Columns, response.ExtractResult.RowCount, response.ExtractResult.Messages);
StringBuilder b = new StringBuilder();
b.AppendLine(ServiceHelpers.ToCsvLine(pivoted.ColumnNames));
foreach (var row in pivoted.Rows)
{
b.AppendLine(ServiceHelpers.ToCsvLine(row));
}
var path = @"e:\temp\missingBonds.csv";
FileInfo i = new FileInfo(path);
StreamWriter writer = i.CreateText();
writer.Write(b.ToString());
writer.Close();
}
}
}
}
catch (Exception ex_)
{
Logger.Error("blast", typeof(UpdateCTBondData), ex_);
}
}