private async Task TestInitializeMethodAsync()
{
Storage = new AzureTableStorage();
await Storage.SetTable("GamePlayer");
await Storage.DeleteTable();
await Storage.SetTable("GamePlayer");
}
//private TopicClient _topicClient;
public TwitterScraper(TwitterApi twitterApi, AzureTableStorage tableStorage, AzureTableStorage tabelStorage2)
{
_twitterApi = twitterApi;
_tableStorage = tableStorage;
_secondTableStorage = tabelStorage2;
//_topicClient = topicClient;
}
/// <summary>
/// Load the BuildableItems
/// </summary>
public async Task LoadBuildableItems()
{
BuildableItems = new Dictionary<BuildableItemEnum, BuildableItem>();
AzureTableStorage storage = new AzureTableStorage();
await storage.SetTable("BuildableItem");
IEnumerable<BuildableItemTableStorage> items =
await storage.GetAll<BuildableItemTableStorage>(
"Version" + Constants.APPLICATION_VERSION);
foreach(BuildableItemTableStorage item in items)
{
BuildableItemStorageInformation storageInfo = new BuildableItemStorageInformation(item.ItemCode.ToString());
BuildableItems[(BuildableItemEnum)item.ItemCode] = (BuildableItem)storageInfo.FromTableStorage(item);
}
}
/// <summary>
/// Load the experience level definitions
/// </summary>
public async Task LoadExperienceLevelDefinitions()
{
ExperienceLevels = new Dictionary<int, ExperienceLevel>();
AzureTableStorage storage = new AzureTableStorage();
await storage.SetTable("ExperienceLevel");
IEnumerable<ExperienceLevelTableStorage> items =
await storage.GetAll<ExperienceLevelTableStorage>(
"Version" + Constants.APPLICATION_VERSION);
foreach (ExperienceLevelTableStorage item in items)
{
ExperienceLevelStorageInformation storageInfo = new
ExperienceLevelStorageInformation(item.Level.ToString());
ExperienceLevels[item.Level] = (ExperienceLevel)storageInfo.FromTableStorage(item);
}
}
protected override void Load(ContainerBuilder builder)
{
var connectionString = _settings.ConnectionString(x => x.LykkeStreams.Azure.StorageConnString);
var personalDatAapiKey = _settings.CurrentValue.LykkeStreams.PersonalDataService.ApiKey;
var personalDataServiceUri = _settings.CurrentValue.LykkeStreams.PersonalDataService.ServiceUri;
var kycSettings = _settings.CurrentValue.KycServiceClient;
builder.RegisterInstance(
AzureBlobStorage.Create(connectionString)).As <IBlobStorage>().SingleInstance();
builder.RegisterInstance(
new ProjectRepository(AzureTableStorage <ProjectEntity> .Create(connectionString, "Projects", _logFactory))
).As <IProjectRepository>().SingleInstance();
builder.RegisterInstance(
new UsersRepository(AzureTableStorage <UserEntity> .Create(connectionString, "Users", _logFactory))
).As <IUsersRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectCommentsRepository(AzureTableStorage <CommentEntity> .Create(connectionString, "ProjectComments", _logFactory))
).As <IProjectCommentsRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectFileInfoRepository(AzureTableStorage <ProjectFileInfoEntity> .Create(connectionString, "ProjectFilesInfo", _logFactory))
).As <IProjectFileInfoRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectVoteRepository(AzureTableStorage <ProjectVoteEntity> .Create(connectionString, "ProjectVotes", _logFactory))
).As <IProjectVoteRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectParticipantsRepository(AzureTableStorage <ProjectParticipateEntity> .Create(connectionString, "ProjectParticipants", _logFactory))
).As <IProjectParticipantsRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectResultRepository(AzureTableStorage <ProjectResultEntity> .Create(connectionString, "ProjectResults", _logFactory))
).As <IProjectResultRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectResultVoteRepository(AzureTableStorage <ProjectResultVoteEntity> .Create(connectionString, "ProjectResultVotes", _logFactory))
).As <IProjectResultVoteRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectFollowRepository(AzureTableStorage <ProjectFollowEntity> .Create(connectionString, "ProjectFollows", _logFactory))
).As <IProjectFollowRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectWinnersRepository(AzureTableStorage <WinnerEntity> .Create(connectionString, "Winners", _logFactory))
).As <IProjectWinnersRepository>().SingleInstance();
builder.RegisterInstance(
new UserRolesRepository(AzureTableStorage <UserRoleEntity> .Create(connectionString, "UserRoles", _logFactory))
).As <IUserRolesRepository>().SingleInstance();
builder.RegisterInstance(
new FollowMailSentRepository(AzureTableStorage <FollowMailSentEntity> .Create(connectionString, "FollowMailSent", _logFactory))
).As <IFollowMailSentRepository>().SingleInstance();
builder.RegisterInstance(
new UserFeedbackRepository(AzureTableStorage <UserFeedbackEntity> .Create(connectionString, "UserFeedback", _logFactory))
).As <IUserFeedbackRepository>().SingleInstance();
builder.RegisterInstance(
new PublicFeedbackRepository(AzureTableStorage <PublicFeedbackEntity> .Create(connectionString, "PublicFeedback", _logFactory))
).As <IPublicFeedbackRepository>().SingleInstance();
builder.RegisterInstance(
new BlogRepository(AzureTableStorage <BlogEntity> .Create(connectionString, "Blogs", _logFactory))
).As <IBlogRepository>().SingleInstance();
builder.RegisterInstance(
new BlogCommentsRepository(AzureTableStorage <BlogCommentEntity> .Create(connectionString, "BlogComments", _logFactory))
).As <IBlogCommentsRepository>().SingleInstance();
builder.RegisterInstance(
new BlogPictureInfoRepository(AzureTableStorage <BlogPictureInfoEntity> .Create(connectionString, "BlogPicturesInfo", _logFactory))
).As <IBlogPictureInfoRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectExpertsRepository(AzureTableStorage <ProjectExpertEntity> .Create(connectionString, "ProjectExperts", _logFactory))
).As <IProjectExpertsRepository>().SingleInstance();
builder.RegisterInstance(
new StreamRepository(AzureTableStorage <StreamEntity> .Create(connectionString, "ProjectStreams", _logFactory))
).As <IStreamRepository>().SingleInstance();
builder.RegisterInstance(
new StreamsIdRepository(AzureTableStorage <StreamsIdEntity> .Create(connectionString, "StreamsId", _logFactory))
).As <IStreamsIdRepository>().SingleInstance();
builder.RegisterInstance(
new TermsPageRepository(AzureTableStorage <TermsPageEntity> .Create(connectionString, "TermsPage", _logFactory))
).As <ITermsPageRepository>().SingleInstance();
builder.RegisterInstance(
new ProjectCategoriesRepository()
).As <IProjectCategoriesRepository>().SingleInstance();
builder.RegisterInstance(
_settings.CurrentValue
).As <BaseSettings>().SingleInstance();
builder.RegisterInstance(
new PersonalDataService(new PersonalDataServiceClientSettings {
ApiKey = personalDatAapiKey, ServiceUri = personalDataServiceUri
}, _log)
).As <IPersonalDataService>().SingleInstance();
builder.RegisterInstance(
new KycProfileServiceV2Client(new KycServiceClientSettings {
ApiKey = kycSettings.ApiKey, ServiceUri = kycSettings.ServiceUri
}, _log)
).As <IKycProfileServiceV2>().SingleInstance();
builder.RegisterEmailSenderViaAzureQueueMessageProducer(_settings.ConnectionString(x => x.LykkeStreams.Azure.MessageQueueConnString));
builder.RegisterType <ProjectFileRepository>().As <IProjectFileRepository>();
builder.RegisterType <BlogPictureRepository>().As <IBlogPictureRepository>();
builder.RegisterType <ProjectWinnersService>().As <IProjectWinnersService>();
builder.RegisterType <ExpertsService>().As <IExpertsService>();
builder.RegisterType <BlogCategoriesRepository>().As <IBlogCategoriesRepository>();
}
private INoSQLTableStorage <T> CreateTable <T>(string tableName)
where T : AzureTableEntity, new()
{
return(AzureTableStorage <T> .Create(_connectionString, tableName, _log));
}
public static OvernightSwapHistoryRepository CreateOvernightSwapHistoryRepository(IReloadingManager <string> connString, ILog log)
{
return(new OvernightSwapHistoryRepository(AzureTableStorage <OvernightSwapHistoryEntity> .Create(connString,
"OvernightSwapHistory", log)));
}
public AccountsReportRepository(IReloadingManager <string> connectionString, ILog log)
{
_tableStorage = AzureTableStorage <AccountsReportEntity> .Create(connectionString,
"ClientAccountsReports", log);
}
public AccountsReportsRepository(IReloadingManager <Settings> settings, ILog log)
{
_tableStorage = AzureTableStorage <AccountsReportEntity> .Create(settings.Nested(s => s.Db.ReportsConnString),
"ClientAccountsReports", log);
}
public AddressRepository(IReloadingManager <string> connectionStringManager, ILog log)
{
_balanceAddressesStorage = AzureTableStorage <BalanceAddressEntity> .Create(connectionStringManager, "ZcashBalanceAddresses", log);
_historyAddressesStorage = AzureTableStorage <HistoryAddressEntity> .Create(connectionStringManager, "ZcashHistoryAddresses", log);
}
public void ConfigureServices(IServiceCollection services)
{
services.AddApplicationInsightsTelemetry();
services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme)
.AddMicrosoftIdentityWebApi(options =>
{
Configuration.Bind("AzureAdB2C", options);
options.TokenValidationParameters.NameClaimType = "name";
},
options => { Configuration.Bind("AzureAdB2C", options); });
services.AddControllers().AddJsonOptions(opts =>
{
opts.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter());
}).AddNewtonsoftJson();
services.AddAuthorization();
services.AddSwaggerGen(c =>
{
c.ResolveConflictingActions(apiDescriptions => apiDescriptions.FirstOrDefault());
c.SwaggerDoc("v1", new OpenApiInfo {
Title = "StarWars5e.Api", Version = "v1"
});
var securitySchema = new OpenApiSecurityScheme
{
Description = "JWT Authorization header using the Bearer scheme. Example: \"Authorization: Bearer {token}\"",
Name = "Authorization",
In = ParameterLocation.Header,
Type = SecuritySchemeType.Http,
Scheme = "bearer",
Reference = new OpenApiReference
{
Type = ReferenceType.SecurityScheme,
Id = "Bearer"
}
};
c.AddSecurityDefinition("Bearer", securitySchema);
var securityRequirement = new OpenApiSecurityRequirement();
securityRequirement.Add(securitySchema, new[] { "Bearer" });
c.AddSecurityRequirement(securityRequirement);
});
services.AddCors(options =>
{
options.AddDefaultPolicy(
builder =>
{
builder.AllowAnyOrigin()
.AllowAnyHeader()
.AllowAnyMethod();
});
});
var tableStorage = new AzureTableStorage(Configuration["StorageAccountConnectionString"]);
var cloudStorageAccount = CloudStorageAccount.Parse(Configuration["StorageAccountConnectionString"]);
var cloudTableClient = cloudStorageAccount.CreateCloudTableClient();
var cloudBlobClient = cloudStorageAccount.CreateCloudBlobClient();
var searchServiceClient = new SearchServiceClient("sw5esearch", new SearchCredentials(Configuration["SearchKey"]));
var searchIndexClient = searchServiceClient.Indexes.GetClient("searchterms-index");
services.AddSingleton <IAzureTableStorage>(tableStorage);
services.Scan(scan => scan
.FromAssemblies(typeof(Program).GetTypeInfo().Assembly)
.AddClasses(true)
.AsImplementedInterfaces()
.WithSingletonLifetime()
);
services.AddSingleton(cloudBlobClient);
services.AddSingleton(cloudTableClient);
services.AddSingleton(searchIndexClient);
}
private INoSQLTableStorage <T> CreateTable <T>(Func <ApplicationSettings, string> connectionString, string name, ILogFactory logFactory)
where T : TableEntity, new()
{
return(AzureTableStorage <T> .Create(_settings.ConnectionString(connectionString), name, logFactory, TimeSpan.FromSeconds(30)));
}
public MaintenanceInfoRepository(IReloadingManager <string> connectionStringManager)
{
_tableStorage = AzureTableStorage <MaintenanceInfoEntity> .Create(connectionStringManager,
"MaintenanceInfo", LogLocator.CommonLog);
}
//CloudStorageAccount storageAccount;
static async Task Main(string[] args)
{
// Load appsettings.json
var config = LoadAppSettings();
if (null == config)
{
Console.WriteLine("Missing or invalid appsettings.json file. Please see README.md for configuration instructions.");
return;
}
SetGlobalConfig(config);
////Logging
//IServiceCollection services = new ServiceCollection();
//// Channel is explicitly configured to do flush on it later.
//var channel = new InMemoryChannel();
//services.Configure<TelemetryConfiguration>(
// (config) =>
// {
// config.TelemetryChannel = channel;
// }
//);
//services.AddLogging(builder =>
//{
// builder.AddConsole();
// builder.AddApplicationInsights(AppInsightsApiKey);
//});
//var provider = services.BuildServiceProvider();
//var logger = provider.GetService<ILogger<Program>>();
//logger.LogInformation("This will show up in Application Insights");
// Explicitly call Flush() followed by sleep is required in Console Apps.
// This is to ensure that even if application terminates, telemetry is sent to the back-end.
//channel.Flush();
searchServiceHelper = new SearchServiceHelper(SearchServiceName, SearchServiceAdminKey);
System.Diagnostics.Trace.TraceWarning("Slow response - database01");
TimeSpan elapsedTime;
//Start stopwatch for timing telemtry
Stopwatch sw = new Stopwatch();
var timeStart = DateTime.Now;
sw.Start();
//Storage
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var storageClient = storageAccount.CreateCloudBlobClient();
AzureTableStorage azTableStorage = new AzureTableStorage(StorageConnectionString, StorageTableName);
AzureTableStorage azTableStorageSpoItems = new AzureTableStorage(StorageConnectionString, SpoItemStorageTableName);
CloudBlobContainer container = await AzureBLOBStorage.CreateAzureBLOBContainer(storageClient, BlobContainerName);
//Search
AzureSearchServiceHelper searchClient = new AzureSearchServiceHelper(SearchServiceName, SearchServiceAdminKey);
//Lookup itemId for the item that needs to be removed from the search index
//Now delete the itemId from the index
await searchClient.DeleteItemFromIndexAsync("aaaa", SearchServiceIndexName, "11");
IDriveItemChildrenCollectionPage docLibItems;
IDriveItemDeltaCollectionPage docLibDeltaItems;
for (int i = 0; i < args.Length; i++)
{
if (args[i].ToLower() == "-incrementalcrawl")
{
IncrementalCrawl = true;
Console.WriteLine("Search Crawl mode set to Incremental");
container = await AzureBLOBStorage.CreateAzureBLOBContainer(storageClient, BlobContainerName);
}
if (args[i].ToLower() == "-fullcrawl")
{
IncrementalCrawl = false;
Console.WriteLine("Search Crawl mode set to Full");
await AzureBLOBStorage.DeleteContainerFromAzureBLOB(container);
container = await AzureBLOBStorage.CreateAzureBLOBContainer(storageClient, BlobContainerName);
}
if (args[i].ToLower() == "-includeAcls")
{
IncludeAcls = true;
Console.WriteLine("Search Crawl mode set to Full");
}
}
SharePointOnlineHelper.metadataFieldsToIgnore = MetadataFieldsToIgnore;
SharePointOnlineHelper.metadataJSONStore = MetadataJSONStore;
SharePointOnlineHelper.acls = IncludeAcls;
SharePointOnlineHelper.azTableStorage = azTableStorageSpoItems;
foreach (var metadataFieldToIgnore in MetadataFieldsToIgnore)
{
Console.WriteLine("Removing key [{0}] from metadata fields to extract", metadataFieldToIgnore);
}
//Query using Graph SDK (preferred when possible)
GraphServiceClient graphClient = SharePointOnlineHelper.GetAuthenticatedGraphClient(config);
Site targetSite = await graphClient.Sites.GetByPath(SiteUrl, SPOHostName).Request().GetAsync();
ISiteDrivesCollectionPage drives = graphClient.Sites[targetSite.Id].Drives.Request().GetAsync().Result;
//Graph BETA supports site pages
//var sitePages = graphClient.Sites[targetSite.Id].Pages.Request().GetAsync().GetAwaiter().GetResult();
//var sitePages = graphClient.Sites[targetSite.Id].Pages.Request().GetAsync().Result;
//var a = 1;
foreach (var drive in drives)
{
var driveName = drive.Name;
var driveUrl = drive.WebUrl;
bool excludedDocLIb = Array.Exists(DocLibsToIgnore, element => element == driveName);
if (excludedDocLIb)
{
Console.WriteLine("Skipping [{0}] as its an excluded docLib", DocLibsToIgnore);
continue;
}
Console.WriteLine("Fetching items from drive [{0}]", driveName);
var driveId = drive.Id;
var driveContents = new List <DriveItem>();
//Full Crawl Logic
if (!IncrementalCrawl)
{
docLibItems = await graphClient
.Drives[driveId]
.Root
.Children
.Request()
.GetAsync();
driveContents.AddRange(docLibItems.CurrentPage);
if (docLibItems.NextPageRequest != null)
{
while (docLibItems.NextPageRequest != null)
{
docLibItems = await docLibItems.NextPageRequest.GetAsync();
driveContents.AddRange(docLibItems.CurrentPage);
await SharePointOnlineHelper.GetSpoDocumentItems(graphClient, driveContents, driveId, container, IncludeAcls);
}
}
else
{
await SharePointOnlineHelper.GetSpoDocumentItems(graphClient, driveContents, driveId, container, IncludeAcls);
}
}
//Incremental Crawl Logic
if (IncrementalCrawl)
{
//Retrieve the last known deltaToken from Table storage, if the value is null it will fetch all items for that drive
//Base64 encode the string to remove special characters
byte[] byt = System.Text.Encoding.UTF8.GetBytes(driveUrl);
var driveUrlEscpaed = Convert.ToBase64String(byt);
var lastDeltaToken = await azTableStorage.GetEntitiesInPartion(driveUrlEscpaed);
docLibDeltaItems = await graphClient
.Drives[driveId]
.Root
.Delta(lastDeltaToken)
.Request()
.GetAsync();
var deltaLink = docLibDeltaItems.AdditionalData["@odata.deltaLink"].ToString();
if (deltaLink != null)
{
var tokenindex = deltaLink.IndexOf("token=");
var token = deltaLink.Substring(tokenindex + 7, deltaLink.ToString().Length - tokenindex - 9);
driveContents.AddRange(docLibDeltaItems.CurrentPage);
if (docLibDeltaItems.NextPageRequest != null)
{
while (docLibDeltaItems.NextPageRequest != null)
{
var docLibItems2 = await docLibDeltaItems.NextPageRequest.GetAsync();
driveContents.AddRange(docLibItems2.CurrentPage);
await SharePointOnlineHelper.GetSpoDocumentItems(graphClient, driveContents, driveId, container, IncludeAcls);
}
}
else
{
await SharePointOnlineHelper.GetSpoDocumentItems(graphClient, driveContents, driveId, container, IncludeAcls);
//Lets persist the changeToken to storage so we can continue the next incrmental crawl from this point.
IndexCrawlEntity indexCrawlEntity = new IndexCrawlEntity(driveUrlEscpaed, token);
azTableStorage.InsertEntity(indexCrawlEntity);
}
//Console.WriteLine("Fetched total of {0} documents from [{1}] data source", DownloadFileCount, driveName);
}
}
}
if (!IncrementalCrawl)
{
//Now lets do a full crawl of all the fetched SPO documents from the BLOB store as the fetching of all documents into storage would have completed by now
//Warning this will perform an entire search index rebuild - so while this phase is running search resultset will be impacted
await IndexDocumentsAsync();
}
sw.Stop();
elapsedTime = sw.Elapsed;
var timeEnd = DateTime.Now;
Console.WriteLine("Fetched total of {0} documents during crawl", AzureBLOBStorage.DownloadFileCount);
Console.WriteLine("Crawl Start time: {0}", timeStart);
Console.WriteLine("Crawl Completed time: {0}", timeEnd);
Console.WriteLine("Total crawl duration time: {0}", elapsedTime);
}
public OperationValidationRepository(IReloadingManager <string> connectionStringManager, ILogFactory logFactory)
{
_storage = AzureTableStorage <OperationValidationEntity> .Create(connectionStringManager, "RiskControlOperationValidations", logFactory);
_index = AzureTableStorage <AzureIndex> .Create(connectionStringManager, "RiskControlOperationValidations", logFactory);
}
public static RiskSystemCommandsLogRepository CreateRiskSystemCommandsLogRepository(IReloadingManager <string> connString, ILog log)
{
return(new RiskSystemCommandsLogRepository(AzureTableStorage <RiskSystemCommandsLogEntity> .Create(connString,
"RiskSystemCommandsLog", log)));
}
public static MarginTradingAccountStatsRepository CreateAccountStatsRepository(IReloadingManager <string> connString, ILog log)
{
return(new MarginTradingAccountStatsRepository(AzureTableStorage <MarginTradingAccountStatsEntity> .Create(connString,
"MarginTradingAccountStats", log)));
}
private static async Task Import(string settingsUrl, string hotwallet, string timestamp, string batchSize)
{
if (!Uri.TryCreate(settingsUrl, UriKind.Absolute, out _))
{
Console.WriteLine($"{SettingsUrl}: {settingsUrl} should be a valid uri");
return;
}
DateTime timeStampTyped;
if (!DateTime.TryParse(timestamp, out timeStampTyped))
{
Console.WriteLine($"{Timestamp}: {timestamp} should be Date time ");
return;
}
if (!DateTime.TryParse(timestamp, out timeStampTyped))
{
Console.WriteLine($"{Timestamp}: {timestamp} should be a valid Date time ");
return;
}
if (!int.TryParse(batchSize, out var batchSizeTyped))
{
Console.WriteLine($"{BatchSize}: {batchSize} should be a valid integer ");
return;
}
var logFactory = LogFactory.Create().AddConsole();
var settings = new SettingsServiceReloadingManager <AppSettings>(settingsUrl).Nested(x => x.BitcoinCashApi);
var network = Network.GetNetwork(settings.CurrentValue.Network);
var bcashNetwork = network == Network.Main ? BCash.Instance.Mainnet : BCash.Instance.Regtest;
var addressValidator = new AddressValidator(network, bcashNetwork);
var hotwalletTyped = addressValidator.GetBitcoinAddress(hotwallet);
if (hotwalletTyped == null)
{
Console.WriteLine($"{HotWallet}: {hotwallet} should be a valid bitcoin address");
return;
}
var authTokenBytes = Encoding.ASCII.GetBytes($"{settings.CurrentValue.Rpc.UserName}:{settings.CurrentValue.Rpc.Password}");
var authToken = Convert.ToBase64String(authTokenBytes);
var httpClient = new HttpClient
{
BaseAddress = new Uri(settings.CurrentValue.Rpc.Host),
Timeout = TimeSpan.FromMinutes(30),
DefaultRequestHeaders =
{
Authorization = new AuthenticationHeaderValue("Basic", authToken)
}
};
var observableWalletRepository = new ObservableWalletRepository(AzureTableStorage <ObservableWalletEntity> .Create(
settings.Nested(p => p.Db.DataConnString),
"ObservableWalletsV2", logFactory));
Console.WriteLine("Retrieving observable wallets");
var observableWallets = await observableWalletRepository.GetAll();
var walletsToImport = observableWallets
.Select(p => addressValidator.GetBitcoinAddress(p.Address))
.Concat(new[] { hotwalletTyped })
.ToList();
Console.WriteLine($"Importing {walletsToImport.Count} addresses in node started at {DateTime.UtcNow}. Timestamp {timeStampTyped}. Batch size {batchSize}");
var batchNum = 1;
foreach (var batch in walletsToImport.Batch(batchSizeTyped))
{
Console.WriteLine($"{DateTime.UtcNow} Importing batch {batchNum++}...");
var payload = new
{
jsonrpc = "1.0",
id = Guid.NewGuid().ToString(),
method = "importmulti",
@params = new object[]
{
batch.Select(address => new
{
scriptPubKey = new ImportMultiAddress.ScriptPubKeyObject(address),
timestamp = Utils.DateTimeToUnixTime(new DateTimeOffset(timeStampTyped))
}),
new
{
rescan = true
}
}
};
await Policy
.Handle <Exception>()
.RetryAsync(5, (e, i) =>
{
Console.WriteLine($"Retrying ({i})...");
})
.ExecuteAsync(async() =>
{
using (var content = new StringContent(JsonConvert.SerializeObject(payload), Encoding.UTF8, "application/json"))
{
using (var response = await httpClient.PostAsync("", content))
{
if (response.StatusCode != HttpStatusCode.OK)
{
Console.WriteLine($"Failed to execute RPC call. Response: {response.StatusCode}");
throw new InvalidOperationException($"Failed to execute RPC call. Response: {response.StatusCode}, {await response.Content.ReadAsStringAsync()}");
}
}
}
});
Console.WriteLine($"{DateTime.UtcNow} Batch imported");
}
Console.WriteLine($"Import completed at {DateTime.UtcNow}");
}
private async Task FindTables()
{
await _dbsettings.Reload();
await _log.WriteInfoAsync(nameof(AzureLogHandler), nameof(FindTables), $"Begin find log tables, count accounts: {_dbsettings.CurrentValue.ScanLogsConnString.Length}");
foreach (var connString in _dbsettings.CurrentValue.ScanLogsConnString)
{
CloudStorageAccount account = CloudStorageAccount.Parse(connString);
var accountName = account.Credentials.AccountName;
try
{
await _log.WriteInfoAsync(nameof(AzureLogHandler), nameof(FindTables), accountName, $"Start scan account: {accountName}");
var tableClient = account.CreateCloudTableClient();
var names = (await tableClient.ListTablesSegmentedAsync(null)).Select(e => e.Name)
.Where(e => !_exclute.Contains(e)).ToArray();
await _log.WriteInfoAsync(nameof(AzureLogHandler), nameof(FindTables), accountName, $"Find {names.Length} tables in subscribtion");
var countAdd = 0;
#if DEBUG
var countHandling = 0;
#endif
foreach (var name in names)
{
try
{
CloudTable table = tableClient.GetTableReference(name);
var operationGet = new TableQuery <LogEntity>().Take(1);
var row = (await table.ExecuteQuerySegmentedAsync(operationGet, null)).FirstOrDefault();
if (row != null && row.DateTime != DateTime.MinValue && row.Level != null && row.Msg != null)
{
if (_tables.All(e => e.Name != name || e.ConnString != connString))
{
var info = new TableInfo
{
Entity = AzureTableStorage <LogEntity> .Create(new FakeReloadingManager(connString), name, _log),
PartitionKey = DateTimeOffset.UtcNow.ToString("yyyy-MM-dd"),
LastRowKey = DateTime.UtcNow.ToString("HH:mm:ss.fffffff"),
Name = name,
Account = accountName,
ConnString = connString
};
_tables.Add(info);
countAdd++;
}
}
#if DEBUG
countHandling++;
Console.Write($"\rhandling: {countHandling} / {names.Length} ");
#endif
}
catch (Exception ex)
{
#if DEBUG
Console.WriteLine();
#endif
await _log.WriteErrorAsync(nameof(AzureLogHandler), nameof(FindTables), $"{accountName} - {name}", ex);
}
}
#if DEBUG
Console.WriteLine();
#endif
await _log.WriteInfoAsync(nameof(AzureLogHandler), nameof(FindTables), accountName, $"Add {countAdd} tables to handling");
}
catch (Exception ex)
{
await _log.WriteErrorAsync(nameof(AzureLogHandler), nameof(FindTables), $"{accountName}", ex);
}
}
await _log.WriteInfoAsync(nameof(AzureLogHandler), nameof(FindTables), $"Start handling {_tables.Count} tables");
}
private void BindRepositories(ContainerBuilder builder)
{
builder.Register <ITransactionsRepository>(ctx =>
new TransactionsRepository(
AzureTableStorage <BitCoinTransactionEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.BitCoinQueueConnectionString),
"BitCoinTransactions", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IWalletCredentialsRepository>(ctx =>
new WalletCredentialsRepository(
AzureTableStorage <WalletCredentialsEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.ClientPersonalInfoConnString),
"WalletCredentials", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IBcnClientCredentialsRepository>(ctx =>
new BcnClientCredentialsRepository(
AzureTableStorage <BcnCredentialsRecordEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.ClientPersonalInfoConnString),
"BcnClientCredentials", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IClientCacheRepository>(ctx =>
new ClientCacheRepository(
AzureTableStorage <ClientCacheEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.ClientPersonalInfoConnString),
"ClientCache", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IEthereumTransactionRequestRepository>(ctx =>
new EthereumTransactionRequestRepository(
AzureTableStorage <EthereumTransactionReqEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.BitCoinQueueConnectionString),
"EthereumTxRequest", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IBitcoinCashinRepository>(ctx =>
new BitcoinCashinRepository(
AzureTableStorage <BitcoinCashinEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.BitCoinQueueConnectionString),
"BitcoinCashin", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IMarketOrdersRepository>(ctx =>
new MarketOrdersRepository(AzureTableStorage <MarketOrderEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.HMarketOrdersConnString),
"MarketOrders", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <ILimitOrdersRepository>(ctx =>
new LimitOrdersRepository(AzureTableStorage <LimitOrderEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.HMarketOrdersConnString),
"LimitOrders", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.RegisterInstance <IEmailCommandProducer>(
new EmailCommandProducer(AzureQueueExt.Create(
_dbSettingsManager.ConnectionString(x => x.ClientPersonalInfoConnString),
"emailsqueue")));
builder.Register <IOffchainRequestRepository>(ctx =>
new OffchainRequestRepository(
AzureTableStorage <OffchainRequestEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.OffchainConnString),
"OffchainRequests", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IOffchainTransferRepository>(ctx =>
new OffchainTransferRepository(
AzureTableStorage <OffchainTransferEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.OffchainConnString),
"OffchainTransfers", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IPaymentTransactionsRepository>(ctx =>
new PaymentTransactionsRepository(
AzureTableStorage <PaymentTransactionEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.ClientPersonalInfoConnString),
"PaymentTransactions", ctx.Resolve <ILogFactory>()),
AzureTableStorage <AzureMultiIndex> .Create(
_dbSettingsManager.ConnectionString(x => x.ClientPersonalInfoConnString),
"PaymentTransactions", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.RegisterInstance(
new BitcoinTransactionContextBlobStorage(AzureBlobStorage.Create(
_dbSettingsManager.ConnectionString(x => x.BitCoinQueueConnectionString))))
.As <IBitcoinTransactionContextBlobStorage>();
builder.Register <IEthererumPendingActionsRepository>(ctx =>
new EthererumPendingActionsRepository(
AzureTableStorage <EthererumPendingActionEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.BitCoinQueueConnectionString),
"EthererumPendingActions", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register <IClientCommentsRepository>(ctx =>
new ClientCommentsRepository(AzureTableStorage <ClientCommentEntity> .Create(
_dbSettingsManager.ConnectionString(x => x.ClientPersonalInfoConnString),
"ClientComments", ctx.Resolve <ILogFactory>())))
.SingleInstance();
builder.Register(ctx =>
EthereumCashinAggregateRepository.Create(
_dbSettingsManager.ConnectionString(x => x.BitCoinQueueConnectionString),
ctx.Resolve <ILogFactory>()))
.As <IEthereumCashinAggregateRepository>()
.SingleInstance();
}
private ILog CreateLogWithSlack(IConfiguration configuration, IServiceCollection services,
IReloadingManager <CandlesHistoryWriterSettings> settings, SlackNotificationsSettings slackSettings)
{
const string tableName = "CandlesHistoryWriterServiceLog";
var consoleLogger = new LogToConsole();
var aggregateLogger = new AggregateLogger();
var settingsValue = settings.CurrentValue;
aggregateLogger.AddLog(consoleLogger);
LykkeLogToAzureSlackNotificationsManager slackNotificationsManager = null;
if (slackSettings?.AzureQueue?.ConnectionString != null &&
slackSettings.AzureQueue.QueueName != null)
{
// Creating slack notification service, which logs own azure queue processing messages to aggregate log
var slackService = services.UseSlackNotificationsSenderViaAzureQueue(new AzureQueueSettings
{
ConnectionString = slackSettings.AzureQueue.ConnectionString,
QueueName = slackSettings.AzureQueue.QueueName
}, aggregateLogger);
slackNotificationsManager = new LykkeLogToAzureSlackNotificationsManager(slackService, consoleLogger);
var logToSlack = LykkeLogToSlack.Create(slackService, "Prices");
aggregateLogger.AddLog(logToSlack);
}
if (settings.CurrentValue.UseSerilog)
{
aggregateLogger.AddLog(new SerilogLogger(typeof(Startup).Assembly, configuration));
}
else if (settingsValue.Db.StorageMode == StorageMode.SqlServer)
{
aggregateLogger.AddLog(new LogToSql(new SqlLogRepository(tableName,
settingsValue.Db.LogsConnectionString)));
}
else if (settingsValue.Db.StorageMode == StorageMode.Azure)
{
var dbLogConnectionString = settingsValue.Db.LogsConnectionString;
// Creating azure storage logger, which logs own messages to console log
if (!string.IsNullOrEmpty(dbLogConnectionString) && !(dbLogConnectionString.StartsWith("${") &&
dbLogConnectionString.EndsWith("}")))
{
var persistenceManager = new LykkeLogToAzureStoragePersistenceManager(
AzureTableStorage <Logs.LogEntity> .Create(settings.Nested(x =>
x.Db.LogsConnectionString), tableName, consoleLogger),
consoleLogger);
var azureStorageLogger = new LykkeLogToAzureStorage(
persistenceManager,
slackNotificationsManager,
consoleLogger);
azureStorageLogger.Start();
aggregateLogger.AddLog(azureStorageLogger);
}
}
return(aggregateLogger);
}
protected override void Load(ContainerBuilder builder)
{
builder.RegisterInstance(_log)
.As <ILog>()
.SingleInstance();
builder.Register <IMarginTradingOperationsLogRepository>(ctx =>
new MarginTradingOperationsLogRepository(
AzureTableStorage <OperationLogEntity> .Create(_settings.Nested(s => s.Db.LogsConnString),
"MarginTradingBackendOperationsLog", _log))
).SingleInstance();
builder.Register <IMarginTradingAccountsRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateAccountsRepository(_settings.Nested(s => s.Db.MarginTradingConnString), _log)
).SingleInstance();
builder.Register <IMarginTradingOrdersHistoryRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateOrdersHistoryRepository(_settings.Nested(s => s.Db.HistoryConnString), _log)
).SingleInstance();
builder.Register <IMarginTradingAccountHistoryRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateAccountHistoryRepository(_settings.Nested(s => s.Db.HistoryConnString), _log)
).SingleInstance();
builder.Register <IMatchingEngineRoutesRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateMatchingEngineRoutesRepository(_settings.Nested(s => s.Db.MarginTradingConnString), _log)
).SingleInstance();
builder.Register <ITradingConditionRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateTradingConditionsRepository(_settings.Nested(s => s.Db.MarginTradingConnString), _log)
).SingleInstance();
builder.Register <IAccountGroupRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateAccountGroupRepository(_settings.Nested(s => s.Db.MarginTradingConnString), _log)
).SingleInstance();
builder.Register <IAccountAssetPairsRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateAccountAssetsRepository(_settings.Nested(s => s.Db.MarginTradingConnString), _log)
).SingleInstance();
builder.Register <IMarginTradingBlobRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateBlobRepository(_settings.Nested(s => s.Db.StateConnString))
).SingleInstance();
builder.Register <IRiskSystemCommandsLogRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateRiskSystemCommandsLogRepository(_settings.Nested(s => s.Db.LogsConnString), _log)
).SingleInstance();
builder.Register <IOvernightSwapStateRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateOvernightSwapStateRepository(_settings.Nested(s => s.Db.StateConnString), _log)
).SingleInstance();
builder.Register <IOvernightSwapHistoryRepository>(ctx =>
AzureRepoFactories.MarginTrading.CreateOvernightSwapHistoryRepository(_settings.Nested(s => s.Db.HistoryConnString), _log)
).SingleInstance();
builder.Register(ctx =>
AzureRepoFactories.MarginTrading.CreateDayOffSettingsRepository(_settings.Nested(s => s.Db.MarginTradingConnString))
).SingleInstance();
builder.Register(ctx =>
AzureRepoFactories.MarginTrading.CreateAssetPairSettingsRepository(
_settings.Nested(s => s.Db.MarginTradingConnString), _log, ctx.Resolve <IConvertService>())
).SingleInstance();
builder.RegisterType <MatchingEngineInMemoryRepository>()
.As <IMatchingEngineRepository>()
.SingleInstance();
}
public BroadcastRepository(IReloadingManager <string> connectionStringManager, ILog log)
{
_table = AzureTableStorage <BroadcastEntity> .Create(connectionStringManager, "Broadcasts", log);
}
public BalancePositiveRepository(IReloadingManager <string> connectionStringManager, ILogFactory logFactory)
{
_table = AzureTableStorage <BalancePositiveEntity> .Create(connectionStringManager, "BalancesPositive", logFactory);
}
public static PledgeRepository CreatePledgeRepository(IReloadingManager <string> connString, ILog log)
{
return(new PledgeRepository(AzureTableStorage <PledgeEntity> .Create(connString, "Pledges", log)));
}
/// <summary>
/// A way to get the items into the table storage for now.
/// @ TO DO Figure out how we actually want to do this
/// </summary>
public async Task StoreExperienceLevelDefintiions()
{
List<ExperienceLevelTableStorage> exls = new List<ExperienceLevelTableStorage>();
ExperienceLevel exl;
ExperienceLevelStorageInformation storageInfo;
exl = new ExperienceLevel
{
Level = 1,
ExperienceRequired = 0,
};
storageInfo = new ExperienceLevelStorageInformation(exl.Level.ToString());
exls.Add((ExperienceLevelTableStorage)storageInfo.ToTableStorage(exl));
exl = new ExperienceLevel
{
Level = 2,
ExperienceRequired = 100,
};
storageInfo = new ExperienceLevelStorageInformation(exl.Level.ToString());
exls.Add((ExperienceLevelTableStorage)storageInfo.ToTableStorage(exl));
exl = new ExperienceLevel
{
Level = 3,
ExperienceRequired = 300,
};
storageInfo = new ExperienceLevelStorageInformation(exl.Level.ToString());
exls.Add((ExperienceLevelTableStorage)storageInfo.ToTableStorage(exl));
exl = new ExperienceLevel
{
Level = 4,
ExperienceRequired = 700,
};
storageInfo = new ExperienceLevelStorageInformation(exl.Level.ToString());
exls.Add((ExperienceLevelTableStorage)storageInfo.ToTableStorage(exl));
AzureTableStorage storage = new AzureTableStorage();
await storage.SetTable("ExperienceLevel");
await storage.DeleteTable();
await storage.SetTable("ExperienceLevel");
await storage.InsertOrReplace<ExperienceLevelTableStorage>(exls);
}
private static ILog CreateLogWithSlack(IServiceCollection services, IReloadingManager <AppSettings> settings)
{
var consoleLogger = new LogToConsole();
var aggregateLogger = new AggregateLogger();
aggregateLogger.AddLog(consoleLogger);
var dbLogConnectionStringManager = settings.Nested(x => x.BitcoinGoldApi.Db.LogsConnString);
var dbLogConnectionString = dbLogConnectionStringManager.CurrentValue;
if (string.IsNullOrEmpty(dbLogConnectionString))
{
consoleLogger.WriteWarningAsync(nameof(Startup), nameof(CreateLogWithSlack), "Table loggger is not inited").Wait();
return(aggregateLogger);
}
if (dbLogConnectionString.StartsWith("${") && dbLogConnectionString.EndsWith("}"))
{
throw new InvalidOperationException($"LogsConnString {dbLogConnectionString} is not filled in settings");
}
var persistenceManager = new LykkeLogToAzureStoragePersistenceManager(
AzureTableStorage <LogEntity> .Create(dbLogConnectionStringManager, "BitcoinGoldApiLog", consoleLogger),
consoleLogger);
// Creating slack notification service, which logs own azure queue processing messages to aggregate log
var slackService = services.UseSlackNotificationsSenderViaAzureQueue(new AzureQueueIntegration.AzureQueueSettings
{
ConnectionString = settings.CurrentValue.SlackNotifications.AzureQueue.ConnectionString,
QueueName = settings.CurrentValue.SlackNotifications.AzureQueue.QueueName
}, aggregateLogger);
var slackNotificationsManager = new LykkeLogToAzureSlackNotificationsManager(slackService, consoleLogger);
// Creating azure storage logger, which logs own messages to concole log
var azureStorageLogger = new LykkeLogToAzureStorage(
persistenceManager,
slackNotificationsManager,
consoleLogger);
azureStorageLogger.Start();
aggregateLogger.AddLog(azureStorageLogger);
var allMessagesSlackLogger = LykkeLogToSlack.Create
(
slackService,
"BlockChainIntegration",
// ReSharper disable once RedundantArgumentDefaultValue
LogLevel.All
);
aggregateLogger.AddLog(allMessagesSlackLogger);
var importantMessagesSlackLogger = LykkeLogToSlack.Create
(
slackService,
"BlockChainIntegrationImportantMessages",
LogLevel.All ^ LogLevel.Info
);
aggregateLogger.AddLog(importantMessagesSlackLogger);
return(aggregateLogger);
}
protected override void Load(ContainerBuilder builder)
{
const string merchantsTableName = "Merchants";
const string paymentRequestsTableName = "PaymentRequests";
const string ordersTableName = "Orders";
const string assetsAvailabilityTableName = "AssetsAvailability";
const string assetsAvailabilityByMerchantTableName = "AssetsAvailabilityByMerchant";
const string transfersTableName = "Transfers";
const string bcnWalletsUsageTableName = "BlockchainWalletsUsage";
const string virtualWalletsTableName = "VirtualWallets";
const string merchantTransactionsTableName = "MerchantWalletTransactions";
const string markupsTableName = "Markups";
builder.RegisterInstance <IMerchantRepository>(new MerchantRepository(
AzureTableStorage <MerchantEntity> .Create(_merchantsConnectionString,
merchantsTableName, _log)));
builder.RegisterInstance <IPaymentRequestRepository>(new PaymentRequestRepository(
AzureTableStorage <PaymentRequestEntity> .Create(_paymentRequestsConnectionString,
paymentRequestsTableName, _log),
AzureTableStorage <AzureIndex> .Create(_paymentRequestsConnectionString,
paymentRequestsTableName, _log),
AzureTableStorage <AzureIndex> .Create(_paymentRequestsConnectionString,
paymentRequestsTableName, _log)));
builder.RegisterInstance <IVirtualWalletRepository>(new VirtualWalletRepository(
AzureTableStorage <VirtualWalletEntity> .Create(_paymentRequestsConnectionString,
virtualWalletsTableName, _log),
AzureTableStorage <AzureIndex> .Create(_paymentRequestsConnectionString,
virtualWalletsTableName, _log),
AzureTableStorage <AzureIndex> .Create(_paymentRequestsConnectionString,
virtualWalletsTableName, _log)));
builder.RegisterInstance <IBcnWalletUsageRepository>(new BcnWalletUsageRepository(
AzureTableStorage <BcnWalletUsageEntity> .Create(_paymentRequestsConnectionString,
bcnWalletsUsageTableName, _log)));
builder.RegisterInstance <IOrderRepository>(new OrdersRepository(
AzureTableStorage <OrderEntity> .Create(_ordersConnectionString,
ordersTableName, _log)));
builder.RegisterInstance <IAssetGeneralAvailabilityRepository>(new AssetGeneralAvailabilityRepository(
AzureTableStorage <AssetAvailabilityEntity> .Create(_paymentRequestsConnectionString,
assetsAvailabilityTableName, _log)));
builder.RegisterInstance <IAssetPersonalAvailabilityRepository>(new AssetPersonalAvailabilityRepository(
AzureTableStorage <AssetAvailabilityByMerchantEntity> .Create(_paymentRequestsConnectionString,
assetsAvailabilityByMerchantTableName, _log)));
builder.RegisterInstance <ITransferRepository>(new TransferRepository(
AzureTableStorage <TransferEntity> .Create(_transfersConnectionString, transfersTableName, _log),
AzureTableStorage <AzureIndex> .Create(_transfersConnectionString, transfersTableName, _log)));
builder.RegisterInstance <IPaymentRequestTransactionRepository>(new PaymentRequestTransactionRepository(
AzureTableStorage <PaymentRequestTransactionEntity> .Create(_merchantsConnectionString, merchantTransactionsTableName, _log),
AzureTableStorage <AzureIndex> .Create(_merchantsConnectionString, merchantTransactionsTableName, _log),
AzureTableStorage <AzureIndex> .Create(_merchantsConnectionString, merchantTransactionsTableName, _log)));
builder.RegisterInstance <IMarkupRepository>(new MarkupRepository(
AzureTableStorage <MarkupEntity> .Create(_merchantsConnectionString, markupsTableName, _log),
AzureTableStorage <AzureIndex> .Create(_merchantsConnectionString, markupsTableName, _log)));
}
static void Main(string[] args)
{
if (args.Contains("help") || !args.Any())
{
Console.WriteLine("params:");
Console.WriteLine(" list - show list of all storage accounts");
Console.WriteLine(" replace oldprefix newprefix resourcegroup subscription - replace to new storage account");
Console.Write("Command (list, replace): ");
var command = Console.ReadLine();
if (command == "list")
{
args = new string[] { "list" };
}
if (command == "replace")
{
args = new string[5];
args[0] = "replace";
Console.Write("oldprefix: "); args[1] = Console.ReadLine();
Console.Write("newprefix: "); args[2] = Console.ReadLine();
Console.Write("resourcegroup: "); args[3] = Console.ReadLine();
Console.Write("subscription: "); args[4] = Console.ReadLine();
}
}
//var connection = "";
Console.Write("Connection string: "); var connection = Console.ReadLine();
var connectionString = ConstantReloadingManager.From(connection);
var keyValueTableStorage =
AzureTableStorage <KeyValueEntity> .Create(
connectionString,
"KeyValues",
EmptyLogFactory.Instance);
var data = keyValueTableStorage.GetDataAsync().Result;
var azureStorage = data.Where(i =>
i.Value != null &&
i.Value.Contains("DefaultEndpointsProtocol=https") && i.Value.Contains("AccountKey=") &&
i.Value.Contains("EndpointSuffix=core.windows.net")).ToList();
Console.WriteLine($"Count keys: {data.Count}, count storages: {azureStorage.Count}");
var names = new List <(string, KeyValueEntity)>();
foreach (var entity in azureStorage)
{
var pattern = "AccountName=(.*?);";
var r = Regex.Match(entity.Value, pattern);
var name = r.Groups[1].Value;
names.Add((name, entity));
}
if (args.Contains("list"))
{
foreach (var name in names.Select(e => e.Item1).OrderBy(e => e))
{
Console.WriteLine(name);
}
return;
}
if (args.Length == 5 && args[0] == "replace")
{
var oldprefix = args[1];
var newPrefix = args[2];
var resourcegroup = args[3];
var subscription = args[4];
foreach (var entity in names)
{
try
{
if (entity.Item1.StartsWith(oldprefix))
{
var newName = entity.Item1.Replace(oldprefix, newPrefix);
var responce = $" storage account check-name -n {newName}".AZ();
var result = JsonConvert.DeserializeObject <StorageAccountCheckNameResponce>(responce);
if (!result.nameAvailable)
{
Console.WriteLine(
$"WARNING. {entity.Item2.RowKey}: {entity.Item1} --> {newName} - name not available");
//continue;
}
responce =
$"storage account create --name {newName} --resource-group {resourcegroup} --subscription {subscription}".AZ();
if (!responce.Contains("Id") || !responce.Contains("creationTime"))
{
Console.WriteLine(
$"ERROR. {entity.Item2.RowKey}: {entity.Item1} --> {newName} - {responce}");
continue;
}
responce = $"storage account show-connection-string -g {resourcegroup} -n {newName}".AZ();
var connString = JsonConvert.DeserializeObject <StorageAccountConnectionString>(responce);
entity.Item2.Value = connString.connectionString;
keyValueTableStorage.ReplaceAsync(entity.Item2).Wait();
Console.WriteLine(
$"OK. {entity.Item2.RowKey}: {entity.Item1} --> {newName}");
}
}
catch (Exception ex)
{
Console.WriteLine($"EXCEPTION. {entity.Item2.RowKey}");
Console.WriteLine(ex);
}
}
}
}
public BroadcastInProgressRepository(IReloadingManager <string> connectionStringManager, ILogFactory logFactory)
{
_table = AzureTableStorage <BroadcastInProgressEntity> .Create(connectionStringManager, "BroadcastsInProgress", logFactory);
}
protected override void Load(ContainerBuilder builder)
{
// RabbitMq
foreach (var exchange in _settings.RabbitMq.SubscribingExchanges)
{
builder.RegisterType <TickPricesSubscriber>()
.AsSelf()
.As <IStartable>()
.As <IStopable>()
.WithParameter("connectionString", _settings.RabbitMq.ConnectionString)
.WithParameter("exchangeName", exchange)
.WithParameter("suffixName", _settings.IndexName)
.SingleInstance();
}
builder.RegisterType <TickPricePublisher>()
.As <ITickPricePublisher>()
.As <IStartable>()
.As <IStopable>()
.WithParameter(TypedParameter.From(_settings.RabbitMq))
.SingleInstance();
// Repositories
// Blob
builder.RegisterInstance(AzureBlobStorage.Create(_connectionString));
builder.RegisterType <IndexHistoryBlobRepository>();
// Tables
builder.Register(c => new SettingsRepository(
AzureTableStorage <SettingsEntity> .Create(_connectionString,
nameof(Settings), c.Resolve <ILogFactory>())))
.As <ISettingsRepository>()
.SingleInstance();
builder.Register(c => new IndexHistoryRepository(
AzureTableStorage <IndexHistoryEntity> .Create(_connectionString,
nameof(IndexHistory), c.Resolve <ILogFactory>()), c.Resolve <IndexHistoryBlobRepository>()))
.As <IIndexHistoryRepository>()
.SingleInstance();
builder.Register(c => new IndexStateRepository(
AzureTableStorage <IndexStateEntity> .Create(_connectionString,
nameof(IndexState), c.Resolve <ILogFactory>())))
.As <IIndexStateRepository>()
.SingleInstance();
builder.Register(c => new FirstStateAfterResetTimeRepository(
AzureTableStorage <FirstStateAfterResetTimeEntity> .Create(_connectionString,
"FirstStateAfterResetTime", c.Resolve <ILogFactory>())))
.As <IFirstStateAfterResetTimeRepository>()
.SingleInstance();
builder.Register(c => new WarningRepository(
AzureTableStorage <WarningEntity> .Create(_connectionString,
nameof(Warning), c.Resolve <ILogFactory>())))
.As <IWarningRepository>()
.SingleInstance();
builder.Register(c => new ChartHistory5DRepository(
AzureTableStorage <HistoryPointEntity> .Create(_connectionString,
"ChartHistory5D", c.Resolve <ILogFactory>())))
.As <IChartHistory5DRepository>()
.SingleInstance();
builder.Register(c => new ChartHistory30DRepository(
AzureTableStorage <HistoryPointEntity> .Create(_connectionString,
"ChartHistory30D", c.Resolve <ILogFactory>())))
.As <IChartHistory30DRepository>()
.SingleInstance();
// Services
builder.RegisterType <TickPricesService>()
.As <ITickPricesService>()
.As <ITickPriceHandler>()
.SingleInstance();
builder.RegisterType <CoinMarketCapClient>()
.As <ICoinMarketCapClient>()
.WithParameter(TypedParameter.From(new CoinMarketCap.Client.Settings(_settings.CoinMarketCapApiKey)))
.SingleInstance();
builder.RegisterType <CoinMarketCapService>()
.As <ICoinMarketCapService>()
.SingleInstance();
builder.RegisterType <SettingsService>()
.As <ISettingsService>()
.WithParameter("indexTickPriceAssetPair", _settings.IndexName)
.SingleInstance();
builder.RegisterType <IndexCalculator>()
.As <IIndexCalculator>()
.As <IStartable>()
.As <IStopable>()
.WithParameter("indexName", _settings.IndexName)
.WithParameter("shortIndexName", _settings.ShortIndexName)
.WithParameter("isShortIndexEnabled", _settings.IsShortIndexEnabled)
.WithParameter("indexCalculationInterval", _settings.IndexCalculationInterval)
.SingleInstance();
builder.RegisterType <StatisticsService>()
.As <IStatisticsService>()
.As <IIndexHandler>()
.SingleInstance();
}
public async Task QueryUserActivitySince_Test()
{
var entires = AzureTableStorage.QueryUserActivitySince("1292686234187122", DateTime.UtcNow - TimeSpan.FromMinutes(10)).ToList();
}
public AccountTransactionsReportsRepository(IReloadingManager <Settings> settings, ILog log)
{
_tableStorage = AzureTableStorage <AccountTransactionsReportsEntity> .Create(settings.Nested(s => s.Db.ReportsConnString),
"MarginTradingAccountTransactionsReports", log);
}
public BalanceRepository(IReloadingManager <string> connectionStringManager, ILog log)
{
_table = AzureTableStorage <BalanceEntity> .Create(connectionStringManager, "Balances", log);
}
public async Task TestTableStorageServiceRoundTrip()
{
BuildableItemTableStorage ts = (BuildableItemTableStorage)storageInfo.ToTableStorage(bitem);
AzureTableStorage storage = new AzureTableStorage();
await storage.SetTable(storageInfo.TableName);
await storage.InsertOrReplace<BuildableItemTableStorage>(ts);
BuildableItemTableStorage storageItem = await storage.Get<BuildableItemTableStorage>(storageInfo.PartitionKey, storageInfo.RowKey);
StorageItem flip = (StorageItem)storageInfo.FromTableStorage(storageItem);
Assert.AreEqual(BuildableItemEnum.White_Pizza_Dough, flip.ItemCode);
Assert.AreEqual(1, flip.Stats.Count);
Assert.AreEqual(2, flip.Stats[0].RequiredLevel);
Assert.AreEqual(50, flip.Stats[0].CoinCost);
Assert.AreEqual(100, flip.Stats[0].Experience);
Assert.AreEqual(60, flip.Stats[0].BuildSeconds);
Assert.AreEqual(0, flip.Stats[0].CouponCost);
Assert.AreEqual(1, flip.Stats[0].SpeedUpCoupons);
Assert.AreEqual(60, flip.Stats[0].SpeedUpSeconds);
Assert.AreEqual(3, flip.Stats[0].RequiredItems.Count);
Assert.AreEqual(BuildableItemEnum.White_Flour, flip.Stats[0].RequiredItems[0].ItemCode);
Assert.AreEqual(1, flip.Stats[0].RequiredItems[0].StoredQuantity);
Assert.AreEqual(BuildableItemEnum.Salt, flip.Stats[0].RequiredItems[1].ItemCode);
Assert.AreEqual(1, flip.Stats[0].RequiredItems[1].StoredQuantity);
Assert.AreEqual(BuildableItemEnum.Yeast, flip.Stats[0].RequiredItems[2].ItemCode);
Assert.AreEqual(1, flip.Stats[0].RequiredItems[2].StoredQuantity);
Assert.AreEqual(1, flip.StorageStats.Count);
Assert.AreEqual(2, flip.StorageStats[0].Capacity);
}
protected override void Load(ContainerBuilder builder)
{
builder.RegisterType <AssetsService>()
.WithParameter(TypedParameter.From(_config.Cache.AssetsCacheDuration))
.As <IAssetsService>()
.As <IStartable>()
.AutoActivate();
builder.RegisterType <OrderbooksService>()
.As <IOrderbooksService>()
.WithParameter(TypedParameter.From(_config.Redis.OrderBooksCacheKeyPattern))
.SingleInstance();
var cache = new RedisCache(new RedisCacheOptions
{
Configuration = _config.Redis.RedisConfiguration,
InstanceName = _config.Redis.InstanceName
});
builder.RegisterInstance(cache)
.As <IDistributedCache>()
.SingleInstance();
builder.RegisterMarketDataClient(new MarketDataServiceClientSettings {
GrpcServiceUrl = _config.Services.MarketDataGrpcServiceUrl
});
builder.Register(ctx =>
{
var logger = ctx.Resolve <ILoggerFactory>();
return(logger.ToLykke());
}).As <ILogFactory>();
builder.RegisterMeClient(_config.MatchingEngine.GetIpEndPoint());
builder.RegisterType <KeyUpdateSubscriber>()
.As <IStartable>()
.AutoActivate()
.WithParameter("connectionString", _config.RabbitMq.HftInternal.ConnectionString)
.WithParameter("exchangeName", _config.RabbitMq.HftInternal.ExchangeName)
.SingleInstance();
builder.RegisterHftInternalClient(_config.Services.HftInternalServiceUrl);
builder.RegisterType <TokenService>()
.As <ITokenService>()
.SingleInstance();
builder.RegisterType <BalanceService>()
.As <IBalanceService>()
.SingleInstance();
builder.RegisterType <ValidationService>()
.AsSelf()
.SingleInstance();
builder.RegisterType <IdempotencyService>()
.AsSelf()
.SingleInstance();
var reconnectTimeoutInSec = Environment.GetEnvironmentVariable("NOSQL_PING_INTERVAL") ?? "15";
builder.Register(ctx =>
{
var client = new MyNoSqlTcpClient(() => _config.MyNoSqlServer.ReaderServiceUrl, $"{ApplicationInformation.AppName}-{Environment.MachineName}", int.Parse(reconnectTimeoutInSec));
client.Start();
return(client);
}).AsSelf().SingleInstance();
builder.RegisterInstance(_config.FeeSettings)
.AsSelf();
builder.Register(ctx =>
new MyNoSqlReadRepository <TickerEntity>(ctx.Resolve <MyNoSqlTcpClient>(), _config.MyNoSqlServer.TickersTableName)
).As <IMyNoSqlServerDataReader <TickerEntity> >().SingleInstance();
builder.Register(ctx =>
new MyNoSqlReadRepository <PriceEntity>(ctx.Resolve <MyNoSqlTcpClient>(), _config.MyNoSqlServer.PricesTableName)
).As <IMyNoSqlServerDataReader <PriceEntity> >().SingleInstance();
builder.Register(ctx =>
new MyNoSqlReadRepository <OrderbookEntity>(ctx.Resolve <MyNoSqlTcpClient>(), _config.MyNoSqlServer.OrderbooksTableName)
).As <IMyNoSqlServerDataReader <OrderbookEntity> >().SingleInstance();
builder.Register(ctx =>
new MyNoSqlReadRepository <BalanceEntity>(ctx.Resolve <MyNoSqlTcpClient>(), _config.MyNoSqlServer.BalancesTableName)
).As <IMyNoSqlServerDataReader <BalanceEntity> >().SingleInstance();
builder.RegisterType <PricesStreamService>()
.WithParameter(TypedParameter.From(true))
.AsSelf()
.SingleInstance();
builder.RegisterType <TickersStreamService>()
.WithParameter(TypedParameter.From(true))
.AsSelf()
.SingleInstance();
builder.RegisterType <OrderbookStreamService>()
.WithParameter(TypedParameter.From(true))
.AsSelf()
.SingleInstance();
builder.RegisterType <BalancesStreamService>()
.WithParameter(TypedParameter.From(true))
.AsSelf()
.SingleInstance();
builder.RegisterType <OrdersStreamService>()
.WithParameter(TypedParameter.From(true))
.AsSelf()
.SingleInstance();
builder.RegisterType <TradesStreamService>()
.WithParameter(TypedParameter.From(true))
.AsSelf()
.SingleInstance();
builder.RegisterType <PublicTradesStreamService>()
.WithParameter(TypedParameter.From(true))
.AsSelf()
.SingleInstance();
builder.RegisterType <StreamsManager>().AsSelf().SingleInstance();
builder.RegisterType <SiriusWalletsService>()
.As <ISiriusWalletsService>()
.WithParameter(TypedParameter.From(_config.Services.SiriusApiServiceClient.BrokerAccountId))
.WithParameter(TypedParameter.From(_config.Services.SiriusApiServiceClient.WalletsActiveRetryCount))
.WithParameter(TypedParameter.From(_config.Services.SiriusApiServiceClient.WaitForActiveWalletsTimeout))
.SingleInstance();
builder.RegisterType <TradesSubscriber>()
.As <IStartable>()
.AutoActivate()
.WithParameter("connectionString", _config.RabbitMq.Orders.ConnectionString)
.WithParameter("exchangeName", _config.RabbitMq.Orders.ExchangeName)
.SingleInstance();
builder.Register(ctx =>
new TradesAdapterClient(_config.Services.TradesAdapterServiceUrl,
ctx.Resolve <ILogFactory>().CreateLog(nameof(TradesAdapterClient)))
)
.As <ITradesAdapterClient>()
.SingleInstance();
#pragma warning disable 618
builder.Register(x => new KycStatusServiceClient(_config.Services.KycServiceClient, x.Resolve <ILogFactory>()))
#pragma warning restore 618
.As <IKycStatusService>()
.SingleInstance();
builder.RegisterClientAccountClient(_config.Services.ClientAccountServiceUrl);
builder.RegisterOperationsClient(_config.Services.OperationsServiceUrl);
builder.RegisterClientDialogsClient(_config.Services.ClientDialogsServiceUrl);
builder.RegisterInstance(
new Swisschain.Sirius.Api.ApiClient.ApiClient(_config.Services.SiriusApiServiceClient.GrpcServiceUrl, _config.Services.SiriusApiServiceClient.ApiKey)
).As <Swisschain.Sirius.Api.ApiClient.IApiClient>();
builder.RegisterType <PublicTradesSubscriber>()
.As <IStartable>()
.AutoActivate()
.WithParameter("connectionString", _config.RabbitMq.PublicTrades.ConnectionString)
.WithParameter("exchangeName", _config.RabbitMq.PublicTrades.ExchangeName)
.SingleInstance();
builder.Register(ctx =>
AzureTableStorage <IdempotentEntity> .Create(ConstantReloadingManager.From(_config.Db.DataConnString),
"HftApiIdempotency", ctx.Resolve <ILogFactory>())
).As <INoSQLTableStorage <IdempotentEntity> >().SingleInstance();
}
/// <summary>
/// A way to get the items into the table storage for now.
/// @ TO DO Figure out how we actually want to do this
/// </summary>
public async Task StoreBuildableItems()
{
List<BuildableItem> items = CreateItemList();
List<BuildableItemTableStorage> tableItems = new List<BuildableItemTableStorage>();
foreach (BuildableItem item in items)
{
BuildableItemStorageInformation storageInfo = new BuildableItemStorageInformation(item.ItemCode.ToString());
tableItems.Add((BuildableItemTableStorage)storageInfo.ToTableStorage(item));
}
AzureTableStorage storage = new AzureTableStorage();
await storage.SetTable("BuildableItem");
await storage.DeleteTable();
await storage.SetTable("BuildableItem");
await storage.InsertOrReplace<BuildableItemTableStorage>(tableItems);
}
public async Task TestTableStorageServiceRoundTrip()
{
GamePlayerTableStorage ts = (GamePlayerTableStorage)storageInfo.ToTableStorage(player);
AzureTableStorage storage = new AzureTableStorage();
await storage.SetTable(storageInfo.TableName);
await storage.InsertOrReplace<GamePlayerTableStorage>(ts);
GamePlayerTableStorage storageItem = await storage.Get<GamePlayerTableStorage>(storageInfo.PartitionKey, storageInfo.RowKey);
GamePlayer flip = (GamePlayer)storageInfo.FromTableStorage(storageItem);
Assert.AreEqual(99, flip.Coins);
Assert.AreEqual(66, flip.Coupons);
Assert.AreEqual(1000, flip.Experience);
Assert.AreEqual(4, flip.Level);
Assert.AreEqual(1, flip.WorkInProgress.Count);
Assert.AreEqual(0, flip.WorkInProgress[0].Location);
Assert.AreEqual(BuildableItemEnum.Dough_Mixer, flip.WorkInProgress[0].Quantity.ItemCode);
Assert.AreEqual(1, flip.WorkInProgress[0].Quantity.Level);
Assert.AreEqual(0, flip.WorkInProgress[0].Quantity.StoredQuantity);
Assert.AreEqual(2, flip.WorkInProgress[0].Quantity.UnStoredQuantity);
Assert.AreEqual(1, flip.Locations.Count);
Assert.AreEqual(2, flip.Locations[0].Storage.Items.Count);
Assert.AreEqual(BuildableItemEnum.White_Flour, flip.Locations[0].Storage.Items[BuildableItemEnum.White_Flour].ItemCode);
Assert.AreEqual(0, flip.Locations[0].Storage.Items[BuildableItemEnum.White_Flour].Level);
Assert.AreEqual(2, flip.Locations[0].Storage.Items[BuildableItemEnum.White_Flour].StoredQuantity);
Assert.AreEqual(1, flip.Locations[0].Storage.Items[BuildableItemEnum.White_Flour].UnStoredQuantity);
Assert.AreEqual(10, flip.TutorialStage);
}
