public JsonResult GetUserDocuments(string userId, int taxYear, int?skip, int?amount) { bool downloadSuccessful = true; var results = new List <Tuple <string, string, string> >(); using (var db = new WorktripEntities()) { try { Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)"); var user = db.Users.FirstOrDefault(u => u.Id == userId); var miscDocs = db.UserMiscDocs.Where(d => d.UserId == userId && d.Year == taxYear); var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear); miscDocs = miscDocs.OrderBy(d => d.Id); taxReturn = taxReturn.OrderBy(d => d.Id); if (skip.HasValue) { miscDocs = miscDocs.Skip(skip.Value); taxReturn = taxReturn.Skip(skip.Value); } if (amount.HasValue) { miscDocs = miscDocs.Take(amount.Value); taxReturn = taxReturn.Take(amount.Value); } var fileUrls = miscDocs.Select(d => d.Path).ToList(); fileUrls.AddRange(taxReturn.Select(d => d.Path).ToList()); var parsedFilePaths = new List <Tuple <string, string> >(); foreach (var url in fileUrls) { Match match = filePattern.Match(url); if (match.Success) { var newTuple = new Tuple <string, string>( match.Groups["directory"].Value, match.Groups["filename"].Value ); parsedFilePaths.Add(newTuple); } } CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference("worktripdocs"); CloudFileDirectory rootDir = share.GetRootDirectoryReference(); CloudFileDirectory userDir = null; var userDirName = ""; foreach (var parsedPath in parsedFilePaths) { if (userDirName != parsedPath.Item1) { userDir = rootDir.GetDirectoryReference(parsedPath.Item1); if (!userDir.Exists()) { continue; } userDirName = parsedPath.Item1; } var filename = parsedPath.Item2; CloudFile file = userDir.GetFileReference(filename); if (!file.Exists()) { continue; } file.FetchAttributes(); string fileContents = ""; if (file.Properties.ContentType != null && file.Properties.ContentType.StartsWith("image/")) { MemoryStream fileStream = new MemoryStream(); file.DownloadToStream(fileStream); fileContents = ConvertImageStreamToBase64String(fileStream); } else if (file.Properties.ContentType.ToLower() == "application/pdf") { MemoryStream fileStream = new MemoryStream(); file.DownloadToStream(fileStream); fileContents = ConvertStreamToBase64String(fileStream); } else { fileContents = file.DownloadText(); } results.Add( new Tuple <string, string, string>(filename, file.Properties.ContentType, fileContents) ); } } catch (Exception e) { //Do some error logging here.. downloadSuccessful = false; } } if (downloadSuccessful) { return(Json(new { status = 0, files = results })); } else { return(Json(new { status = -1, message = "Error in downloading files" })); } }
public void CloudStorageAccountClientMethods() { CloudStorageAccount account = new CloudStorageAccount(TestBase.StorageCredentials, false); CloudBlobClient blob = account.CreateCloudBlobClient(); CloudQueueClient queue = account.CreateCloudQueueClient(); CloudTableClient table = account.CreateCloudTableClient(); CloudFileClient file = account.CreateCloudFileClient(); // check endpoints Assert.AreEqual(account.BlobEndpoint, blob.BaseUri, "Blob endpoint doesn't match account"); Assert.AreEqual(account.QueueEndpoint, queue.BaseUri, "Queue endpoint doesn't match account"); Assert.AreEqual(account.TableEndpoint, table.BaseUri, "Table endpoint doesn't match account"); Assert.AreEqual(account.FileEndpoint, file.BaseUri, "File endpoint doesn't match account"); // check storage uris Assert.AreEqual(account.BlobStorageUri, blob.StorageUri, "Blob endpoint doesn't match account"); Assert.AreEqual(account.QueueStorageUri, queue.StorageUri, "Queue endpoint doesn't match account"); Assert.AreEqual(account.TableStorageUri, table.StorageUri, "Table endpoint doesn't match account"); Assert.AreEqual(account.FileStorageUri, file.StorageUri, "File endpoint doesn't match account"); // check creds Assert.AreEqual(account.Credentials, blob.Credentials, "Blob creds don't match account"); Assert.AreEqual(account.Credentials, queue.Credentials, "Queue creds don't match account"); Assert.AreEqual(account.Credentials, table.Credentials, "Table creds don't match account"); Assert.AreEqual(account.Credentials, file.Credentials, "File creds don't match account"); }
public async Task RunPipelineAsync(CancellationToken cancellationToken) { if (parser.Document.SelectToken("$.imageRegistryCredentials") is JArray imageRegistryCredentials) { foreach (JObject imageRegistryCredential in imageRegistryCredentials) { HandleObjReplacement(imageRegistryCredential); logger.LogInformation("using {@imageRegistryCredential}", imageRegistryCredential); } } foreach (var part in parser.Document.SelectToken("$.pipe")) { var command = part.SelectToken("$.command"); var dependsOn = part.SelectToken("$.dependsOn")?.Select(c => c.ToString()).ToArray(); var parallelBlocker = int.MaxValue; var parallelKey = part.SelectToken("$.name").ToString(); if (part.SelectToken("$.parallelContext") is JObject parallelContext) { HandleObjReplacement(parallelContext); parallelBlocker = parallelContext.SelectToken("$.max")?.ToObject <int>() ?? parallelBlocker; parallelKey = parallelContext.SelectToken("$.contextKey")?.ToString() ?? parallelKey; logger.LogInformation("using {@parallelContext}: {parallelKey} {parallelBlocker}", parallelContext, parallelKey, parallelBlocker); } var semaphoreObject = options.GetSemaphore(parallelKey, parallelBlocker); // parser.Functions["parallelContext"] = (document, arguments) => JToken.FromObject(new { counter = semaphoreObject.RunningCounter }); var volumnes = new Dictionary <string, JObject>(); var dependenciesResolved = true; if (dependsOn != null) { foreach (var dependency in dependsOn) { if (dependency.StartsWith("/volumes/")) { var volumeName = dependency.Substring("/volumes/".Length); var volume = parser.Document.SelectToken($"$.volumes.{volumeName}")?.DeepClone(); if (volume is JObject volumeObj) { HandleObjReplacement(volumeObj); if (volume.SelectToken("$.azureFileShare") is JObject azureFileShare) { var storageAccount = new CloudStorageAccount(new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(azureFileShare.SelectToken("$.storageAccountName").ToString(), azureFileShare.SelectToken("$.storageAccountKey").ToString()), true); await storageAccount.CreateCloudFileClient().GetShareReference(azureFileShare.SelectToken("$.shareName").ToString()).CreateIfNotExistsAsync(); } if (volume.SelectToken("$.local") is JObject local) { var uniqueId = local.SelectToken("$.uniqueId")?.ToString(); if (!string.IsNullOrEmpty(uniqueId)) { local.Replace(CreateMD5(uniqueId)); } } //if (volume.SelectToken("$.mountedFrom").ToString().StartsWith("[")) //{ // volume.SelectToken("$.mountedFrom").Replace(parser.Evaluate(volume.SelectToken("$.mountedFrom").ToString())); //} // volumnes.Add(volume.SelectToken("$.mountedAt").ToString(),new EmptyStruct { }); volumnes.Add(volumeName, volumeObj); } else { dependenciesResolved = false; break; } } else { var completed = parser.Document.SelectToken($"$.pipe[?(@.name == '{dependency}')].outputs.completed")?.ToObject <bool>() ?? false; if (!completed) { dependenciesResolved = false; break; } } } if (!dependenciesResolved) { continue; } } parser.Functions["mountedAt"] = (d, a) => volumnes[a.First().ToString()]["mountedAt"].ToString(); parser.Functions["mountedFrom"] = (d, a) => volumnes[a.First().ToString()]["mountedFrom"].ToString(); var commands = new List <string[]>() { }; var loop = part.SelectToken("loop"); if (loop != null) { if (loop.Type == JTokenType.String && loop.ToString().StartsWith("[")) { loop = parser.Evaluate(loop.ToString()); } logger.LogInformation("using {@loop}", loop); foreach (var loopVar in loop) { parser.Functions["loop"] = (document, arguments) => arguments[0].ToString() == "var" ? loopVar: Path.GetFileName(loopVar.ToString()); var skip = part.SelectToken("$.skip"); if (skip == null || !skip.ToString().StartsWith("[") || !parser.Evaluate(skip.ToString()).ToObject <bool>()) { commands.Add(command.Select(c => c.ToString().StartsWith("[") ? parser.Evaluate(c.ToString())?.ToString() : c.ToString()).ToArray()); } } } else { var skip = part.SelectToken("$.skip"); if (skip == null || !skip.ToString().StartsWith("[") || !parser.Evaluate(skip.ToString()).ToObject <bool>()) { commands.Add(command.Select(c => c.ToString().StartsWith("[") ? parser.Evaluate(c.ToString())?.ToString() : c.ToString()).ToArray()); } } if (!command.Any()) { var outputs = part["outputs"] as JObject ?? new JObject(); outputs["completed"] = true; part["outputs"] = outputs; Console.WriteLine(part.ToString()); continue; } var image = part.SelectToken("$.image").ToString(); if (image.StartsWith("[")) { part["image"] = parser.Evaluate(image).ToString(); } var block = new ActionBlock <string[]>(async arguments => { try { semaphoreObject.WaitOne(); var binds = volumnes.Select((v, i) => $"{GetMountedFrom(v)}:{v.Value.SelectToken("$.mountedAt").ToString()}").ToList(); var runCommand = $"docker run {string.Join(" ", binds.Select(b => $"-v {b}"))} {part.SelectToken("$.image").ToString()} {string.Join(" ", arguments)}"; logger.LogInformation(runCommand); string stdOut = await executor.ExecuteStepAsync(parser, arguments, part, volumnes, cancellationToken); parser.Functions["stdout"] = (d, o) => stdOut; var outputs = part["outputs"] as JObject ?? new JObject(); foreach (var outputProperty in outputs.Properties()) { if (outputProperty.Value.ToString().StartsWith("[")) { outputProperty.Value.Replace(parser.Evaluate(outputProperty.Value.ToString())); } } //Console.WriteLine("stdout:"); // Console.WriteLine(stdOut.ToString()); // await client.Containers.RemoveContainerAsync(container.ID, new ContainerRemoveParameters { RemoveVolumes = true, RemoveLinks = true, Force = true }); // Console.WriteLine(part.ToString()); } finally { semaphoreObject.Release(); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = parser.Evaluate("[parameters('numberOfProcesses')]").ToObject <int>() }); // var prop = command.Parent; // Console.WriteLine(command.ToString()); foreach (var arguments in commands) { await block.SendAsync(arguments); } block.Complete(); await block.Completion; { var outputs = part["outputs"] as JObject ?? new JObject(); outputs["completed"] = true; part["outputs"] = outputs; logger.LogInformation("pipeline completed with {output}", outputs); } } await executor.PipelineFinishedAsync(parser); }
// ************************************************************************************************************************* // Instructions: This sample can be run against Microsoft Azure Storage Service by updating the App.Config with your AccountName and AccountKey. // // To run the sample using the Storage Service // 1. Create a Storage Account through the Azure Portal and provide your [AccountName] and [AccountKey] in // the App.Config file. See http://go.microsoft.com/fwlink/?LinkId=325277 for more information // 2. Set breakpoints and run the project using F10. // // ************************************************************************************************************************* /// <summary> /// Basic operations to work with Azure Files /// </summary> /// <returns>Task</returns> public async Task BasicAzureFileOperationsAsync() { const string DemoShare = "demofileshare"; const string DemoDirectory = "demofiledirectory"; const string ImageToUpload = "HelloWorld.png"; // Retrieve storage account information from connection string // How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create a file client for interacting with the file service. CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); // Create a share for organizing files and directories within the storage account. Console.WriteLine("1. Creating file share"); CloudFileShare share = fileClient.GetShareReference(DemoShare); try { await share.CreateIfNotExistsAsync(); } catch (StorageException) { Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample."); Console.WriteLine("Press any key to exit"); Console.ReadLine(); throw; } // Get a reference to the root directory of the share. CloudFileDirectory root = share.GetRootDirectoryReference(); // Create a directory under the root directory Console.WriteLine("2. Creating a directory under the root directory"); CloudFileDirectory dir = root.GetDirectoryReference(DemoDirectory); await dir.CreateIfNotExistsAsync(); // Uploading a local file to the directory created above Console.WriteLine("3. Uploading a file to directory"); CloudFile file = dir.GetFileReference(ImageToUpload); await file.UploadFromFileAsync(ImageToUpload); // List all files/directories under the root directory Console.WriteLine("4. List Files/Directories in root directory"); List <IListFileItem> results = new List <IListFileItem>(); FileContinuationToken token = null; do { FileResultSegment resultSegment = await share.GetRootDirectoryReference().ListFilesAndDirectoriesSegmentedAsync(token); results.AddRange(resultSegment.Results); token = resultSegment.ContinuationToken; }while (token != null); // Print all files/directories listed above foreach (IListFileItem listItem in results) { // listItem type will be CloudFile or CloudFileDirectory Console.WriteLine("- {0} (type: {1})", listItem.Uri, listItem.GetType()); } // Download the uploaded file to your file system Console.WriteLine("5. Download file from {0}", file.Uri.AbsoluteUri); await file.DownloadToFileAsync(string.Format("./CopyOf{0}", ImageToUpload), FileMode.Create); // Clean up after the demo Console.WriteLine("6. Delete file"); await file.DeleteAsync(); // When you delete a share it could take several seconds before you can recreate a share with the same // name - hence to enable you to run the demo in quick succession the share is not deleted. If you want // to delete the share uncomment the line of code below. // Console.WriteLine("7. Delete Share"); // await share.DeleteAsync(); }
/// <summary> /// Deploy Batch resourses for cloud computing. Open a batch client. /// </summary> public static async Task InitializeAsync() { try { Console.WriteLine("Optimization / Azure Start: {0}", DateTime.Now); Console.WriteLine(); _timer = Stopwatch.StartNew(); // This will boost parallel submission speed for REST APIs. If your use requires many simultaneous service calls set this number to something large, such as 100. // See: https://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit.aspx for more info. System.Net.ServicePointManager.DefaultConnectionLimit = 100; // Create a Batch client and authenticate with shared key credentials. // The Batch client allows the app to interact with the Batch service. var batchAccountUrl = Shared.Config.BatchAccountUrl; var batchAccountName = Shared.Config.BatchAccountName; var batchAccountKey = Shared.Config.BatchAccountKey; BatchSharedKeyCredentials sharedKeyCredentials = new BatchSharedKeyCredentials(batchAccountUrl, batchAccountName, batchAccountKey); BatchClient = BatchClient.Open(sharedKeyCredentials); // Construct the Storage account connection string string storageConnectionString = $"DefaultEndpointsProtocol=https;AccountName={Shared.Config.StorageAccountName};AccountKey={Shared.Config.StorageAccountKey}"; // Retrieve the storage account object CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); // Create the blob client, to reference the blob storage containers BlobClient = storageAccount.CreateCloudBlobClient(); // Create File Client, to access Azure Files FileClient = storageAccount.CreateCloudFileClient(); // Create share if not exist DataFileShare = FileClient.GetShareReference(DataFileShareName); await DataFileShare.CreateIfNotExistsAsync(); // Synchronize data between data folder and Cloud File Share Console.Write("Synchronize data with file share? [yes] no: "); string response = Console.ReadLine()?.ToLower(); if (response != "n" && response != "no") { await SynchronizeHistoricalDataWithFileShareAsync(FileClient); } // Creat Containers: OutPut Container (where tasks will upload results) and Container for Algorithm DLL // Obtain a shared access signature that provides write access to the output await CreateContainerIfNotExistAsync(BlobClient, DllContainerName); await CreateContainerIfNotExistAsync(BlobClient, OutputContainerName); OutputContainerSasUrl = GetContainerSasUrl(BlobClient, OutputContainerName, SharedAccessBlobPermissions.Write); // Create the Batch pool, if not exist, which contains the compute nodes that execute the tasks. await CreatePoolIfNotExistAsync(BatchClient, PoolId); // Create the job that runs the tasks. await CreateJobAsync(BatchClient, JobId, PoolId); } // Catch for an aggregate exception catch (AggregateException ae) { // Flatten agregates all inner exception in one foreach (var innerException in ae.Flatten().InnerExceptions) { Shared.Logger.Error($"AzureBatchManager.DeployAsync(): {innerException.Message}"); } throw; } }
public FileService(CloudStorageAccount cloudStorageClient) { this._cloudFileClient = cloudStorageClient.CreateCloudFileClient(); }
public async Task <IActionResult> GetFile() { try { var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true); IConfigurationRoot configuration = builder.Build(); var allCitDebtsLines = new string[10000000]; // while (true) //{ // if (DateTime.Now.AddHours(3).ToShortTimeString() == "09:58:00 aM") // { //List<string> fileNames = new List<string>(); //List<string[]> lstCloudFilesdata = new List<string[]>(); CloudStorageAccount storageAccount = CloudStorageAccount.Parse($"{configuration["ConnectionString1"]}"); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare fileShare = fileClient.GetShareReference("import"); //looks for a file share in the cloud bool fileShareExists = await fileShare.ExistsAsync(); if (fileShareExists) { List <CloudFile> lstCloudFiles = new List <CloudFile>(); CloudFileDirectory rootDir = fileShare.GetRootDirectoryReference(); List <string> sl = new List <string>(); DeptResults DeptResults = new DeptResults(); List <ErrorLines> ErrorLines = new List <ErrorLines>(); ErrorLines ErrorLine = new ErrorLines(); string bigfilename = "CitizenDebts_1M_3Big.txt"; string fileName = "CitizenDebts_1M_3.txt"; //"DEBTS_" + DateTime.Now.ToString("yyyyMMdd") + ".txt"; CloudFile file = rootDir.GetFileReference(fileName); string checkfile = bigfilename; //if the file exists bool asd = await file.ExistsAsync(); if (asd) { //adds new datasting array sl = await ReadDataAsync(file); if (sl is null) { //return NotFound(HttpStatusCode.NotFound + "\n" + "\nΣφάλμα\nΤο αρχείο δεν περιέχει σωστό αριθμό στηλών</b>"); Redirect(DeptResults.BillsCount = 0, DeptResults.NewUsers = 0, HttpStatusCode.NotFound.ToString(), null, "\n" + "\nΣφάλμα\nΤο αρχείο δεν περιέχει σωστό αριθμό στηλών</b>"); } } else { Redirect(DeptResults.BillsCount = 0, DeptResults.NewUsers = 0, HttpStatusCode.NotFound.ToString(), null, "\n" + "\nΣφάλμα\nΔεν βρέθηκε το αρχείο</b>"); // return NotFound(HttpStatusCode.NotFound+ "\n" + "\nΣφάλμα\nΔεν βρέθηκε το αρχείο</b>"); } Console.WriteLine("File into List " + DateTime.Now.ToString()); //foreach (string y in sl) //{ Console.WriteLine("From list new : " + y); }; string[] cols; for (int i = sl.Count - 1; i >= 0; i--) { cols = sl.ElementAt(i).Split(';'); if (cols[0].Trim().Length != 10 || !cols[0].All(char.IsDigit)) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος ΑΦΜ"; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); continue; //return NotFound(HttpStatusCode.NotFound + "\n" + "\nΣφάλμα Λάθος ΑΦΜ");} } if (cols[1].Trim().Length == 0) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Όνομα "; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); //return NotFound(HttpStatusCode.NotFound + "\n" + "\nΣφάλμα Λάθος Όνομα "); } if (cols[2].Trim().Length == 0) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Επώνυμο "; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); } if (cols[3].Trim().Length == 0 || !Regex.IsMatch(cols[3], @"\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*")) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Email "; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); } if (cols[4].Trim().Length == 0 || !cols[4].All(char.IsDigit)) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Τηλέφωνο "; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); } if (cols[5].Trim().Length == 0) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Διεύθυσνη "; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); } if (cols[6].Trim().Length == 0) { sl.RemoveAt(i); return(NotFound(HttpStatusCode.NotFound + "\n" + "\nΣφάλμα Λάθος Περιοχή ")); } //!Regex.IsMatch(cols[7], @"^(\{{0,1}([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}\}{0,1})$") if (cols[7].Trim().Length == 0) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Αρ.Λογαριασμού "; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); } if (cols[8].Trim().Length == 0) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Περιγραφή Λογαριασμού"; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); } decimal number; if (cols[9].Trim().Length == 0 || !Decimal.TryParse(cols[9], out number) || cols[9].Contains('.')) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Ποσό"; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); } DateTime d; if (cols[10].Trim().Length == 0 || !DateTime.TryParseExact(cols[10], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out d)) { ErrorLine.line = i; ErrorLine.ErrorMessage = "Σφάλμα Λάθος Ημερομηνία"; ErrorLine.LineString = sl[i]; ErrorLines.Add(ErrorLine); sl.RemoveAt(i); } } IEnumerable <CitizenDepts> o = from eachLine in ( from inner in sl select inner.Split(';') ) select new CitizenDepts { VAT = eachLine[0], FirstName = eachLine[1], LastName = eachLine[2], Email = eachLine[3], Phone = eachLine[4], Address = eachLine[5], County = eachLine[6], BillId = eachLine[7], Bill_description = eachLine[8], //Amount = Decimal.Parse(eachLine[9]), Amount = Decimal.Parse(eachLine[9], System.Globalization.CultureInfo.GetCultureInfo("el-GR")), DueDate = DateTime.ParseExact(eachLine[10], "yyyyMMdd", CultureInfo.InvariantCulture) }; Console.WriteLine("File splitted " + DateTime.Now.ToString()); var all = from c in _context.CitizenDepts select c; _context.CitizenDepts.RemoveRange(all); await _context.SaveChangesAsync(); Console.WriteLine("CitizenDept table deleted " + DateTime.Now.ToString()); foreach (var p in o) { _context.Add(p); } Console.WriteLine("Context filledup " + DateTime.Now.ToString()); await _context.SaveChangesAsync(); Console.WriteLine("Context saved to DB " + DateTime.Now.ToString()); //filter citizens NewCitizens = o. Where(x => !((_context.ApplicationUser.Any(y => y.VAT == x.VAT)) || (_context.ApplicationUser.Any(y => y.Email == x.Email)))) .Select(x => x).AsEnumerable(); Console.WriteLine("Citizens filterd " + DateTime.Now.ToString()); foreach (var p in NewCitizens) { Console.WriteLine(p.VAT + " , " + p.Email + " , " + p.LastName + " , " + p.Bill_description); } Console.WriteLine("Citizens log " + DateTime.Now.ToString()); //only new citizens foreach (var p in NewCitizens.Distinct()) { var user = new ApplicationUser { UserName = p.Email, Email = p.Email }; var TempPass = GeneratePassword(3, 3, 3, 3); // int lowercase, int uppercase, int numerics, int symbols Console.WriteLine(TempPass); var result = await _userManager.CreateAsync(user, TempPass); if (result.Succeeded) { if (checkfile != "CitizenDebts_1M_3.txt") { Console.WriteLine("Sending Emails"); SendMail(p.LastName, p.Email, TempPass); } var query = from UserUpd in _context.ApplicationUser where UserUpd.Email == p.Email select UserUpd; foreach (ApplicationUser UserUpd in query) { UserUpd.VAT = p.VAT; UserUpd.IsFirst = true; UserUpd.LastName = p.LastName; UserUpd.FirstName = p.FirstName; UserUpd.Phone = p.Phone; UserUpd.County = p.County; UserUpd.Address = p.Address; } } else { Console.WriteLine("#############################ALREADY REGISTERED " + p.VAT + " , " + p.Email + " , " + p.LastName); } NewUsers++; } ; await _context.SaveChangesAsync(); Console.WriteLine("New Users Registered and Emailed " + DateTime.Now.ToString()); foreach (var a in _context.ApplicationUser) { var query2 = from UIdUpd in _context.CitizenDepts where UIdUpd.VAT == a.VAT select UIdUpd; foreach (CitizenDepts UIdUpd in query2) { UIdUpd.UserGUId = a.Id; } } ; await _context.SaveChangesAsync(); List <Bills> NewBillsls = new List <Bills>(); Bills NewBill = new Bills(); List <CitizenDepts> UpdCit = new List <CitizenDepts>(); UpdCit.AddRange(_context.CitizenDepts); foreach (var e in UpdCit) { Console.WriteLine("#############################CITIZEN_DEPTS " + e.VAT + " , " + e.Email + " , " + e.UserGUId); NewBill.GuId = e.BillId; NewBill.Amount = e.Amount; NewBill.DueDate = e.DueDate; NewBill.Bill_description = e.Bill_description; NewBill.Status = 0; NewBill.UserId = e.UserGUId; NewBill.PaymentMethodId = 1; NewBill.SettlementId = 1; NewBillsls.Add(NewBill); NewBill = new Bills(); } //.... delete bills var allBills = from c in _context.Bills select c; _context.Bills.RemoveRange(allBills); await _context.SaveChangesAsync(); //.... delete settlements _context.Settlements.RemoveRange( _context.Settlements .Where(s => s.ID != 1) ); await _context.SaveChangesAsync(); _context.Bills.AddRange(NewBillsls); await _context.SaveChangesAsync(); string Mes; if (ErrorLines.Count() > 0) { Mes = "Ύπήρχαν " + ErrorLines.Count() + " σφάλματα στην εισαγωγή του αρχείου"; } else { Mes = "Δεν Υπήρχαν Σφάλματα στην εισαγωγή του αρχείου"; } // return View(NewCitizens); return(RedirectToAction("DeptResults", "DeptResults", new DeptResults { BillsCount = NewBillsls.Count(), NewUsers = NewUsers, HttpStatus = "", ErrorLines = ErrorLines, Message = Mes })); } else { return(NotFound(HttpStatusCode.NotFound + "\n" + "\nΔεν βρέθηκε ο κοινός φάκελος!")); } } catch (Exception ex) { return(NotFound(HttpStatusCode.ExpectationFailed + "\n" + ex.Message + "\nΣφάλμα\nΑπροσδιόριστο σφάλμα Στην Εισαγωγή του αρχείου</b>")); } finally { Console.WriteLine("Import Finished Successfully " + DateTime.Now.ToString()); } return(View()); }
internal IEnumerable <string> UploadResourceFilesToAzureFileStorage(List <Entity> resources, XDocument importXml, Configuration config, string folderDateTime, inRiverContext context) { Stopwatch saveFileStopWatch = new Stopwatch(); List <string> cloudFileNames = new List <string>(); try { // validate configuration settings if (string.IsNullOrEmpty(config.StorageAccountName) || string.IsNullOrEmpty(config.StorageAccountKey) || string.IsNullOrEmpty(config.StorageAccountShareReference) || string.IsNullOrEmpty(config.StorageAccountCatalogDirectoryReference) || string.IsNullOrEmpty(config.StorageAccountResourcesDirectoryReference)) { context.Log(LogLevel.Warning, $"Azure config settings are invalid: " + $"StorageAccountName: {config.StorageAccountName}, " + $"StorageAccountKey: {config.StorageAccountKey}, " + $"StorageAccountShareReference: {config.StorageAccountShareReference}, " + $"StorageAccountCatalogDirectoryReference: {config.StorageAccountCatalogDirectoryReference}, " + $"StorageAccountResourcesDirectoryReference: {config.StorageAccountResourcesDirectoryReference}"); return(cloudFileNames); } // validate resources argument if (resources == null) { context.Log(LogLevel.Error, "Resource is null!"); return(cloudFileNames); } // create varible files to hold filename and binary Dictionary <string, byte[]> files = new Dictionary <string, byte[]>(); // setup credentials and storage account StorageCredentials credentials = new StorageCredentials(config.StorageAccountName, config.StorageAccountKey); CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true); // setup file client and remoge share CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference(config.StorageAccountShareReference); share.CreateIfNotExists(); // setup root directory and resource directory CloudFileDirectory root = share.GetRootDirectoryReference(); CloudFileDirectory directory = root.GetDirectoryReference(config.GetAzureStorageDirectoryName(XmlDocumentType.Resources)); directory.CreateIfNotExists(); saveFileStopWatch.Start(); // setup resource helper object Resources resourcesObj = new Resources(context); // setup the uncompressed file size total counter int totalFileSize = 0; // setup the azure file counter int fileCount = 0; // setup the file id list List <int> fileIdList = new List <int>(); foreach (Entity resource in resources) { // get the file id of the resource int resourceFileId = resourcesObj.GetResourceFileId(resource); // ensure the resource id has a proper id if (resourceFileId < 0) { context.Log(LogLevel.Information, $"Resource with id:{resource.Id} has no value for ResourceFileId"); continue; } // loop through each display configurations foreach (string displayConfiguration in resourcesObj.GetDisplayConfigurations(resource, config)) { // setup the fileName to use the output file extension from display configuration string fileName = resourcesObj.GetResourceFileName(resource, resourceFileId, displayConfiguration, config); // if the file for some reason already exists, continue to the next display configuration if (files.ContainsKey(fileName)) { context.Log(LogLevel.Debug, $"{fileName} already exists in the files collection and is skipped"); continue; } // get file bytes byte[] resourceData = context.ExtensionManager.UtilityService.GetFile(resourceFileId, displayConfiguration); // make sure we recieved the file from the utility service if (resourceData == null) { context.Log(LogLevel.Error, $"Resource with id:{resource.Id} and ResourceFileId: {resourceFileId} could not get file"); continue; } // add the current resource file id to the list fileIdList.Add(resource.Id); // log the resource file name context.Log(LogLevel.Debug, $"Adding resource {displayConfiguration}/{fileName}"); // add the file to the files collection files.Add($"{displayConfiguration}/{fileName}", resourceData); // add size to total file size counter totalFileSize += resourceData.Length; } if (totalFileSize > (config.TotalResourceSizeLimitMb * 1024 * 1024)) { try { // increase file counter fileCount++; // setup remote zip file CloudFile cloudFile = directory.GetFileReference(GetZipFileName(config, XmlDocumentType.Resources, folderDateTime, fileCount)); // setup reader using (XmlReader reader = importXml.CreateReader()) { // create a new file that only contains the elements specified in the file id list XmlDocument doc = XmlSplitUtility.GetPartialResourcesXml(reader, fileIdList); // log the resource file name context.Log(LogLevel.Debug, "Adding Resources.xml"); // setup memory a stream using (MemoryStream stream = new MemoryStream()) { // create a xml writer to format output using (XmlWriter writer = XmlWriter.Create(stream, new XmlWriterSettings { Indent = true })) { // save partial document to the xml writer doc.Save(writer); // add the partial document to the files collection files.Add("Resources.xml", stream.ToArray()); } } // send the zipped file to Azure and store the file name cloudFileNames.Add(ZipAndUploadToCloud(config, context, files, directory, cloudFile)); // clear the id list fileIdList.Clear(); // clear the file list files.Clear(); // reset total file size totalFileSize = 0; } } catch (Exception ex) { context.Log(LogLevel.Error, "An error occured while sending the resources to the cloud.", ex); } } } // make sure to send the final files if (files.Any()) { try { // increase file counter fileCount++; // setup remote zip file CloudFile cloudFile = directory.GetFileReference(GetZipFileName(config, XmlDocumentType.Resources, folderDateTime, fileCount)); // setup reader using (XmlReader reader = importXml.CreateReader()) { // create a new file that only contains the elements specified in the file id list XmlDocument doc = XmlSplitUtility.GetPartialResourcesXml(reader, fileIdList); // log the resource file name context.Log(LogLevel.Debug, "Adding Resources.xml"); // setup memory a stream using (MemoryStream stream = new MemoryStream()) { // create a xml writer to format output using (XmlWriter writer = XmlWriter.Create(stream, new XmlWriterSettings { Indent = true })) { // save partial document to the xml writer doc.Save(writer); // add the partial document to the files collection files.Add("Resources.xml", stream.ToArray()); } } // send the zipped file to Azure and store the file name cloudFileNames.Add(ZipAndUploadToCloud(config, context, files, directory, cloudFile)); } } catch (Exception ex) { context.Log(LogLevel.Error, "An error occured while sending the resources to the cloud.", ex); } } } catch (Exception ex) { context.Log(LogLevel.Error, "An error occured while sending the resources to the cloud.", ex); } return(cloudFileNames); }
public static bool ZipDocumentAndUploadToAzure(XmlDocumentType xmlDocumentType, XDocument doc, Configuration config, string dateTimeStamp, IDictionary <string, byte[]> files = null) { if (IsNullOrEmpty(config.StorageAccountName) || IsNullOrEmpty(config.StorageAccountKey) || IsNullOrEmpty(config.StorageAccountShareReference) || IsNullOrEmpty(config.StorageAccountCatalogDirectoryReference) || IsNullOrEmpty(config.StorageAccountResourcesDirectoryReference)) { return(false); } StorageCredentials cred = new StorageCredentials(config.StorageAccountName, config.StorageAccountKey); CloudStorageAccount storageAccount = new CloudStorageAccount(cred, true); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference(config.StorageAccountShareReference); share.CreateIfNotExists(); CloudFileDirectory root = share.GetRootDirectoryReference(); CloudFileDirectory dir = root.GetDirectoryReference(config.GetAzureStorageDirectoryName(xmlDocumentType)); dir.CreateIfNotExists(); CloudFile cloudFile = dir.GetFileReference(GetZipFileName(config, xmlDocumentType, dateTimeStamp)); using (MemoryStream stream = new MemoryStream()) { XmlWriterSettings xws = new XmlWriterSettings { OmitXmlDeclaration = false, Indent = true }; using (ZipArchive archive = new ZipArchive(stream, ZipArchiveMode.Create, true)) { if (files != null) { foreach (KeyValuePair <string, byte[]> imageFile in files) { ZipArchiveEntry zipEntry = archive.CreateEntry(imageFile.Key); using (Stream entryStream = zipEntry.Open()) { entryStream.Write(imageFile.Value, 0, imageFile.Value.Length); } } } ZipArchiveEntry entry = archive.CreateEntry("catalog.xml"); using (Stream entryStream = entry.Open()) { using (XmlWriter xw = XmlWriter.Create(entryStream, xws)) { doc.WriteTo(xw); } } } stream.Position = 0; cloudFile.UploadFromStream(stream); switch (xmlDocumentType) { case XmlDocumentType.Catalog: config.CatalogPathInCloud = cloudFile.Name; break; default: config.ResourceNameInCloud = cloudFile.Name; break; } return(true); } }
// Please set the following connection strings in app.config for this WebJob to run: // AzureWebJobsDashboard and AzureWebJobsStorage static void Main() { CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureWebJobsStorage")); // Create a CloudFileClient object for credentialed access to File storage. CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); // Get a reference to the file share we created previously. CloudFileShare share = fileClient.GetShareReference("Log"); // Ensure that the share exists. if (share.Exists()) { //do something } var config = new JobHostConfiguration(); if (config.IsDevelopment) { config.UseDevelopmentSettings(); } config.UseTimers(); _logger.Log(LogLevel.Info, "Service started"); try { if (profile == null) { Console.WriteLine(string.Format("No Profile id : {0}", ProfileId)); _logger.Log(LogLevel.Info, string.Format("No Profile id : {0}", ProfileId)); return; } Instagram.ProfileId = profile.Id; //Instagram.InitPhantomDriver(); Instagram.InitFirefoxDriver(); Instagram.Open("https://www.instagram.com/"); Instagram.InitCookiesAndRefreshPage(profile.ProfileName); if (Instagram.IsNeedToLogin()) { Instagram.Login(profile.Login, profile.Password, profile.ProfileName); } else { _logger.Log(LogLevel.Info, "Service already logged!"); } } catch (Exception ex) { Console.WriteLine(string.Format("Service exception: {0}", ex)); _logger.Log(LogLevel.Error, string.Format("Service exception: {0}", ex)); } var host = new JobHost(config); // The following code ensures that the WebJob will be running continuously host.RunAndBlock(); }
static void Main(string[] args) { int tsOffset = Convert.ToInt32(ConfigurationManager.AppSettings.Get("TimeStampOffset")); long timestamp_from = (long)Functions.ToEpoch(DateTime.UtcNow.AddMinutes(tsOffset)); long timestamp_to = (long)Functions.ToEpoch(DateTime.UtcNow); List <RTVehiclePositions> vehiclePositions = db.Database .SqlQuery <RTVehiclePositions>("RTVehiclePositionsGetActive @timestamp_from, @timestamp_to" , new SqlParameter("@timestamp_from", timestamp_from) , new SqlParameter("@timestamp_to", timestamp_to)) .ToList(); FeedMessage feed = new FeedMessage(); if (vehiclePositions.Count > 0) { foreach (RTVehiclePositions vehiclePosition in vehiclePositions) { TripDescriptor td = new TripDescriptor(); td.TripId = vehiclePosition.trip_id; td.RouteId = vehiclePosition.route_id; td.DirectionId = (uint)vehiclePosition.direction_id; td.StartDate = vehiclePosition.start_date; td.StartTime = vehiclePosition.start_time; Position pos = new Position(); pos.Latitude = (float)vehiclePosition.latitude; pos.Longitude = (float)vehiclePosition.longitude; VehicleDescriptor v = new VehicleDescriptor(); v.Id = vehiclePosition.vehicle_id; v.Label = vehiclePosition.vehicle_label; v.LicensePlate = vehiclePosition.vehicle_license_plate; VehiclePosition vp = new VehiclePosition(); vp.Position = pos; vp.Trip = td; vp.Vehicle = v; vp.CurrentStopSequence = (uint)vehiclePosition.current_stop_sequence; vp.StopId = vehiclePosition.stop_id; vp.Timestamp = (ulong)vehiclePosition.timestamp; FeedEntity entity = new FeedEntity(); entity.Id = vehiclePosition.id.ToString(); entity.Vehicle = vp; feed.Entities.Add(entity); } byte[] objSerialized = Functions.ProtoSerialize(feed); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString")); string filename = "vehicleposition.pb"; // Create a CloudFileClient object for credentialed access to Azure Files. CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); // Get a reference to the file share we created previously. CloudFileShare share = fileClient.GetShareReference("gtfsrt"); share.CreateIfNotExists(); var rootDir = share.GetRootDirectoryReference(); using (var stream = new MemoryStream(objSerialized, writable: false)) { rootDir.GetFileReference(filename).UploadFromStream(stream); } } }
public CloudFileUtil(CloudStorageAccount account) { this.account = account; this.client = account.CreateCloudFileClient(); }
public StorageManager(string settingsString) { StorageAccount = CloudStorageAccount.Parse(settingsString); blobClient = StorageAccount.CreateCloudBlobClient(); fileClient = StorageAccount.CreateCloudFileClient(); }
/// <summary> /// Test some of the file storage operations. /// </summary> private static async Task RunFileStorageOperationsAsync() { try { //***** Setup *****// Console.WriteLine("Getting reference to the storage account."); // Retrieve storage account information from connection string // How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString")); Console.WriteLine("Instantiating file client."); // Create a file client for interacting with the file service. CloudFileClient cloudFileClient = storageAccount.CreateCloudFileClient(); // Create the share name -- use part of a guid in the name so it's most likely to be unique. // This will also be used as the container name for blob storage when copying the file to blob storage. string shareName = "demotest-" + System.Guid.NewGuid().ToString().Substring(0, 12); // Name of folder to put the files in string sourceFolder = "testfolder"; // Name of file to upload and download string testFile = "HelloWorld.png"; // Folder where the HelloWorld.png file resides string localFolder = @".\"; // It won't let you download in the same folder as the exe file, // so use a temporary folder with the same name as the share. string downloadFolder = Path.Combine(Path.GetTempPath(), shareName); //***** Create a file share *****// // Create the share if it doesn't already exist. Console.WriteLine("Creating share with name {0}", shareName); CloudFileShare cloudFileShare = cloudFileClient.GetShareReference(shareName); try { await cloudFileShare.CreateIfNotExistsAsync(); Console.WriteLine(" Share created successfully."); } catch (StorageException exStorage) { WriteException(exStorage); Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample."); Console.WriteLine("Press any key to exit"); Console.ReadLine(); throw; } catch (Exception ex) { Console.WriteLine(" Exception thrown creating share."); WriteException(ex); throw; } //***** Create a directory on the file share *****// // Create a directory on the share. Console.WriteLine("Creating directory named {0}", sourceFolder); // First, get a reference to the root directory, because that's where you're going to put the new directory. CloudFileDirectory rootDirectory = cloudFileShare.GetRootDirectoryReference(); CloudFileDirectory fileDirectory = null; // Set a reference to the file directory. // If the source folder is null, then use the root folder. // If the source folder is specified, then get a reference to it. if (string.IsNullOrWhiteSpace(sourceFolder)) { // There is no folder specified, so return a reference to the root directory. fileDirectory = rootDirectory; Console.WriteLine(" Using root directory."); } else { // There was a folder specified, so return a reference to that folder. fileDirectory = rootDirectory.GetDirectoryReference(sourceFolder); await fileDirectory.CreateIfNotExistsAsync(); Console.WriteLine(" Directory created successfully."); } //***** Upload a file to the file share *****// // Set a reference to the file. CloudFile cloudFile = fileDirectory.GetFileReference(testFile); // Upload a file to the share. Console.WriteLine("Uploading file {0} to share", testFile); // Set up the name and path of the local file. string sourceFile = Path.Combine(localFolder, testFile); if (File.Exists(sourceFile)) { // Upload from the local file to the file share in azure. await cloudFile.UploadFromFileAsync(sourceFile, FileMode.OpenOrCreate); Console.WriteLine(" Successfully uploaded file to share."); } else { Console.WriteLine("File not found, so not uploaded."); } //***** Get list of all files/directories on the file share*****// // List all files/directories under the root directory. Console.WriteLine("Getting list of all files/directories under the root directory of the share."); IEnumerable <IListFileItem> fileList = cloudFileShare.GetRootDirectoryReference().ListFilesAndDirectories(); // Print all files/directories listed above. foreach (IListFileItem listItem in fileList) { // listItem type will be CloudFile or CloudFileDirectory. Console.WriteLine(" - {0} (type: {1})", listItem.Uri, listItem.GetType()); } Console.WriteLine("Getting list of all files/directories in the file directory on the share."); // Now get the list of all files/directories in your directory. // Ordinarily, you'd write something recursive to do this for all directories and subdirectories. fileList = fileDirectory.ListFilesAndDirectories(); // Print all files/directories in the folder. foreach (IListFileItem listItem in fileList) { // listItem type will be CloudFile or CloudFileDirectory. Console.WriteLine(" - {0} (type: {1})", listItem.Uri, listItem.GetType()); } //***** Download a file from the file share *****// // Download the file to the downloadFolder in the temp directory. // Check and if the directory doesn't exist (which it shouldn't), create it. Console.WriteLine("Downloading file from share to local temp folder {0}.", downloadFolder); if (!Directory.Exists(downloadFolder)) { Directory.CreateDirectory(downloadFolder); } // Download the file. await cloudFile.DownloadToFileAsync(Path.Combine(downloadFolder, testFile), FileMode.OpenOrCreate); Console.WriteLine(" Successfully downloaded file from share to local temp folder."); //***** Copy a file from the file share to blob storage, then abort the copy *****// // To really test this, you might have to find a large file, or else the file finishes copying before you can abort the copy. // You need to upload the file to the share. Then you can assign the name of hte file to the testFile variable, and it will use // that file for the copy and abort here. CloudFile cloudFileCopy = fileDirectory.GetFileReference(testFile); // Upload a file to the share. Console.WriteLine("Uploading file {0} to share", testFile); // Set up the name and path of the local file. string sourceFileCopy = Path.Combine(localFolder, testFile); await cloudFileCopy.UploadFromFileAsync(sourceFileCopy, FileMode.OpenOrCreate); Console.WriteLine(" Successfully uploaded file to share."); // Copy the file to blob storage. Console.WriteLine("Copying file to blob storage. Container name = {0}", shareName); // First get a reference to the blob. CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient(); // Get a reference to the blob container and create it if it doesn't already exist. CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(shareName); cloudBlobContainer.CreateIfNotExists(); // Get a blob reference to the target blob. CloudBlob targetBlob = cloudBlobContainer.GetBlobReference(testFile); string copyId = string.Empty; // Get a reference to the file to be copied. cloudFile = fileDirectory.GetFileReference(testFile); // Create a SAS for the file that's valid for 24 hours. // Note that when you are copying a file to a blob, or a blob to a file, you must use a SAS // to authenticate access to the source object, even if you are copying within the same // storage account. string fileSas = cloudFile.GetSharedAccessSignature(new SharedAccessFilePolicy() { // Only read permissions are required for the source file. Permissions = SharedAccessFilePermissions.Read, SharedAccessExpiryTime = DateTime.UtcNow.AddHours(24) }); // Construct the URI to the source file, including the SAS token. Uri fileSasUri = new Uri(cloudFile.StorageUri.PrimaryUri.ToString() + fileSas); // Start the copy of the file to the blob. copyId = await targetBlob.StartCopyAsync(fileSasUri); Console.WriteLine(" File copy started successfully. copyID = {0}", copyId); // Abort the copy of the file to blob storage. // Note that you call Abort on the target object, i.e. the blob, not the file. // If you were copying from one file to another on the file share, the target object would be a file. Console.WriteLine("Cancelling the copy operation."); // Print out the copy state information. targetBlob.FetchAttributes(); Console.WriteLine(" targetBlob.copystate.CopyId = {0}", targetBlob.CopyState.CopyId); Console.WriteLine(" targetBlob.copystate.Status = {0}", targetBlob.CopyState.Status); // Do the actual abort copy. // This only works if the copy is still pending or ongoing. if (targetBlob.CopyState.Status == CopyStatus.Pending) { // Call to stop the copy, passing in the copyId of the operation. // This won't work if it has already finished copying. await targetBlob.AbortCopyAsync(copyId); Console.WriteLine(" Cancelling the copy succeeded."); } else { // If this happens, try a larger file. Console.WriteLine(" Cancellation of copy not performed; copy has already finished."); } // Now clean up after yourself. Console.WriteLine("Deleting the files from the file share."); // Delete the files because cloudFile is a different file in the range sample. cloudFile = fileDirectory.GetFileReference(testFile); cloudFile.DeleteIfExists(); Console.WriteLine("Setting up files to test WriteRange and ListRanges."); //***** Write 2 ranges to a file, then list the ranges *****// // This is the code for trying out writing data to a range in a file, // and then listing those ranges. // Get a reference to a file and write a range of data to it . // Then write another range to it. // Then list the ranges. // Start at the very beginning of the file. long startOffset = 0; // Set the destination file name -- this is the file on the file share that you're writing to. string destFile = "rangeops.txt"; cloudFile = fileDirectory.GetFileReference(destFile); // Create a string with 512 a's in it. This will be used to write the range. int testStreamLen = 512; string textToStream = string.Empty; textToStream = textToStream.PadRight(testStreamLen, 'a'); // Name to be used for the file when downloading it so you can inspect it locally string downloadFile; using (MemoryStream ms = new MemoryStream(Encoding.Default.GetBytes(textToStream))) { // Max size of the output file; have to specify this when you create the file // I picked this number arbitrarily. long maxFileSize = 65536; Console.WriteLine("Write first range."); // Set the stream back to the beginning, in case it's been read at all. ms.Position = 0; // If the file doesn't exist, create it. // The maximum file size is passed in. It has to be big enough to hold // all the data you're going to write, so don't set it to 256k and try to write two 256-k blocks to it. if (!cloudFile.Exists()) { Console.WriteLine("File doesn't exist, create empty file to write ranges to."); // Create a file with a maximum file size of 64k. await cloudFile.CreateAsync(maxFileSize); Console.WriteLine(" Empty file created successfully."); } // Write the stream to the file starting at startOffset for the length of the stream. Console.WriteLine("Writing range to file."); await cloudFile.WriteRangeAsync(ms, startOffset, null); // Download the file to your temp directory so you can inspect it locally. downloadFile = Path.Combine(downloadFolder, "__testrange.txt"); Console.WriteLine("Downloading file to examine."); await cloudFile.DownloadToFileAsync(downloadFile, FileMode.OpenOrCreate); Console.WriteLine(" Successfully downloaded file with ranges in it to examine."); } // Now add the second range, but don't make it adjacent to the first one, or it will show only // one range, with the two combined. Put it like 1000 spaces away. When you get the range back, it will // start at the position at the 512-multiple border prior or equal to the beginning of the data written, // and it will end at the 512-multliple border after the actual end of the data. //For example, if you write to 2000-3000, the range will be the 512-multiple prior to 2000, which is // position 1536, or offset 1535 (because it's 0-based). // And the right offset of the range will be the 512-multiple after 3000, which is position 3072, // or offset 3071 (because it's 0-based). Console.WriteLine("Getting ready to write second range to file."); startOffset += testStreamLen + 1000; //randomly selected number // Create a string with 512 b's in it. This will be used to write the range. textToStream = string.Empty; textToStream = textToStream.PadRight(testStreamLen, 'b'); using (MemoryStream ms = new MemoryStream(Encoding.Default.GetBytes(textToStream))) { ms.Position = 0; // Write the stream to the file starting at startOffset for the length of the stream. Console.WriteLine("Write second range to file."); await cloudFile.WriteRangeAsync(ms, startOffset, null); Console.WriteLine(" Successful writing second range to file."); // Download the file to your temp directory so you can examine it. downloadFile = Path.Combine(downloadFolder, "__testrange2.txt"); Console.WriteLine("Downloading file with two ranges in it to examine."); await cloudFile.DownloadToFileAsync(downloadFile, FileMode.OpenOrCreate); Console.WriteLine(" Successfully downloaded file to examine."); } // Query and view the list of ranges. Console.WriteLine("Call to get the list of ranges."); IEnumerable <FileRange> listOfRanges = await cloudFile.ListRangesAsync(); Console.WriteLine(" Successfully retrieved list of ranges."); foreach (FileRange fileRange in listOfRanges) { Console.WriteLine(" --> filerange startOffset = {0}, endOffset = {1}", fileRange.StartOffset, fileRange.EndOffset); } //***** Clean up *****// // Clean up after yourself. Console.WriteLine("Removing all files, folders, shares, blobs, and containers created in this demo."); // Delete the file with the ranges in it. cloudFile = fileDirectory.GetFileReference(destFile); await cloudFile.DeleteIfExistsAsync(); Console.WriteLine("Deleting the directory on the file share."); // Delete the directory. bool success = false; success = await fileDirectory.DeleteIfExistsAsync(); if (success) { Console.WriteLine(" Directory on the file share deleted successfully."); } else { Console.WriteLine(" Directory on the file share NOT deleted successfully; may not exist."); } Console.WriteLine("Deleting the file share."); // Delete the share. await cloudFileShare.DeleteAsync(); Console.WriteLine(" Deleted the file share successfully."); Console.WriteLine("Deleting the temporary download directory and the file in it."); // Delete the download folder and its contents. Directory.Delete(downloadFolder, true); Console.WriteLine(" Successfully deleted the temporary download directory."); Console.WriteLine("Deleting the container and blob used in the Copy/Abort test."); await targetBlob.DeleteIfExistsAsync(); await cloudBlobContainer.DeleteIfExistsAsync(); Console.WriteLine(" Successfully deleted the blob and its container."); } catch (Exception ex) { Console.WriteLine(" Exception thrown. Message = {0}{1} Strack Trace = {2}", ex.Message, Environment.NewLine, ex.StackTrace); } }
public AzureStorageHelper(string connectionString) { _account = CloudStorageAccount.Parse(connectionString); _client = _account.CreateCloudFileClient(); }
public AzureFileService(CloudStorageAccount cloudStorageAccount, ILogger <AzureFileService> logger) { this.cloudStorageAccount = cloudStorageAccount; this.cloudFileClient = cloudStorageAccount.CreateCloudFileClient(); this.logger = logger; }
public static void addRecordToAzureFileShare(string myQueueItem, string fileName, ILogger log) { try { string valueLine = string.Empty; string clientHeader = string.Empty; var searcResult = new string[10]; StringBuilder builder = new StringBuilder(); StringBuilder headerBuilder = new StringBuilder(); StringBuilder valueBuilder = new StringBuilder(); string[] arrQueueItem = myQueueItem.Split("||", StringSplitOptions.RemoveEmptyEntries); for (int i = 0; i < arrQueueItem.Length; i++) { var arrHeader = arrQueueItem[i]?.Split(":", StringSplitOptions.RemoveEmptyEntries); if (!string.IsNullOrEmpty(arrHeader[0]) && !arrHeader[0].Trim().Equals("CampaignName")) { string header = arrHeader[0].Replace(",", "-"); string value = arrHeader.Length > 1 ? arrHeader[1].Replace(",", "-") : ""; //Replace comma with hyphen in values headerBuilder.Append(header + ","); valueBuilder.Append(value + ","); } } var headerLastIndex = headerBuilder.ToString().LastIndexOf(","); if (headerLastIndex >= 0) { clientHeader = headerBuilder.ToString().Remove(headerLastIndex, 1) + Environment.NewLine; } builder.Append(clientHeader); var valueLastIndex = valueBuilder.ToString().LastIndexOf(","); if (valueLastIndex >= 0) { valueLine = valueBuilder.ToString().Remove(valueLastIndex, 1); } builder.Append(valueLine); CloudStorageAccount storageAccount = CloudStorageAccount.Parse(GetEnvironmentVariable("AzureWebJobsStorage")); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare fileShare = fileClient.GetShareReference(GetEnvironmentVariable("AzureStorageFileShare")); string existingFiletxt = ""; if (fileShare.Exists()) { CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference(); if (rootDirectory.Exists()) { CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(GetEnvironmentVariable("AzureStorageFileDirectory")); if (customDirectory.Exists()) { CloudFile file = customDirectory.GetFileReference(fileName + ".csv"); if (file.Exists()) { existingFiletxt = file.DownloadTextAsync().Result; string existingHeader = existingFiletxt.Split(new[] { Environment.NewLine }, StringSplitOptions.None)?[0]; string existingValue = existingFiletxt.Split(new[] { Environment.NewLine }, StringSplitOptions.None)[1]; if (existingHeader + Environment.NewLine == clientHeader) { customDirectory.GetFileReference(fileName + ".csv").UploadText(existingFiletxt + Environment.NewLine + valueLine); } else { existingFiletxt = clientHeader + existingValue; customDirectory.GetFileReference(fileName + ".csv").UploadText(existingFiletxt + Environment.NewLine + valueLine.ToString()); } } else { customDirectory.GetFileReference(fileName + ".csv").UploadText(builder.ToString()); } } } } } catch (Exception ex) { log.LogInformation("Some error occured " + ex); } }
public async Task <FileStreamResult> Get(int id) { var claimsIdentity = User.Identity as ClaimsIdentity; var userId = Convert.ToInt64(claimsIdentity.Claims.FirstOrDefault(claim => claim.Type == "Id").Value); var user = _multiSourcePlaylistRepository.GetUser(userId); var track = _multiSourcePlaylistRepository.GetTrack(id); byte[] audioArray = new byte[0]; CloudStorageAccount storageAccount = CloudStorageAccount.Parse( _configuration["Production:StorageConnectionString"]); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); // Get a reference to the file share we created previously. CloudFileShare share = fileClient.GetShareReference(user.FileFolder); CloudFileDirectory userDir = null; // Ensure that the share exists. if (await share.ExistsAsync()) { // Get a reference to the root directory for the share. CloudFileDirectory rootDir = share.GetRootDirectoryReference(); // Get a reference to the directory we created previously. userDir = rootDir.GetDirectoryReference("audio"); // Ensure that the directory exists. if (await userDir.ExistsAsync()) { var audiofile = userDir.GetFileReference(track.Address); if (await audiofile.ExistsAsync()) { await audiofile.FetchAttributesAsync(); audioArray = new byte[audiofile.Properties.Length]; await audiofile.DownloadToByteArrayAsync(audioArray, 0); } } else { return(null); } } else { return(null); } long fSize = audioArray.Length; long startbyte = 0; long endbyte = fSize - 1; int statusCode = 200; var rangeRequest = Request.Headers["Range"].ToString(); if (rangeRequest != "") { string[] range = Request.Headers["Range"].ToString().Split(new char[] { '=', '-' }); startbyte = Convert.ToInt64(range[1]); if (range.Length > 2 && range[2] != "") { endbyte = Convert.ToInt64(range[2]); } if (startbyte != 0 || endbyte != fSize - 1 || range.Length > 2 && range[2] == "") { statusCode = 206; } } long desSize = endbyte - startbyte + 1; Response.StatusCode = statusCode; Response.ContentType = "audio/mp3"; Response.Headers.Add("Content-Accept", Response.ContentType); Response.Headers.Add("Content-Length", desSize.ToString()); Response.Headers.Add("Content-Range", string.Format("bytes {0}-{1}/{2}", startbyte, endbyte, fSize)); Response.Headers.Add("Accept-Ranges", "bytes"); Response.Headers.Remove("Cache-Control"); var stream = new MemoryStream(audioArray, (int)startbyte, (int)desSize); return(new FileStreamResult(stream, "audio/mp3") { FileDownloadName = track.Name }); }
public async static Task Run([TimerTrigger("0 0 0 * * Sun" #if DEBUG , RunOnStartup = true #endif )] TimerInfo myTimer, ILogger log, ExecutionContext context) { log.LogInformation($"SimpleGhostBackup function started execution at: {DateTime.Now}"); var config = new ConfigurationBuilder() .SetBasePath(context.FunctionAppDirectory) .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true) .AddEnvironmentVariables() .Build(); var clientId = config["ClientId"]; if (String.IsNullOrEmpty(clientId)) { throw new ArgumentNullException("ClientId is Required!"); } var clientSecret = config["ClientSecret"]; if (String.IsNullOrEmpty(clientSecret)) { throw new ArgumentNullException("ClientSecret is Required!"); } var blogUrl = config["BlogUrl"]; if (String.IsNullOrEmpty(blogUrl)) { throw new ArgumentNullException("BlogUrl is Required!"); } var storageShareName = config["StorageShareName"]; if (String.IsNullOrEmpty(storageShareName)) { throw new ArgumentNullException("StorageShareName is Required!"); } var storageConnection = config["StorageConnectionString"]; if (String.IsNullOrEmpty(storageConnection)) { throw new ArgumentNullException("storageConnection is Required!"); } // Let get the number of snapshots that we should keep, default to last 4 int maxSnapshots = 4; Int32.TryParse(config["MaxSnapshots"], out maxSnapshots); var client = new HttpClient(new HttpRetryMessageHandler(new HttpClientHandler())) { BaseAddress = new Uri(String.Format("https://{0}", blogUrl)) }; log.LogInformation($"Requesting Ghost Backup"); var response = await client.PostAsync(String.Format("/ghost/api/v0.1/db/backup?client_id={0}&client_secret={1}", clientId, clientSecret), null); if (response.StatusCode == System.Net.HttpStatusCode.OK) { // Get our response content which contains the created backup file name var content = await response.Content.ReadAsStringAsync(); var json = JObject.Parse(content); // Connect to our Azure Storage Account CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnection); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference(storageShareName); CloudFileDirectory root = share.GetRootDirectoryReference(); CloudFileDirectory data = root.GetDirectoryReference("data"); //Does the data folder exist if (await data.ExistsAsync()) { log.LogInformation($"Data folder exists."); // get the backup file name var filename = System.IO.Path.GetFileName((string)json["db"][0]["filename"]); CloudFile file = data.GetFileReference(filename); //Confirm that the backup file exists if (await file.ExistsAsync()) { // Create the snapshotg of the file share log.LogInformation($"Backup file created - {filename}"); log.LogInformation($"Creating Azure Fileshare Snapshot"); var s = await share.SnapshotAsync(); if (s != null) { //Lets get all the current shares/snapshots FileContinuationToken token = null; var snapshots = new List <CloudFileShare>(); do { ShareResultSegment resultSegment = await fileClient.ListSharesSegmentedAsync(storageShareName, ShareListingDetails.Snapshots, 5, token, null, null); snapshots.AddRange(resultSegment.Results); token = resultSegment.ContinuationToken; }while (token != null); //lets delete the old ones var toDelete = snapshots.Where(os => os.IsSnapshot).OrderByDescending(oos => oos.SnapshotTime).Skip(maxSnapshots).ToList(); foreach (var snapshot in toDelete) { try { log.LogInformation($"Deleting snapshot - {snapshot.Name}, Created at {snapshot.SnapshotTime}"); await snapshot.DeleteAsync(); } catch (Exception ex) { log.LogError($"Failed to delete snapshot - '{ex}'"); } } } } } } log.LogInformation($"SimpleGhostBackup function ended execution at: {DateTime.Now}"); }
public JsonResult GetSingleUserTaxReturn(string userId, int taxYear) { bool downloadSuccessful = true; var results = new List <Tuple <string, string, byte[]> >(); using (var db = new WorktripEntities()) { try { Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)"); var user = db.Users.FirstOrDefault(u => u.Id == userId); var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear); taxReturn = taxReturn.OrderBy(d => d.Id); var fileUrls = new List <UserTaxReturn>(); if (taxReturn.Count() != 0) { fileUrls.Add(taxReturn.AsEnumerable().Last()); byte[] bytes = new byte[64000]; var parsedFilePaths = new List <Tuple <string, string> >(); foreach (var url in fileUrls) { Match match = filePattern.Match(url.Path); if (match.Success) { var newTuple = new Tuple <string, string>( match.Groups["directory"].Value, match.Groups["filename"].Value ); parsedFilePaths.Add(newTuple); } } CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference("worktripdocs"); CloudFileDirectory rootDir = share.GetRootDirectoryReference(); CloudFileDirectory userDir = null; var userDirName = ""; foreach (var parsedPath in parsedFilePaths) { if (userDirName != parsedPath.Item1) { userDir = rootDir.GetDirectoryReference(parsedPath.Item1); if (!userDir.Exists()) { continue; } userDirName = parsedPath.Item1; } var filename = parsedPath.Item2; CloudFile file = userDir.GetFileReference(filename); if (!file.Exists()) { continue; } file.FetchAttributes(); string fileContents = ""; if (file.Properties.ContentType.ToLower() == "application/pdf") { MemoryStream fileStream = new MemoryStream(); file.DownloadToStream(fileStream); bytes = fileStream.ToArray(); fileContents = ConvertStreamToBase64String(fileStream); } else { fileContents = file.DownloadText(); } results.Add( new Tuple <string, string, byte[]>(filename, file.Properties.ContentType, bytes) ); } } } catch (Exception e) { //Do some error logging here.. downloadSuccessful = false; } } if (downloadSuccessful && results.Count > 0) { return(Json(new MyJsonResult { status = 0, fileName = results.ElementAtOrDefault(0).Item1, fileContentType = results.ElementAtOrDefault(0).Item2, fileContents = results.ElementAtOrDefault(0).Item3 })); } else { return(Json(new { status = -1, message = "Error in downloading files" })); } }
/// <summary> /// Test some of the file storage operations. /// </summary> public async Task RunFileStorageAdvancedOpsAsync() { // Keep a list of the file shares so you can compare this list // against the list of shares that we retrieve . List <string> fileShareNames = new List <string>(); // Create a file client for interacting with the file service. CloudFileClient cloudFileClient = null; try { //***** Setup *****// Console.WriteLine("Getting reference to the storage account."); // Retrieve storage account information from connection string // How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx CloudStorageAccount storageAccount = Common.CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString")); Console.WriteLine("Instantiating file client."); Console.WriteLine(string.Empty); // Create a file client for interacting with the file service. cloudFileClient = storageAccount.CreateCloudFileClient(); // Create 3 file shares. // Create the share name -- use a guid in the name so it's unique. // This will also be used as the container name for blob storage when copying the file to blob storage. string baseShareName = "demotest-" + System.Guid.NewGuid().ToString(); for (int i = 0; i < 3; i++) { // Set the name of the share, then add it to the generic list. string shareName = baseShareName + "-0" + i; fileShareNames.Add(shareName); // Create the share with this name. Console.WriteLine("Creating share with name {0}", shareName); CloudFileShare cloudFileShare = cloudFileClient.GetShareReference(shareName); try { await cloudFileShare.CreateIfNotExistsAsync(); Console.WriteLine(" Share created successfully."); } catch (StorageException exStorage) { Common.WriteException(exStorage); Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample."); Console.WriteLine("Press any key to exit"); Console.ReadLine(); throw; } catch (Exception ex) { Console.WriteLine(" Exception thrown creating share."); Common.WriteException(ex); throw; } } Console.WriteLine(string.Empty); Console.WriteLine("List of shares in the storage account:"); // List the file shares for this storage account IEnumerable <CloudFileShare> cloudShareList = cloudFileClient.ListShares(); try { foreach (CloudFileShare cloudShare in cloudShareList) { Console.WriteLine("Cloud Share name = {0}", cloudShare.Name); } } catch (Exception ex) { Console.WriteLine(" Exception thrown listing shares."); Common.WriteException(ex); throw; } } catch (Exception ex) { Console.WriteLine(" Exception thrown. Message = {0}{1} Strack Trace = {2}", ex.Message, Environment.NewLine, ex.StackTrace); } finally { // If it created the file shares, remove them (cleanup). if (fileShareNames != null && cloudFileClient != null) { // Now clean up after yourself, using the list of shares that you created in case there were other shares in the account. foreach (string fileShareName in fileShareNames) { CloudFileShare cloudFileShare = cloudFileClient.GetShareReference(fileShareName); cloudFileShare.DeleteIfExists(); } } } }
public JsonResult UploadUserTaxReturn(int taxYear, string subFolder, string userId) { var curUserId = userId == null?User.Identity.GetUserId() : userId; bool uploadedSuccessfully = false; string uploadedURI = ""; using (var db = new WorktripEntities()) { try { var user = db.Users.FirstOrDefault(u => u.Id == curUserId); HttpPostedFileBase uploadedFile = Request.Files.Get(0); var compressedStream = uploadedFile.InputStream; if (uploadedFile.ContentType.StartsWith("image/")) { compressedStream = ResizePictureForBandwidth(uploadedFile); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference("worktripdocs"); CloudFileDirectory rootDir = share.GetRootDirectoryReference(); CloudFileDirectory userDir = rootDir.GetDirectoryReference(user.FirstName + " " + user.LastName + " " + user.PhoneNumber); userDir.CreateIfNotExists(); if (!string.IsNullOrEmpty(subFolder)) { userDir = userDir.GetDirectoryReference(subFolder); userDir.CreateIfNotExists(); } var newFileName = uploadedFile.FileName; var fileExtension = Path.GetExtension(newFileName); CloudFile file = userDir.GetFileReference(newFileName); int fileDuplicateCount = 1; while (file.Exists()) { //generate a file name that doesn't exist yet newFileName = Path.GetFileNameWithoutExtension(newFileName) + "(" + (fileDuplicateCount++) + ")" + fileExtension; file = userDir.GetFileReference(newFileName);; } file.Properties.ContentType = uploadedFile.ContentType; file.UploadFromStream(compressedStream); uploadedURI = file.Uri.ToString(); UserTaxReturn newReturn = new UserTaxReturn() { UserId = curUserId, Date = DateTime.UtcNow, Path = uploadedURI, Year = taxYear }; db.UserTaxReturns.Add(newReturn); db.SaveChanges(); uploadedSuccessfully = true; UserInfoViewModel.UpdateUserActionsLog(curUserId, "uploaded " + taxYear + " " + (fileExtension == ".pdf" ? "tax return" : "tax form(s)")); } catch (Exception e) { //Do some error logging here.. uploadedSuccessfully = false; } } if (uploadedSuccessfully) { return(Json(new { status = 0 })); } else { return(Json(new { status = -1, message = "Error in saving file" })); } }
public static void GetAthleteUpdates(TextWriter log) { Console.WriteLine("Checking the Ironman Athlete Tracker!"); string urlpattern = ConfigurationManager.AppSettings["trackURL"]; string allReferences = ConfigurationManager.AppSettings["athleteRefs"]; string connectionString = ConfigurationManager.ConnectionStrings["AzureCloudStorage"].ConnectionString; // In format raceid-name:bib-name;bib-name/ string pushoverAPIkey = ConfigurationManager.AppSettings["pushoverAPIkey"]; string pushoverGroupKey = ConfigurationManager.AppSettings["pushoverGroupKey"]; CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString); // Create a reference to the file client. CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference("tracking"); // Create a reference to the Azure path CloudFileDirectory cloudFileDirectory = share.GetRootDirectoryReference(); foreach (string trackinfo in allReferences.Split('/')) { string raceinfo = trackinfo.Split(':')[0]; string raceid = raceinfo.Split('-')[0]; string racename = raceinfo.Split('-')[1]; string[] athletes = trackinfo.Split(':')[1].Split(';'); foreach (string athlete in athletes) { string bib = athlete.Split('-')[0]; string athleteName = athlete.Split('-')[1]; string url = urlpattern.Replace("$rid$", raceid).Replace("$bib$", bib); try { Console.WriteLine("About to download athelete url: " + url); WebClient client = new WebClient(); string newFile = client.DownloadString(url); Console.WriteLine(newFile); int pFrom = newFile.IndexOf(@"<!-- Begin: main content area -->"); int pTo = newFile.IndexOf(@"<!-- End: main content area -->"); string latestTrackingData = newFile.Substring(pFrom, pTo - pFrom); string filename = raceid + "-" + bib + ".html"; try { CloudFile cloudFile = cloudFileDirectory.GetFileReference(filename); bool isUpdated = false; if (cloudFile.Exists()) { StreamReader sr = new StreamReader(cloudFile.OpenRead()); string previousTrackingData = sr.ReadToEnd(); if (!latestTrackingData.Equals(previousTrackingData, StringComparison.InvariantCultureIgnoreCase)) { isUpdated = true; } sr.Close(); } cloudFile.UploadTextAsync(latestTrackingData); if (isUpdated) { Pushover pclient = new Pushover(pushoverAPIkey); PushResponse response = pclient.Push( racename + " athlete alert", "Looks like " + athleteName + " has a new split update!", pushoverGroupKey ); } } catch (Exception ex1) { Console.WriteLine("Error!"); Console.WriteLine(ex1.Message); Console.WriteLine(ex1.StackTrace); } } catch (Exception ex2) { Console.WriteLine("Error!"); Console.WriteLine(ex2.Message); Console.WriteLine(ex2.StackTrace); } } } }
static void Main(string[] args) { string shareName = "\\\\unipertest.file.core.windows.net\\SecretDocuments"; string driveLetterAndColon = "z:"; string username = "******"; string password = "******"; CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create a CloudFileClient object for credentialed access to File storage. CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); // Get a reference to the file share we created previously. CloudFileShare share = fileClient.GetShareReference("secretdocuments"); // Ensure that the share exists. if (share.Exists()) { // Get a reference to the root directory for the share. CloudFileDirectory rootDir = share.GetRootDirectoryReference(); // Get a reference to the directory we created previously. CloudFileDirectory sampleDir = rootDir.GetDirectoryReference("CustomLogs"); // Ensure that the directory exists. if (sampleDir.Exists()) { // Get a reference to the file we created previously. CloudFile file = sampleDir.GetFileReference("Log1.txt"); // Ensure that the file exists. if (file.Exists()) { // Write the contents of the file to the console window. Console.WriteLine(file.DownloadTextAsync().Result); Console.WriteLine("Successfully connected to file share"); System.Threading.Thread.Sleep(300); } } if (!String.IsNullOrEmpty(driveLetterAndColon)) { // Make sure we aren't using this driveLetter for another mapping WNetCancelConnection2(driveLetterAndColon, 0, true); } NETRESOURCE nr = new NETRESOURCE(); nr.dwType = ResourceType.RESOURCETYPE_DISK; nr.lpRemoteName = shareName; nr.lpLocalName = driveLetterAndColon; int result = WNetAddConnection2(nr, password, username, 0); if (result != 0) { throw new Exception("WNetAddConnection2 failed with error " + result); } } }
public void CloudStorageAccountWithStorageUri() { StorageUri blobEndpoint = new StorageUri( new Uri("http://" + AccountName + BlobService + EndpointSuffix), new Uri("http://" + AccountName + SecondarySuffix + BlobService + EndpointSuffix)); StorageUri queueEndpoint = new StorageUri( new Uri("http://" + AccountName + QueueService + EndpointSuffix), new Uri("http://" + AccountName + SecondarySuffix + QueueService + EndpointSuffix)); StorageUri tableEndpoint = new StorageUri( new Uri("http://" + AccountName + TableService + EndpointSuffix), new Uri("http://" + AccountName + SecondarySuffix + TableService + EndpointSuffix)); StorageUri fileEndpoint = new StorageUri( new Uri("http://" + AccountName + FileService + EndpointSuffix), new Uri("http://" + AccountName + SecondarySuffix + FileService + EndpointSuffix)); #if WINDOWS_RT || ASPNET_K CloudStorageAccount account = CloudStorageAccount.Create(new StorageCredentials(), blobEndpoint, queueEndpoint, tableEndpoint, fileEndpoint); #else CloudStorageAccount account = new CloudStorageAccount(new StorageCredentials(), blobEndpoint, queueEndpoint, tableEndpoint, fileEndpoint); #endif Assert.IsTrue(blobEndpoint.Equals(account.BlobStorageUri)); Assert.IsTrue(queueEndpoint.Equals(account.QueueStorageUri)); Assert.IsTrue(tableEndpoint.Equals(account.TableStorageUri)); Assert.IsTrue(fileEndpoint.Equals(account.FileStorageUri)); account = new CloudStorageAccount(new StorageCredentials(AccountName, TestBase.StorageCredentials.ExportBase64EncodedKey()), false); Assert.IsTrue(blobEndpoint.Equals(account.BlobStorageUri)); Assert.IsTrue(queueEndpoint.Equals(account.QueueStorageUri)); Assert.IsTrue(tableEndpoint.Equals(account.TableStorageUri)); Assert.IsTrue(fileEndpoint.Equals(account.FileStorageUri)); account = CloudStorageAccount.Parse(string.Format("DefaultEndpointsProtocol=http;AccountName={0};AccountKey=", AccountName)); Assert.IsTrue(blobEndpoint.Equals(account.BlobStorageUri)); Assert.IsTrue(queueEndpoint.Equals(account.QueueStorageUri)); Assert.IsTrue(tableEndpoint.Equals(account.TableStorageUri)); Assert.IsTrue(fileEndpoint.Equals(account.FileStorageUri)); Assert.IsTrue(blobEndpoint.Equals(account.CreateCloudBlobClient().StorageUri)); Assert.IsTrue(queueEndpoint.Equals(account.CreateCloudQueueClient().StorageUri)); Assert.IsTrue(tableEndpoint.Equals(account.CreateCloudTableClient().StorageUri)); Assert.IsTrue(fileEndpoint.Equals(account.CreateCloudFileClient().StorageUri)); Assert.IsTrue(blobEndpoint.PrimaryUri.Equals(account.BlobEndpoint)); Assert.IsTrue(queueEndpoint.PrimaryUri.Equals(account.QueueEndpoint)); Assert.IsTrue(tableEndpoint.PrimaryUri.Equals(account.TableEndpoint)); Assert.IsTrue(fileEndpoint.PrimaryUri.Equals(account.FileEndpoint)); }
private static async Task mainAsync(string[] args) { try { var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appsettings.json", optional: true, reloadOnChange: true); IConfigurationRoot configuration = builder.Build(); var allCitDebtsLines = new string[10000000]; var allSettlementsLines = new string[10000000]; var allPaymentLines = new string[10000000]; while (true) { if (DateTime.Now.AddHours(3).ToShortTimeString() == "12:00 PM") { List <string> fileNames = new List <string>(); List <string[]> lstCloudFilesdata = new List <string[]>(); CloudStorageAccount storageAccount = CloudStorageAccount.Parse($"{configuration["ConnectionString1"]}"); CloudFileClient fileClient = storageAccount.CreateCloudFileClient(); CloudFileShare fileShare = fileClient.GetShareReference("import"); //looks for a file share in the cloud bool fileShareExists = await fileShare.ExistsAsync(); if (fileShareExists) { List <CloudFile> lstCloudFiles = new List <CloudFile>(); CloudFileDirectory rootDir = fileShare.GetRootDirectoryReference(); //for each file in my fileshare FileContinuationToken token = null; FileResultSegment k = await rootDir.ListFilesAndDirectoriesSegmentedAsync(token); token = k.ContinuationToken; //var context_ = new Db.Data.(); List <string> sl = new List <string>(); foreach (IListFileItem fiile in k.Results) { //if the file exists CloudFile file = (CloudFile)fiile; bool asd = await file.ExistsAsync(); if (asd) { //adds new datasting array sl = await ReadDataAsync(lstCloudFilesdata, file, fileNames); } foreach (string y in sl) { Console.WriteLine("From list new : " + y); } ; IEnumerable <CitizenDepts> o = from eachLine in ( from inner in sl select inner.Split(';') ) select new CitizenDepts { VAT = eachLine[0], FirstName = eachLine[1], LastName = eachLine[2], Email = eachLine[3], Phone = eachLine[4], Address = eachLine[5], County = eachLine[6], BillId = eachLine[7], Bill_description = eachLine[8], Amount = Decimal.Parse(eachLine[9]), DueDate = DateTime.ParseExact(eachLine[10], "yyyyMMdd", CultureInfo.InvariantCulture) }; foreach (var p in o) { Console.WriteLine(p.FirstName + " - " + p.BillId + ", - " + p.DueDate); } ; //string s = context_.Database.ProviderName; // Console.WriteLine(s); /// var all = from c in context_.CitizenDepts select c; //context_.CitizenDepts.RemoveRange(all); /// context_.SaveChanges(); foreach (var p in o) { //Add Student object into Students DBset //if (p.VAT!=null) //context_.Add(p); } //// call SaveChanges method to save student into database //context_.SaveChanges(); } } if (lstCloudFilesdata != null && fileNames != null) { ProccessData(lstCloudFilesdata, fileNames); } } } } catch (Exception ex) { Console.WriteLine(ex.Message); } finally { } }
public void CloudStorageAccountClientUriVerify() { StorageCredentials cred = new StorageCredentials(TestBase.TargetTenantConfig.AccountName, TestBase.TargetTenantConfig.AccountKey); CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(cred, true); CloudBlobClient blobClient = cloudStorageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference("container1"); Assert.AreEqual(cloudStorageAccount.BlobEndpoint.ToString() + "container1", container.Uri.ToString()); CloudQueueClient queueClient = cloudStorageAccount.CreateCloudQueueClient(); CloudQueue queue = queueClient.GetQueueReference("queue1"); Assert.AreEqual(cloudStorageAccount.QueueEndpoint.ToString() + "queue1", queue.Uri.ToString()); CloudTableClient tableClient = cloudStorageAccount.CreateCloudTableClient(); CloudTable table = tableClient.GetTableReference("table1"); Assert.AreEqual(cloudStorageAccount.TableEndpoint.ToString() + "table1", table.Uri.ToString()); CloudFileClient fileClient = cloudStorageAccount.CreateCloudFileClient(); CloudFileShare share = fileClient.GetShareReference("share1"); Assert.AreEqual(cloudStorageAccount.FileEndpoint.ToString() + "share1", share.Uri.ToString()); }
async Task <bool> ExecuteLoadFileSharesAsync(bool force = false) { if (IsBusy) { return(false); } var realm = App.GetRealm(); try { IsBusy = true; NoFileSharesFound = false; var realmFileShares = realm.All <RealmCloudFileShare>(); if (realmFileShares.Count() > 0 && force == false) { var storageAccounts = realm.All <StorageAccountExt>().Where(sa => sa.IsStorageAccountOn); List <ASECloudFileShare> aseFileShares = new List <ASECloudFileShare>(); if (storageAccounts.Count() > 0) { foreach (var fShare in realmFileShares) { StorageAccountsExist = true; var storageAccount = storageAccounts.Where((arg) => arg.Name == fShare.StorageAccountName).FirstOrDefault(); if (storageAccount != null) { var te = new CloudFileShare(new Uri(fShare.FileShareUri), new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storageAccount.Name, storageAccount.PrimaryKey)); aseFileShares.Add(new ASECloudFileShare(te, storageAccount.Name)); } } FileShares.Clear(); FileShares.AddRange(aseFileShares); } } else { var storageAccounts = realm.All <StorageAccountExt>().Where(sa => sa.IsStorageAccountOn); FileShares.Clear(); foreach (var account in storageAccounts) { string connectionString = Constants.StorageConnectionString; connectionString = connectionString.Replace("<ACCOUNTNAME>", account.Name); connectionString = connectionString.Replace("<ACCOUNTKEY>", account.PrimaryKey); CloudStorageAccount sa = CloudStorageAccount.Parse(connectionString); var fileClient = sa.CreateCloudFileClient(); var fileShares = await fileClient.ListFileSharesAsync(); List <ASECloudFileShare> aseFileShares = new List <ASECloudFileShare>(); for (int i = 0; i < fileShares.Count; i++) { aseFileShares.Add(new ASECloudFileShare(fileShares[i])); } aseFileShares.All(c => { c.StorageAccountName = account.Name; return(true); }); FileShares.AddRange(aseFileShares); } if (storageAccounts.Count() > 0) { StorageAccountsExist = true; } else { StorageAccountsExist = false; } await realm.WriteAsync(temprealm => { temprealm.RemoveAll <RealmCloudFileShare>(); foreach (var fShare in FileShares) { temprealm.Add(new RealmCloudFileShare(fShare.FileShareName, fShare.StorageAccountName, fShare.BaseFileShare.Uri.ToString())); } }); realm.All <RealmCloudFileShare>().SubscribeForNotifications((sender, changes, error) => { Console.WriteLine("Change to CloudFileShares"); }); } SortFileShares(); if (FileShares.Count == 0) { NoFileSharesFound = true; } else { NoFileSharesFound = false; } } catch (Exception ex) { Logger.Report(ex, "Method", "ExecuteLoadFileSharesAsync"); MessagingService.Current.SendMessage(MessageKeys.Error, ex); } finally { IsBusy = false; } return(true); }