public JsonResult GetUserDocuments(string userId, int taxYear, int?skip, int?amount)
{
bool downloadSuccessful = true;
var results = new List <Tuple <string, string, string> >();
using (var db = new WorktripEntities())
{
try
{
Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)");
var user = db.Users.FirstOrDefault(u => u.Id == userId);
var miscDocs = db.UserMiscDocs.Where(d => d.UserId == userId && d.Year == taxYear);
var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear);
miscDocs = miscDocs.OrderBy(d => d.Id);
taxReturn = taxReturn.OrderBy(d => d.Id);
if (skip.HasValue)
{
miscDocs = miscDocs.Skip(skip.Value);
taxReturn = taxReturn.Skip(skip.Value);
}
if (amount.HasValue)
{
miscDocs = miscDocs.Take(amount.Value);
taxReturn = taxReturn.Take(amount.Value);
}
var fileUrls = miscDocs.Select(d => d.Path).ToList();
fileUrls.AddRange(taxReturn.Select(d => d.Path).ToList());
var parsedFilePaths = new List <Tuple <string, string> >();
foreach (var url in fileUrls)
{
Match match = filePattern.Match(url);
if (match.Success)
{
var newTuple = new Tuple <string, string>(
match.Groups["directory"].Value,
match.Groups["filename"].Value
);
parsedFilePaths.Add(newTuple);
}
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("worktripdocs");
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
CloudFileDirectory userDir = null;
var userDirName = "";
foreach (var parsedPath in parsedFilePaths)
{
if (userDirName != parsedPath.Item1)
{
userDir = rootDir.GetDirectoryReference(parsedPath.Item1);
if (!userDir.Exists())
{
continue;
}
userDirName = parsedPath.Item1;
}
var filename = parsedPath.Item2;
CloudFile file = userDir.GetFileReference(filename);
if (!file.Exists())
{
continue;
}
file.FetchAttributes();
string fileContents = "";
if (file.Properties.ContentType != null &&
file.Properties.ContentType.StartsWith("image/"))
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
fileContents = ConvertImageStreamToBase64String(fileStream);
}
else if (file.Properties.ContentType.ToLower() == "application/pdf")
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
fileContents = ConvertStreamToBase64String(fileStream);
}
else
{
fileContents = file.DownloadText();
}
results.Add(
new Tuple <string, string, string>(filename, file.Properties.ContentType, fileContents)
);
}
}
catch (Exception e)
{
//Do some error logging here..
downloadSuccessful = false;
}
}
if (downloadSuccessful)
{
return(Json(new {
status = 0,
files = results
}));
}
else
{
return(Json(new { status = -1, message = "Error in downloading files" }));
}
}
public void CloudStorageAccountClientMethods()
{
CloudStorageAccount account = new CloudStorageAccount(TestBase.StorageCredentials, false);
CloudBlobClient blob = account.CreateCloudBlobClient();
CloudQueueClient queue = account.CreateCloudQueueClient();
CloudTableClient table = account.CreateCloudTableClient();
CloudFileClient file = account.CreateCloudFileClient();
// check endpoints
Assert.AreEqual(account.BlobEndpoint, blob.BaseUri, "Blob endpoint doesn't match account");
Assert.AreEqual(account.QueueEndpoint, queue.BaseUri, "Queue endpoint doesn't match account");
Assert.AreEqual(account.TableEndpoint, table.BaseUri, "Table endpoint doesn't match account");
Assert.AreEqual(account.FileEndpoint, file.BaseUri, "File endpoint doesn't match account");
// check storage uris
Assert.AreEqual(account.BlobStorageUri, blob.StorageUri, "Blob endpoint doesn't match account");
Assert.AreEqual(account.QueueStorageUri, queue.StorageUri, "Queue endpoint doesn't match account");
Assert.AreEqual(account.TableStorageUri, table.StorageUri, "Table endpoint doesn't match account");
Assert.AreEqual(account.FileStorageUri, file.StorageUri, "File endpoint doesn't match account");
// check creds
Assert.AreEqual(account.Credentials, blob.Credentials, "Blob creds don't match account");
Assert.AreEqual(account.Credentials, queue.Credentials, "Queue creds don't match account");
Assert.AreEqual(account.Credentials, table.Credentials, "Table creds don't match account");
Assert.AreEqual(account.Credentials, file.Credentials, "File creds don't match account");
}
public async Task RunPipelineAsync(CancellationToken cancellationToken)
{
if (parser.Document.SelectToken("$.imageRegistryCredentials") is JArray imageRegistryCredentials)
{
foreach (JObject imageRegistryCredential in imageRegistryCredentials)
{
HandleObjReplacement(imageRegistryCredential);
logger.LogInformation("using {@imageRegistryCredential}", imageRegistryCredential);
}
}
foreach (var part in parser.Document.SelectToken("$.pipe"))
{
var command = part.SelectToken("$.command");
var dependsOn = part.SelectToken("$.dependsOn")?.Select(c => c.ToString()).ToArray();
var parallelBlocker = int.MaxValue;
var parallelKey = part.SelectToken("$.name").ToString();
if (part.SelectToken("$.parallelContext") is JObject parallelContext)
{
HandleObjReplacement(parallelContext);
parallelBlocker = parallelContext.SelectToken("$.max")?.ToObject <int>() ?? parallelBlocker;
parallelKey = parallelContext.SelectToken("$.contextKey")?.ToString() ?? parallelKey;
logger.LogInformation("using {@parallelContext}: {parallelKey} {parallelBlocker}", parallelContext, parallelKey, parallelBlocker);
}
var semaphoreObject = options.GetSemaphore(parallelKey, parallelBlocker); //
parser.Functions["parallelContext"] = (document, arguments) => JToken.FromObject(new { counter = semaphoreObject.RunningCounter });
var volumnes = new Dictionary <string, JObject>();
var dependenciesResolved = true;
if (dependsOn != null)
{
foreach (var dependency in dependsOn)
{
if (dependency.StartsWith("/volumes/"))
{
var volumeName = dependency.Substring("/volumes/".Length);
var volume = parser.Document.SelectToken($"$.volumes.{volumeName}")?.DeepClone();
if (volume is JObject volumeObj)
{
HandleObjReplacement(volumeObj);
if (volume.SelectToken("$.azureFileShare") is JObject azureFileShare)
{
var storageAccount = new CloudStorageAccount(new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(azureFileShare.SelectToken("$.storageAccountName").ToString(), azureFileShare.SelectToken("$.storageAccountKey").ToString()), true);
await storageAccount.CreateCloudFileClient().GetShareReference(azureFileShare.SelectToken("$.shareName").ToString()).CreateIfNotExistsAsync();
}
if (volume.SelectToken("$.local") is JObject local)
{
var uniqueId = local.SelectToken("$.uniqueId")?.ToString();
if (!string.IsNullOrEmpty(uniqueId))
{
local.Replace(CreateMD5(uniqueId));
}
}
//if (volume.SelectToken("$.mountedFrom").ToString().StartsWith("["))
//{
// volume.SelectToken("$.mountedFrom").Replace(parser.Evaluate(volume.SelectToken("$.mountedFrom").ToString()));
//}
// volumnes.Add(volume.SelectToken("$.mountedAt").ToString(),new EmptyStruct { });
volumnes.Add(volumeName, volumeObj);
}
else
{
dependenciesResolved = false;
break;
}
}
else
{
var completed = parser.Document.SelectToken($"$.pipe[?(@.name == '{dependency}')].outputs.completed")?.ToObject <bool>() ?? false;
if (!completed)
{
dependenciesResolved = false;
break;
}
}
}
if (!dependenciesResolved)
{
continue;
}
}
parser.Functions["mountedAt"] = (d, a) => volumnes[a.First().ToString()]["mountedAt"].ToString();
parser.Functions["mountedFrom"] = (d, a) => volumnes[a.First().ToString()]["mountedFrom"].ToString();
var commands = new List <string[]>()
{
};
var loop = part.SelectToken("loop");
if (loop != null)
{
if (loop.Type == JTokenType.String && loop.ToString().StartsWith("["))
{
loop = parser.Evaluate(loop.ToString());
}
logger.LogInformation("using {@loop}", loop);
foreach (var loopVar in loop)
{
parser.Functions["loop"] = (document, arguments) => arguments[0].ToString() == "var" ? loopVar: Path.GetFileName(loopVar.ToString());
var skip = part.SelectToken("$.skip");
if (skip == null || !skip.ToString().StartsWith("[") || !parser.Evaluate(skip.ToString()).ToObject <bool>())
{
commands.Add(command.Select(c => c.ToString().StartsWith("[") ? parser.Evaluate(c.ToString())?.ToString() : c.ToString()).ToArray());
}
}
}
else
{
var skip = part.SelectToken("$.skip");
if (skip == null || !skip.ToString().StartsWith("[") || !parser.Evaluate(skip.ToString()).ToObject <bool>())
{
commands.Add(command.Select(c => c.ToString().StartsWith("[") ? parser.Evaluate(c.ToString())?.ToString() : c.ToString()).ToArray());
}
}
if (!command.Any())
{
var outputs = part["outputs"] as JObject ?? new JObject();
outputs["completed"] = true;
part["outputs"] = outputs;
Console.WriteLine(part.ToString());
continue;
}
var image = part.SelectToken("$.image").ToString();
if (image.StartsWith("["))
{
part["image"] = parser.Evaluate(image).ToString();
}
var block = new ActionBlock <string[]>(async arguments =>
{
try
{
semaphoreObject.WaitOne();
var binds = volumnes.Select((v, i) => $"{GetMountedFrom(v)}:{v.Value.SelectToken("$.mountedAt").ToString()}").ToList();
var runCommand = $"docker run {string.Join(" ", binds.Select(b => $"-v {b}"))} {part.SelectToken("$.image").ToString()} {string.Join(" ", arguments)}";
logger.LogInformation(runCommand);
string stdOut = await executor.ExecuteStepAsync(parser, arguments, part, volumnes, cancellationToken);
parser.Functions["stdout"] = (d, o) => stdOut;
var outputs = part["outputs"] as JObject ?? new JObject();
foreach (var outputProperty in outputs.Properties())
{
if (outputProperty.Value.ToString().StartsWith("["))
{
outputProperty.Value.Replace(parser.Evaluate(outputProperty.Value.ToString()));
}
}
//Console.WriteLine("stdout:");
// Console.WriteLine(stdOut.ToString());
// await client.Containers.RemoveContainerAsync(container.ID, new ContainerRemoveParameters { RemoveVolumes = true, RemoveLinks = true, Force = true });
// Console.WriteLine(part.ToString());
}
finally
{
semaphoreObject.Release();
}
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = parser.Evaluate("[parameters('numberOfProcesses')]").ToObject <int>()
});
// var prop = command.Parent;
// Console.WriteLine(command.ToString());
foreach (var arguments in commands)
{
await block.SendAsync(arguments);
}
block.Complete();
await block.Completion;
{
var outputs = part["outputs"] as JObject ?? new JObject();
outputs["completed"] = true;
part["outputs"] = outputs;
logger.LogInformation("pipeline completed with {output}", outputs);
}
}
await executor.PipelineFinishedAsync(parser);
}
// *************************************************************************************************************************
// Instructions: This sample can be run against Microsoft Azure Storage Service by updating the App.Config with your AccountName and AccountKey.
//
// To run the sample using the Storage Service
// 1. Create a Storage Account through the Azure Portal and provide your [AccountName] and [AccountKey] in
// the App.Config file. See http://go.microsoft.com/fwlink/?LinkId=325277 for more information
// 2. Set breakpoints and run the project using F10.
//
// *************************************************************************************************************************
/// <summary>
/// Basic operations to work with Azure Files
/// </summary>
/// <returns>Task</returns>
public async Task BasicAzureFileOperationsAsync()
{
const string DemoShare = "demofileshare";
const string DemoDirectory = "demofiledirectory";
const string ImageToUpload = "HelloWorld.png";
// Retrieve storage account information from connection string
// How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create a file client for interacting with the file service.
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
// Create a share for organizing files and directories within the storage account.
Console.WriteLine("1. Creating file share");
CloudFileShare share = fileClient.GetShareReference(DemoShare);
try
{
await share.CreateIfNotExistsAsync();
}
catch (StorageException)
{
Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample.");
Console.WriteLine("Press any key to exit");
Console.ReadLine();
throw;
}
// Get a reference to the root directory of the share.
CloudFileDirectory root = share.GetRootDirectoryReference();
// Create a directory under the root directory
Console.WriteLine("2. Creating a directory under the root directory");
CloudFileDirectory dir = root.GetDirectoryReference(DemoDirectory);
await dir.CreateIfNotExistsAsync();
// Uploading a local file to the directory created above
Console.WriteLine("3. Uploading a file to directory");
CloudFile file = dir.GetFileReference(ImageToUpload);
await file.UploadFromFileAsync(ImageToUpload);
// List all files/directories under the root directory
Console.WriteLine("4. List Files/Directories in root directory");
List <IListFileItem> results = new List <IListFileItem>();
FileContinuationToken token = null;
do
{
FileResultSegment resultSegment = await share.GetRootDirectoryReference().ListFilesAndDirectoriesSegmentedAsync(token);
results.AddRange(resultSegment.Results);
token = resultSegment.ContinuationToken;
}while (token != null);
// Print all files/directories listed above
foreach (IListFileItem listItem in results)
{
// listItem type will be CloudFile or CloudFileDirectory
Console.WriteLine("- {0} (type: {1})", listItem.Uri, listItem.GetType());
}
// Download the uploaded file to your file system
Console.WriteLine("5. Download file from {0}", file.Uri.AbsoluteUri);
await file.DownloadToFileAsync(string.Format("./CopyOf{0}", ImageToUpload), FileMode.Create);
// Clean up after the demo
Console.WriteLine("6. Delete file");
await file.DeleteAsync();
// When you delete a share it could take several seconds before you can recreate a share with the same
// name - hence to enable you to run the demo in quick succession the share is not deleted. If you want
// to delete the share uncomment the line of code below.
// Console.WriteLine("7. Delete Share");
// await share.DeleteAsync();
}
/// <summary>
/// Deploy Batch resourses for cloud computing. Open a batch client.
/// </summary>
public static async Task InitializeAsync()
{
try
{
Console.WriteLine("Optimization / Azure Start: {0}", DateTime.Now);
Console.WriteLine();
_timer = Stopwatch.StartNew();
// This will boost parallel submission speed for REST APIs. If your use requires many simultaneous service calls set this number to something large, such as 100.
// See: https://msdn.microsoft.com/en-us/library/system.net.servicepointmanager.defaultconnectionlimit.aspx for more info.
System.Net.ServicePointManager.DefaultConnectionLimit = 100;
// Create a Batch client and authenticate with shared key credentials.
// The Batch client allows the app to interact with the Batch service.
var batchAccountUrl = Shared.Config.BatchAccountUrl;
var batchAccountName = Shared.Config.BatchAccountName;
var batchAccountKey = Shared.Config.BatchAccountKey;
BatchSharedKeyCredentials sharedKeyCredentials = new BatchSharedKeyCredentials(batchAccountUrl, batchAccountName, batchAccountKey);
BatchClient = BatchClient.Open(sharedKeyCredentials);
// Construct the Storage account connection string
string storageConnectionString =
$"DefaultEndpointsProtocol=https;AccountName={Shared.Config.StorageAccountName};AccountKey={Shared.Config.StorageAccountKey}";
// Retrieve the storage account object
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client, to reference the blob storage containers
BlobClient = storageAccount.CreateCloudBlobClient();
// Create File Client, to access Azure Files
FileClient = storageAccount.CreateCloudFileClient();
// Create share if not exist
DataFileShare = FileClient.GetShareReference(DataFileShareName);
await DataFileShare.CreateIfNotExistsAsync();
// Synchronize data between data folder and Cloud File Share
Console.Write("Synchronize data with file share? [yes] no: ");
string response = Console.ReadLine()?.ToLower();
if (response != "n" && response != "no")
{
await SynchronizeHistoricalDataWithFileShareAsync(FileClient);
}
// Creat Containers: OutPut Container (where tasks will upload results) and Container for Algorithm DLL
// Obtain a shared access signature that provides write access to the output
await CreateContainerIfNotExistAsync(BlobClient, DllContainerName);
await CreateContainerIfNotExistAsync(BlobClient, OutputContainerName);
OutputContainerSasUrl = GetContainerSasUrl(BlobClient, OutputContainerName, SharedAccessBlobPermissions.Write);
// Create the Batch pool, if not exist, which contains the compute nodes that execute the tasks.
await CreatePoolIfNotExistAsync(BatchClient, PoolId);
// Create the job that runs the tasks.
await CreateJobAsync(BatchClient, JobId, PoolId);
}
// Catch for an aggregate exception
catch (AggregateException ae)
{
// Flatten agregates all inner exception in one
foreach (var innerException in ae.Flatten().InnerExceptions)
{
Shared.Logger.Error($"AzureBatchManager.DeployAsync(): {innerException.Message}");
}
throw;
}
}
public FileService(CloudStorageAccount cloudStorageClient)
{
this._cloudFileClient = cloudStorageClient.CreateCloudFileClient();
}
public async Task <IActionResult> GetFile()
{
try
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
IConfigurationRoot configuration = builder.Build();
var allCitDebtsLines = new string[10000000];
// while (true)
//{
// if (DateTime.Now.AddHours(3).ToShortTimeString() == "09:58:00 aM")
// {
//List<string> fileNames = new List<string>();
//List<string[]> lstCloudFilesdata = new List<string[]>();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse($"{configuration["ConnectionString1"]}");
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare fileShare = fileClient.GetShareReference("import");
//looks for a file share in the cloud
bool fileShareExists = await fileShare.ExistsAsync();
if (fileShareExists)
{
List <CloudFile> lstCloudFiles = new List <CloudFile>();
CloudFileDirectory rootDir = fileShare.GetRootDirectoryReference();
List <string> sl = new List <string>();
DeptResults DeptResults = new DeptResults();
List <ErrorLines> ErrorLines = new List <ErrorLines>();
ErrorLines ErrorLine = new ErrorLines();
string bigfilename = "CitizenDebts_1M_3Big.txt";
string fileName = "CitizenDebts_1M_3.txt";
//"DEBTS_" + DateTime.Now.ToString("yyyyMMdd") + ".txt";
CloudFile file = rootDir.GetFileReference(fileName);
string checkfile = bigfilename;
//if the file exists
bool asd = await file.ExistsAsync();
if (asd)
{
//adds new datasting array
sl = await ReadDataAsync(file);
if (sl is null)
{
//return NotFound(HttpStatusCode.NotFound + "\n" + "\nΣφάλμα\nΤο αρχείο δεν περιέχει σωστό αριθμό στηλών</b>");
Redirect(DeptResults.BillsCount = 0, DeptResults.NewUsers = 0, HttpStatusCode.NotFound.ToString(), null, "\n" + "\nΣφάλμα\nΤο αρχείο δεν περιέχει σωστό αριθμό στηλών</b>");
}
}
else
{
Redirect(DeptResults.BillsCount = 0, DeptResults.NewUsers = 0, HttpStatusCode.NotFound.ToString(), null, "\n" + "\nΣφάλμα\nΔεν βρέθηκε το αρχείο</b>");
// return NotFound(HttpStatusCode.NotFound+ "\n" + "\nΣφάλμα\nΔεν βρέθηκε το αρχείο</b>");
}
Console.WriteLine("File into List " + DateTime.Now.ToString());
//foreach (string y in sl)
//{ Console.WriteLine("From list new : " + y); };
string[] cols;
for (int i = sl.Count - 1; i >= 0; i--)
{
cols = sl.ElementAt(i).Split(';');
if (cols[0].Trim().Length != 10 || !cols[0].All(char.IsDigit))
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος ΑΦΜ";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
continue;
//return NotFound(HttpStatusCode.NotFound + "\n" + "\nΣφάλμα Λάθος ΑΦΜ");}
}
if (cols[1].Trim().Length == 0)
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Όνομα ";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i); //return NotFound(HttpStatusCode.NotFound + "\n" + "\nΣφάλμα Λάθος Όνομα ");
}
if (cols[2].Trim().Length == 0)
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Επώνυμο ";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
}
if (cols[3].Trim().Length == 0 || !Regex.IsMatch(cols[3], @"\w+([-+.']\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*"))
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Email ";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
}
if (cols[4].Trim().Length == 0 || !cols[4].All(char.IsDigit))
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Τηλέφωνο ";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
}
if (cols[5].Trim().Length == 0)
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Διεύθυσνη ";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
}
if (cols[6].Trim().Length == 0)
{
sl.RemoveAt(i); return(NotFound(HttpStatusCode.NotFound + "\n" + "\nΣφάλμα Λάθος Περιοχή "));
}
//!Regex.IsMatch(cols[7], @"^(\{{0,1}([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}\}{0,1})$")
if (cols[7].Trim().Length == 0)
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Αρ.Λογαριασμού ";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
}
if (cols[8].Trim().Length == 0)
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Περιγραφή Λογαριασμού";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
}
decimal number;
if (cols[9].Trim().Length == 0 || !Decimal.TryParse(cols[9], out number) || cols[9].Contains('.'))
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Ποσό";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
}
DateTime d;
if (cols[10].Trim().Length == 0 || !DateTime.TryParseExact(cols[10], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out d))
{
ErrorLine.line = i;
ErrorLine.ErrorMessage = "Σφάλμα Λάθος Ημερομηνία";
ErrorLine.LineString = sl[i];
ErrorLines.Add(ErrorLine);
sl.RemoveAt(i);
}
}
IEnumerable <CitizenDepts> o = from eachLine in (
from inner in sl
select inner.Split(';')
)
select new CitizenDepts
{
VAT = eachLine[0],
FirstName = eachLine[1],
LastName = eachLine[2],
Email = eachLine[3],
Phone = eachLine[4],
Address = eachLine[5],
County = eachLine[6],
BillId = eachLine[7],
Bill_description = eachLine[8],
//Amount = Decimal.Parse(eachLine[9]),
Amount = Decimal.Parse(eachLine[9], System.Globalization.CultureInfo.GetCultureInfo("el-GR")),
DueDate = DateTime.ParseExact(eachLine[10],
"yyyyMMdd", CultureInfo.InvariantCulture)
};
Console.WriteLine("File splitted " + DateTime.Now.ToString());
var all = from c in _context.CitizenDepts select c;
_context.CitizenDepts.RemoveRange(all);
await _context.SaveChangesAsync();
Console.WriteLine("CitizenDept table deleted " + DateTime.Now.ToString());
foreach (var p in o)
{
_context.Add(p);
}
Console.WriteLine("Context filledup " + DateTime.Now.ToString());
await _context.SaveChangesAsync();
Console.WriteLine("Context saved to DB " + DateTime.Now.ToString());
//filter citizens
NewCitizens = o.
Where(x => !((_context.ApplicationUser.Any(y => y.VAT == x.VAT)) || (_context.ApplicationUser.Any(y => y.Email == x.Email))))
.Select(x => x).AsEnumerable();
Console.WriteLine("Citizens filterd " + DateTime.Now.ToString());
foreach (var p in NewCitizens)
{
Console.WriteLine(p.VAT + " , " + p.Email + " , " + p.LastName + " , " + p.Bill_description);
}
Console.WriteLine("Citizens log " + DateTime.Now.ToString());
//only new citizens
foreach (var p in NewCitizens.Distinct())
{
var user = new ApplicationUser {
UserName = p.Email, Email = p.Email
};
var TempPass = GeneratePassword(3, 3, 3, 3);
// int lowercase, int uppercase, int numerics, int symbols
Console.WriteLine(TempPass);
var result = await _userManager.CreateAsync(user, TempPass);
if (result.Succeeded)
{
if (checkfile != "CitizenDebts_1M_3.txt")
{
Console.WriteLine("Sending Emails");
SendMail(p.LastName, p.Email, TempPass);
}
var query =
from UserUpd in _context.ApplicationUser
where UserUpd.Email == p.Email
select UserUpd;
foreach (ApplicationUser UserUpd in query)
{
UserUpd.VAT = p.VAT;
UserUpd.IsFirst = true;
UserUpd.LastName = p.LastName;
UserUpd.FirstName = p.FirstName;
UserUpd.Phone = p.Phone;
UserUpd.County = p.County;
UserUpd.Address = p.Address;
}
}
else
{
Console.WriteLine("#############################ALREADY REGISTERED " + p.VAT + " , " + p.Email + " , " + p.LastName);
}
NewUsers++;
}
;
await _context.SaveChangesAsync();
Console.WriteLine("New Users Registered and Emailed " + DateTime.Now.ToString());
foreach (var a in _context.ApplicationUser)
{
var query2 =
from UIdUpd in _context.CitizenDepts
where UIdUpd.VAT == a.VAT
select UIdUpd;
foreach (CitizenDepts UIdUpd in query2)
{
UIdUpd.UserGUId = a.Id;
}
}
;
await _context.SaveChangesAsync();
List <Bills> NewBillsls = new List <Bills>();
Bills NewBill = new Bills();
List <CitizenDepts> UpdCit = new List <CitizenDepts>();
UpdCit.AddRange(_context.CitizenDepts);
foreach (var e in UpdCit)
{
Console.WriteLine("#############################CITIZEN_DEPTS " + e.VAT + " , " + e.Email + " , " + e.UserGUId);
NewBill.GuId = e.BillId;
NewBill.Amount = e.Amount;
NewBill.DueDate = e.DueDate;
NewBill.Bill_description = e.Bill_description;
NewBill.Status = 0;
NewBill.UserId = e.UserGUId;
NewBill.PaymentMethodId = 1;
NewBill.SettlementId = 1;
NewBillsls.Add(NewBill);
NewBill = new Bills();
}
//.... delete bills
var allBills = from c in _context.Bills select c;
_context.Bills.RemoveRange(allBills);
await _context.SaveChangesAsync();
//.... delete settlements
_context.Settlements.RemoveRange(
_context.Settlements
.Where(s => s.ID != 1)
);
await _context.SaveChangesAsync();
_context.Bills.AddRange(NewBillsls);
await _context.SaveChangesAsync();
string Mes;
if (ErrorLines.Count() > 0)
{
Mes = "Ύπήρχαν " + ErrorLines.Count() + " σφάλματα στην εισαγωγή του αρχείου";
}
else
{
Mes = "Δεν Υπήρχαν Σφάλματα στην εισαγωγή του αρχείου";
}
// return View(NewCitizens);
return(RedirectToAction("DeptResults", "DeptResults", new DeptResults
{
BillsCount = NewBillsls.Count(),
NewUsers = NewUsers,
HttpStatus = "",
ErrorLines = ErrorLines,
Message = Mes
}));
}
else
{
return(NotFound(HttpStatusCode.NotFound + "\n" + "\nΔεν βρέθηκε ο κοινός φάκελος!"));
}
}
catch (Exception ex)
{
return(NotFound(HttpStatusCode.ExpectationFailed + "\n" + ex.Message + "\nΣφάλμα\nΑπροσδιόριστο σφάλμα Στην Εισαγωγή του αρχείου</b>"));
}
finally
{
Console.WriteLine("Import Finished Successfully " + DateTime.Now.ToString());
}
return(View());
}
internal IEnumerable <string> UploadResourceFilesToAzureFileStorage(List <Entity> resources, XDocument importXml, Configuration config, string folderDateTime, inRiverContext context)
{
Stopwatch saveFileStopWatch = new Stopwatch();
List <string> cloudFileNames = new List <string>();
try
{
// validate configuration settings
if (string.IsNullOrEmpty(config.StorageAccountName) ||
string.IsNullOrEmpty(config.StorageAccountKey) ||
string.IsNullOrEmpty(config.StorageAccountShareReference) ||
string.IsNullOrEmpty(config.StorageAccountCatalogDirectoryReference) ||
string.IsNullOrEmpty(config.StorageAccountResourcesDirectoryReference))
{
context.Log(LogLevel.Warning, $"Azure config settings are invalid: " +
$"StorageAccountName: {config.StorageAccountName}, " +
$"StorageAccountKey: {config.StorageAccountKey}, " +
$"StorageAccountShareReference: {config.StorageAccountShareReference}, " +
$"StorageAccountCatalogDirectoryReference: {config.StorageAccountCatalogDirectoryReference}, " +
$"StorageAccountResourcesDirectoryReference: {config.StorageAccountResourcesDirectoryReference}");
return(cloudFileNames);
}
// validate resources argument
if (resources == null)
{
context.Log(LogLevel.Error, "Resource is null!");
return(cloudFileNames);
}
// create varible files to hold filename and binary
Dictionary <string, byte[]> files = new Dictionary <string, byte[]>();
// setup credentials and storage account
StorageCredentials credentials = new StorageCredentials(config.StorageAccountName, config.StorageAccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(credentials, true);
// setup file client and remoge share
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference(config.StorageAccountShareReference);
share.CreateIfNotExists();
// setup root directory and resource directory
CloudFileDirectory root = share.GetRootDirectoryReference();
CloudFileDirectory directory = root.GetDirectoryReference(config.GetAzureStorageDirectoryName(XmlDocumentType.Resources));
directory.CreateIfNotExists();
saveFileStopWatch.Start();
// setup resource helper object
Resources resourcesObj = new Resources(context);
// setup the uncompressed file size total counter
int totalFileSize = 0;
// setup the azure file counter
int fileCount = 0;
// setup the file id list
List <int> fileIdList = new List <int>();
foreach (Entity resource in resources)
{
// get the file id of the resource
int resourceFileId = resourcesObj.GetResourceFileId(resource);
// ensure the resource id has a proper id
if (resourceFileId < 0)
{
context.Log(LogLevel.Information, $"Resource with id:{resource.Id} has no value for ResourceFileId");
continue;
}
// loop through each display configurations
foreach (string displayConfiguration in resourcesObj.GetDisplayConfigurations(resource, config))
{
// setup the fileName to use the output file extension from display configuration
string fileName = resourcesObj.GetResourceFileName(resource, resourceFileId, displayConfiguration, config);
// if the file for some reason already exists, continue to the next display configuration
if (files.ContainsKey(fileName))
{
context.Log(LogLevel.Debug, $"{fileName} already exists in the files collection and is skipped");
continue;
}
// get file bytes
byte[] resourceData = context.ExtensionManager.UtilityService.GetFile(resourceFileId, displayConfiguration);
// make sure we recieved the file from the utility service
if (resourceData == null)
{
context.Log(LogLevel.Error, $"Resource with id:{resource.Id} and ResourceFileId: {resourceFileId} could not get file");
continue;
}
// add the current resource file id to the list
fileIdList.Add(resource.Id);
// log the resource file name
context.Log(LogLevel.Debug, $"Adding resource {displayConfiguration}/{fileName}");
// add the file to the files collection
files.Add($"{displayConfiguration}/{fileName}", resourceData);
// add size to total file size counter
totalFileSize += resourceData.Length;
}
if (totalFileSize > (config.TotalResourceSizeLimitMb * 1024 * 1024))
{
try
{
// increase file counter
fileCount++;
// setup remote zip file
CloudFile cloudFile = directory.GetFileReference(GetZipFileName(config, XmlDocumentType.Resources, folderDateTime, fileCount));
// setup reader
using (XmlReader reader = importXml.CreateReader())
{
// create a new file that only contains the elements specified in the file id list
XmlDocument doc = XmlSplitUtility.GetPartialResourcesXml(reader, fileIdList);
// log the resource file name
context.Log(LogLevel.Debug, "Adding Resources.xml");
// setup memory a stream
using (MemoryStream stream = new MemoryStream())
{
// create a xml writer to format output
using (XmlWriter writer = XmlWriter.Create(stream, new XmlWriterSettings {
Indent = true
}))
{
// save partial document to the xml writer
doc.Save(writer);
// add the partial document to the files collection
files.Add("Resources.xml", stream.ToArray());
}
}
// send the zipped file to Azure and store the file name
cloudFileNames.Add(ZipAndUploadToCloud(config, context, files, directory, cloudFile));
// clear the id list
fileIdList.Clear();
// clear the file list
files.Clear();
// reset total file size
totalFileSize = 0;
}
}
catch (Exception ex)
{
context.Log(LogLevel.Error, "An error occured while sending the resources to the cloud.", ex);
}
}
}
// make sure to send the final files
if (files.Any())
{
try
{
// increase file counter
fileCount++;
// setup remote zip file
CloudFile cloudFile = directory.GetFileReference(GetZipFileName(config, XmlDocumentType.Resources, folderDateTime, fileCount));
// setup reader
using (XmlReader reader = importXml.CreateReader())
{
// create a new file that only contains the elements specified in the file id list
XmlDocument doc = XmlSplitUtility.GetPartialResourcesXml(reader, fileIdList);
// log the resource file name
context.Log(LogLevel.Debug, "Adding Resources.xml");
// setup memory a stream
using (MemoryStream stream = new MemoryStream())
{
// create a xml writer to format output
using (XmlWriter writer = XmlWriter.Create(stream, new XmlWriterSettings {
Indent = true
}))
{
// save partial document to the xml writer
doc.Save(writer);
// add the partial document to the files collection
files.Add("Resources.xml", stream.ToArray());
}
}
// send the zipped file to Azure and store the file name
cloudFileNames.Add(ZipAndUploadToCloud(config, context, files, directory, cloudFile));
}
}
catch (Exception ex)
{
context.Log(LogLevel.Error, "An error occured while sending the resources to the cloud.", ex);
}
}
}
catch (Exception ex)
{
context.Log(LogLevel.Error, "An error occured while sending the resources to the cloud.", ex);
}
return(cloudFileNames);
}
public static bool ZipDocumentAndUploadToAzure(XmlDocumentType xmlDocumentType, XDocument doc, Configuration config, string dateTimeStamp, IDictionary <string, byte[]> files = null)
{
if (IsNullOrEmpty(config.StorageAccountName) ||
IsNullOrEmpty(config.StorageAccountKey) ||
IsNullOrEmpty(config.StorageAccountShareReference) ||
IsNullOrEmpty(config.StorageAccountCatalogDirectoryReference) ||
IsNullOrEmpty(config.StorageAccountResourcesDirectoryReference))
{
return(false);
}
StorageCredentials cred = new StorageCredentials(config.StorageAccountName, config.StorageAccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(cred, true);
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference(config.StorageAccountShareReference);
share.CreateIfNotExists();
CloudFileDirectory root = share.GetRootDirectoryReference();
CloudFileDirectory dir = root.GetDirectoryReference(config.GetAzureStorageDirectoryName(xmlDocumentType));
dir.CreateIfNotExists();
CloudFile cloudFile = dir.GetFileReference(GetZipFileName(config, xmlDocumentType, dateTimeStamp));
using (MemoryStream stream = new MemoryStream())
{
XmlWriterSettings xws = new XmlWriterSettings
{
OmitXmlDeclaration = false,
Indent = true
};
using (ZipArchive archive = new ZipArchive(stream, ZipArchiveMode.Create, true))
{
if (files != null)
{
foreach (KeyValuePair <string, byte[]> imageFile in files)
{
ZipArchiveEntry zipEntry = archive.CreateEntry(imageFile.Key);
using (Stream entryStream = zipEntry.Open())
{
entryStream.Write(imageFile.Value, 0, imageFile.Value.Length);
}
}
}
ZipArchiveEntry entry = archive.CreateEntry("catalog.xml");
using (Stream entryStream = entry.Open())
{
using (XmlWriter xw = XmlWriter.Create(entryStream, xws))
{
doc.WriteTo(xw);
}
}
}
stream.Position = 0;
cloudFile.UploadFromStream(stream);
switch (xmlDocumentType)
{
case XmlDocumentType.Catalog:
config.CatalogPathInCloud = cloudFile.Name;
break;
default:
config.ResourceNameInCloud = cloudFile.Name;
break;
}
return(true);
}
}
// Please set the following connection strings in app.config for this WebJob to run:
// AzureWebJobsDashboard and AzureWebJobsStorage
static void Main()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("AzureWebJobsStorage"));
// Create a CloudFileClient object for credentialed access to File storage.
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
// Get a reference to the file share we created previously.
CloudFileShare share = fileClient.GetShareReference("Log");
// Ensure that the share exists.
if (share.Exists())
{
//do something
}
var config = new JobHostConfiguration();
if (config.IsDevelopment)
{
config.UseDevelopmentSettings();
}
config.UseTimers();
_logger.Log(LogLevel.Info, "Service started");
try
{
if (profile == null)
{
Console.WriteLine(string.Format("No Profile id : {0}", ProfileId));
_logger.Log(LogLevel.Info, string.Format("No Profile id : {0}", ProfileId));
return;
}
Instagram.ProfileId = profile.Id;
//Instagram.InitPhantomDriver();
Instagram.InitFirefoxDriver();
Instagram.Open("https://www.instagram.com/");
Instagram.InitCookiesAndRefreshPage(profile.ProfileName);
if (Instagram.IsNeedToLogin())
{
Instagram.Login(profile.Login, profile.Password, profile.ProfileName);
}
else
{
_logger.Log(LogLevel.Info, "Service already logged!");
}
}
catch (Exception ex)
{
Console.WriteLine(string.Format("Service exception: {0}", ex));
_logger.Log(LogLevel.Error, string.Format("Service exception: {0}", ex));
}
var host = new JobHost(config);
// The following code ensures that the WebJob will be running continuously
host.RunAndBlock();
}
static void Main(string[] args)
{
int tsOffset = Convert.ToInt32(ConfigurationManager.AppSettings.Get("TimeStampOffset"));
long timestamp_from = (long)Functions.ToEpoch(DateTime.UtcNow.AddMinutes(tsOffset));
long timestamp_to = (long)Functions.ToEpoch(DateTime.UtcNow);
List <RTVehiclePositions> vehiclePositions = db.Database
.SqlQuery <RTVehiclePositions>("RTVehiclePositionsGetActive @timestamp_from, @timestamp_to"
, new SqlParameter("@timestamp_from", timestamp_from)
, new SqlParameter("@timestamp_to", timestamp_to))
.ToList();
FeedMessage feed = new FeedMessage();
if (vehiclePositions.Count > 0)
{
foreach (RTVehiclePositions vehiclePosition in vehiclePositions)
{
TripDescriptor td = new TripDescriptor();
td.TripId = vehiclePosition.trip_id;
td.RouteId = vehiclePosition.route_id;
td.DirectionId = (uint)vehiclePosition.direction_id;
td.StartDate = vehiclePosition.start_date;
td.StartTime = vehiclePosition.start_time;
Position pos = new Position();
pos.Latitude = (float)vehiclePosition.latitude;
pos.Longitude = (float)vehiclePosition.longitude;
VehicleDescriptor v = new VehicleDescriptor();
v.Id = vehiclePosition.vehicle_id;
v.Label = vehiclePosition.vehicle_label;
v.LicensePlate = vehiclePosition.vehicle_license_plate;
VehiclePosition vp = new VehiclePosition();
vp.Position = pos;
vp.Trip = td;
vp.Vehicle = v;
vp.CurrentStopSequence = (uint)vehiclePosition.current_stop_sequence;
vp.StopId = vehiclePosition.stop_id;
vp.Timestamp = (ulong)vehiclePosition.timestamp;
FeedEntity entity = new FeedEntity();
entity.Id = vehiclePosition.id.ToString();
entity.Vehicle = vp;
feed.Entities.Add(entity);
}
byte[] objSerialized = Functions.ProtoSerialize(feed);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
string filename = "vehicleposition.pb";
// Create a CloudFileClient object for credentialed access to Azure Files.
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
// Get a reference to the file share we created previously.
CloudFileShare share = fileClient.GetShareReference("gtfsrt");
share.CreateIfNotExists();
var rootDir = share.GetRootDirectoryReference();
using (var stream = new MemoryStream(objSerialized, writable: false))
{
rootDir.GetFileReference(filename).UploadFromStream(stream);
}
}
}
public CloudFileUtil(CloudStorageAccount account)
{
this.account = account;
this.client = account.CreateCloudFileClient();
}
public StorageManager(string settingsString)
{
StorageAccount = CloudStorageAccount.Parse(settingsString);
blobClient = StorageAccount.CreateCloudBlobClient();
fileClient = StorageAccount.CreateCloudFileClient();
}
/// <summary>
/// Test some of the file storage operations.
/// </summary>
private static async Task RunFileStorageOperationsAsync()
{
try
{
//***** Setup *****//
Console.WriteLine("Getting reference to the storage account.");
// Retrieve storage account information from connection string
// How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
CloudStorageAccount storageAccount = CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString"));
Console.WriteLine("Instantiating file client.");
// Create a file client for interacting with the file service.
CloudFileClient cloudFileClient = storageAccount.CreateCloudFileClient();
// Create the share name -- use part of a guid in the name so it's most likely to be unique.
// This will also be used as the container name for blob storage when copying the file to blob storage.
string shareName = "demotest-" + System.Guid.NewGuid().ToString().Substring(0, 12);
// Name of folder to put the files in
string sourceFolder = "testfolder";
// Name of file to upload and download
string testFile = "HelloWorld.png";
// Folder where the HelloWorld.png file resides
string localFolder = @".\";
// It won't let you download in the same folder as the exe file,
// so use a temporary folder with the same name as the share.
string downloadFolder = Path.Combine(Path.GetTempPath(), shareName);
//***** Create a file share *****//
// Create the share if it doesn't already exist.
Console.WriteLine("Creating share with name {0}", shareName);
CloudFileShare cloudFileShare = cloudFileClient.GetShareReference(shareName);
try
{
await cloudFileShare.CreateIfNotExistsAsync();
Console.WriteLine(" Share created successfully.");
}
catch (StorageException exStorage)
{
WriteException(exStorage);
Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample.");
Console.WriteLine("Press any key to exit");
Console.ReadLine();
throw;
}
catch (Exception ex)
{
Console.WriteLine(" Exception thrown creating share.");
WriteException(ex);
throw;
}
//***** Create a directory on the file share *****//
// Create a directory on the share.
Console.WriteLine("Creating directory named {0}", sourceFolder);
// First, get a reference to the root directory, because that's where you're going to put the new directory.
CloudFileDirectory rootDirectory = cloudFileShare.GetRootDirectoryReference();
CloudFileDirectory fileDirectory = null;
// Set a reference to the file directory.
// If the source folder is null, then use the root folder.
// If the source folder is specified, then get a reference to it.
if (string.IsNullOrWhiteSpace(sourceFolder))
{
// There is no folder specified, so return a reference to the root directory.
fileDirectory = rootDirectory;
Console.WriteLine(" Using root directory.");
}
else
{
// There was a folder specified, so return a reference to that folder.
fileDirectory = rootDirectory.GetDirectoryReference(sourceFolder);
await fileDirectory.CreateIfNotExistsAsync();
Console.WriteLine(" Directory created successfully.");
}
//***** Upload a file to the file share *****//
// Set a reference to the file.
CloudFile cloudFile = fileDirectory.GetFileReference(testFile);
// Upload a file to the share.
Console.WriteLine("Uploading file {0} to share", testFile);
// Set up the name and path of the local file.
string sourceFile = Path.Combine(localFolder, testFile);
if (File.Exists(sourceFile))
{
// Upload from the local file to the file share in azure.
await cloudFile.UploadFromFileAsync(sourceFile, FileMode.OpenOrCreate);
Console.WriteLine(" Successfully uploaded file to share.");
}
else
{
Console.WriteLine("File not found, so not uploaded.");
}
//***** Get list of all files/directories on the file share*****//
// List all files/directories under the root directory.
Console.WriteLine("Getting list of all files/directories under the root directory of the share.");
IEnumerable <IListFileItem> fileList = cloudFileShare.GetRootDirectoryReference().ListFilesAndDirectories();
// Print all files/directories listed above.
foreach (IListFileItem listItem in fileList)
{
// listItem type will be CloudFile or CloudFileDirectory.
Console.WriteLine(" - {0} (type: {1})", listItem.Uri, listItem.GetType());
}
Console.WriteLine("Getting list of all files/directories in the file directory on the share.");
// Now get the list of all files/directories in your directory.
// Ordinarily, you'd write something recursive to do this for all directories and subdirectories.
fileList = fileDirectory.ListFilesAndDirectories();
// Print all files/directories in the folder.
foreach (IListFileItem listItem in fileList)
{
// listItem type will be CloudFile or CloudFileDirectory.
Console.WriteLine(" - {0} (type: {1})", listItem.Uri, listItem.GetType());
}
//***** Download a file from the file share *****//
// Download the file to the downloadFolder in the temp directory.
// Check and if the directory doesn't exist (which it shouldn't), create it.
Console.WriteLine("Downloading file from share to local temp folder {0}.", downloadFolder);
if (!Directory.Exists(downloadFolder))
{
Directory.CreateDirectory(downloadFolder);
}
// Download the file.
await cloudFile.DownloadToFileAsync(Path.Combine(downloadFolder, testFile), FileMode.OpenOrCreate);
Console.WriteLine(" Successfully downloaded file from share to local temp folder.");
//***** Copy a file from the file share to blob storage, then abort the copy *****//
// To really test this, you might have to find a large file, or else the file finishes copying before you can abort the copy.
// You need to upload the file to the share. Then you can assign the name of hte file to the testFile variable, and it will use
// that file for the copy and abort here.
CloudFile cloudFileCopy = fileDirectory.GetFileReference(testFile);
// Upload a file to the share.
Console.WriteLine("Uploading file {0} to share", testFile);
// Set up the name and path of the local file.
string sourceFileCopy = Path.Combine(localFolder, testFile);
await cloudFileCopy.UploadFromFileAsync(sourceFileCopy, FileMode.OpenOrCreate);
Console.WriteLine(" Successfully uploaded file to share.");
// Copy the file to blob storage.
Console.WriteLine("Copying file to blob storage. Container name = {0}", shareName);
// First get a reference to the blob.
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
// Get a reference to the blob container and create it if it doesn't already exist.
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(shareName);
cloudBlobContainer.CreateIfNotExists();
// Get a blob reference to the target blob.
CloudBlob targetBlob = cloudBlobContainer.GetBlobReference(testFile);
string copyId = string.Empty;
// Get a reference to the file to be copied.
cloudFile = fileDirectory.GetFileReference(testFile);
// Create a SAS for the file that's valid for 24 hours.
// Note that when you are copying a file to a blob, or a blob to a file, you must use a SAS
// to authenticate access to the source object, even if you are copying within the same
// storage account.
string fileSas = cloudFile.GetSharedAccessSignature(new SharedAccessFilePolicy()
{
// Only read permissions are required for the source file.
Permissions = SharedAccessFilePermissions.Read,
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(24)
});
// Construct the URI to the source file, including the SAS token.
Uri fileSasUri = new Uri(cloudFile.StorageUri.PrimaryUri.ToString() + fileSas);
// Start the copy of the file to the blob.
copyId = await targetBlob.StartCopyAsync(fileSasUri);
Console.WriteLine(" File copy started successfully. copyID = {0}", copyId);
// Abort the copy of the file to blob storage.
// Note that you call Abort on the target object, i.e. the blob, not the file.
// If you were copying from one file to another on the file share, the target object would be a file.
Console.WriteLine("Cancelling the copy operation.");
// Print out the copy state information.
targetBlob.FetchAttributes();
Console.WriteLine(" targetBlob.copystate.CopyId = {0}", targetBlob.CopyState.CopyId);
Console.WriteLine(" targetBlob.copystate.Status = {0}", targetBlob.CopyState.Status);
// Do the actual abort copy.
// This only works if the copy is still pending or ongoing.
if (targetBlob.CopyState.Status == CopyStatus.Pending)
{
// Call to stop the copy, passing in the copyId of the operation.
// This won't work if it has already finished copying.
await targetBlob.AbortCopyAsync(copyId);
Console.WriteLine(" Cancelling the copy succeeded.");
}
else
{
// If this happens, try a larger file.
Console.WriteLine(" Cancellation of copy not performed; copy has already finished.");
}
// Now clean up after yourself.
Console.WriteLine("Deleting the files from the file share.");
// Delete the files because cloudFile is a different file in the range sample.
cloudFile = fileDirectory.GetFileReference(testFile);
cloudFile.DeleteIfExists();
Console.WriteLine("Setting up files to test WriteRange and ListRanges.");
//***** Write 2 ranges to a file, then list the ranges *****//
// This is the code for trying out writing data to a range in a file,
// and then listing those ranges.
// Get a reference to a file and write a range of data to it .
// Then write another range to it.
// Then list the ranges.
// Start at the very beginning of the file.
long startOffset = 0;
// Set the destination file name -- this is the file on the file share that you're writing to.
string destFile = "rangeops.txt";
cloudFile = fileDirectory.GetFileReference(destFile);
// Create a string with 512 a's in it. This will be used to write the range.
int testStreamLen = 512;
string textToStream = string.Empty;
textToStream = textToStream.PadRight(testStreamLen, 'a');
// Name to be used for the file when downloading it so you can inspect it locally
string downloadFile;
using (MemoryStream ms = new MemoryStream(Encoding.Default.GetBytes(textToStream)))
{
// Max size of the output file; have to specify this when you create the file
// I picked this number arbitrarily.
long maxFileSize = 65536;
Console.WriteLine("Write first range.");
// Set the stream back to the beginning, in case it's been read at all.
ms.Position = 0;
// If the file doesn't exist, create it.
// The maximum file size is passed in. It has to be big enough to hold
// all the data you're going to write, so don't set it to 256k and try to write two 256-k blocks to it.
if (!cloudFile.Exists())
{
Console.WriteLine("File doesn't exist, create empty file to write ranges to.");
// Create a file with a maximum file size of 64k.
await cloudFile.CreateAsync(maxFileSize);
Console.WriteLine(" Empty file created successfully.");
}
// Write the stream to the file starting at startOffset for the length of the stream.
Console.WriteLine("Writing range to file.");
await cloudFile.WriteRangeAsync(ms, startOffset, null);
// Download the file to your temp directory so you can inspect it locally.
downloadFile = Path.Combine(downloadFolder, "__testrange.txt");
Console.WriteLine("Downloading file to examine.");
await cloudFile.DownloadToFileAsync(downloadFile, FileMode.OpenOrCreate);
Console.WriteLine(" Successfully downloaded file with ranges in it to examine.");
}
// Now add the second range, but don't make it adjacent to the first one, or it will show only
// one range, with the two combined. Put it like 1000 spaces away. When you get the range back, it will
// start at the position at the 512-multiple border prior or equal to the beginning of the data written,
// and it will end at the 512-multliple border after the actual end of the data.
//For example, if you write to 2000-3000, the range will be the 512-multiple prior to 2000, which is
// position 1536, or offset 1535 (because it's 0-based).
// And the right offset of the range will be the 512-multiple after 3000, which is position 3072,
// or offset 3071 (because it's 0-based).
Console.WriteLine("Getting ready to write second range to file.");
startOffset += testStreamLen + 1000; //randomly selected number
// Create a string with 512 b's in it. This will be used to write the range.
textToStream = string.Empty;
textToStream = textToStream.PadRight(testStreamLen, 'b');
using (MemoryStream ms = new MemoryStream(Encoding.Default.GetBytes(textToStream)))
{
ms.Position = 0;
// Write the stream to the file starting at startOffset for the length of the stream.
Console.WriteLine("Write second range to file.");
await cloudFile.WriteRangeAsync(ms, startOffset, null);
Console.WriteLine(" Successful writing second range to file.");
// Download the file to your temp directory so you can examine it.
downloadFile = Path.Combine(downloadFolder, "__testrange2.txt");
Console.WriteLine("Downloading file with two ranges in it to examine.");
await cloudFile.DownloadToFileAsync(downloadFile, FileMode.OpenOrCreate);
Console.WriteLine(" Successfully downloaded file to examine.");
}
// Query and view the list of ranges.
Console.WriteLine("Call to get the list of ranges.");
IEnumerable <FileRange> listOfRanges = await cloudFile.ListRangesAsync();
Console.WriteLine(" Successfully retrieved list of ranges.");
foreach (FileRange fileRange in listOfRanges)
{
Console.WriteLine(" --> filerange startOffset = {0}, endOffset = {1}", fileRange.StartOffset, fileRange.EndOffset);
}
//***** Clean up *****//
// Clean up after yourself.
Console.WriteLine("Removing all files, folders, shares, blobs, and containers created in this demo.");
// Delete the file with the ranges in it.
cloudFile = fileDirectory.GetFileReference(destFile);
await cloudFile.DeleteIfExistsAsync();
Console.WriteLine("Deleting the directory on the file share.");
// Delete the directory.
bool success = false;
success = await fileDirectory.DeleteIfExistsAsync();
if (success)
{
Console.WriteLine(" Directory on the file share deleted successfully.");
}
else
{
Console.WriteLine(" Directory on the file share NOT deleted successfully; may not exist.");
}
Console.WriteLine("Deleting the file share.");
// Delete the share.
await cloudFileShare.DeleteAsync();
Console.WriteLine(" Deleted the file share successfully.");
Console.WriteLine("Deleting the temporary download directory and the file in it.");
// Delete the download folder and its contents.
Directory.Delete(downloadFolder, true);
Console.WriteLine(" Successfully deleted the temporary download directory.");
Console.WriteLine("Deleting the container and blob used in the Copy/Abort test.");
await targetBlob.DeleteIfExistsAsync();
await cloudBlobContainer.DeleteIfExistsAsync();
Console.WriteLine(" Successfully deleted the blob and its container.");
}
catch (Exception ex)
{
Console.WriteLine(" Exception thrown. Message = {0}{1} Strack Trace = {2}", ex.Message, Environment.NewLine, ex.StackTrace);
}
}
public AzureStorageHelper(string connectionString)
{
_account = CloudStorageAccount.Parse(connectionString);
_client = _account.CreateCloudFileClient();
}
public AzureFileService(CloudStorageAccount cloudStorageAccount, ILogger <AzureFileService> logger)
{
this.cloudStorageAccount = cloudStorageAccount;
this.cloudFileClient = cloudStorageAccount.CreateCloudFileClient();
this.logger = logger;
}
public static void addRecordToAzureFileShare(string myQueueItem, string fileName, ILogger log)
{
try
{
string valueLine = string.Empty;
string clientHeader = string.Empty;
var searcResult = new string[10];
StringBuilder builder = new StringBuilder();
StringBuilder headerBuilder = new StringBuilder();
StringBuilder valueBuilder = new StringBuilder();
string[] arrQueueItem = myQueueItem.Split("||", StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < arrQueueItem.Length; i++)
{
var arrHeader = arrQueueItem[i]?.Split(":", StringSplitOptions.RemoveEmptyEntries);
if (!string.IsNullOrEmpty(arrHeader[0]) && !arrHeader[0].Trim().Equals("CampaignName"))
{
string header = arrHeader[0].Replace(",", "-");
string value = arrHeader.Length > 1 ? arrHeader[1].Replace(",", "-") : ""; //Replace comma with hyphen in values
headerBuilder.Append(header + ",");
valueBuilder.Append(value + ",");
}
}
var headerLastIndex = headerBuilder.ToString().LastIndexOf(",");
if (headerLastIndex >= 0)
{
clientHeader = headerBuilder.ToString().Remove(headerLastIndex, 1) + Environment.NewLine;
}
builder.Append(clientHeader);
var valueLastIndex = valueBuilder.ToString().LastIndexOf(",");
if (valueLastIndex >= 0)
{
valueLine = valueBuilder.ToString().Remove(valueLastIndex, 1);
}
builder.Append(valueLine);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(GetEnvironmentVariable("AzureWebJobsStorage"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare fileShare = fileClient.GetShareReference(GetEnvironmentVariable("AzureStorageFileShare"));
string existingFiletxt = "";
if (fileShare.Exists())
{
CloudFileDirectory rootDirectory = fileShare.GetRootDirectoryReference();
if (rootDirectory.Exists())
{
CloudFileDirectory customDirectory = rootDirectory.GetDirectoryReference(GetEnvironmentVariable("AzureStorageFileDirectory"));
if (customDirectory.Exists())
{
CloudFile file = customDirectory.GetFileReference(fileName + ".csv");
if (file.Exists())
{
existingFiletxt = file.DownloadTextAsync().Result;
string existingHeader = existingFiletxt.Split(new[] { Environment.NewLine }, StringSplitOptions.None)?[0];
string existingValue = existingFiletxt.Split(new[] { Environment.NewLine }, StringSplitOptions.None)[1];
if (existingHeader + Environment.NewLine == clientHeader)
{
customDirectory.GetFileReference(fileName + ".csv").UploadText(existingFiletxt + Environment.NewLine + valueLine);
}
else
{
existingFiletxt = clientHeader + existingValue;
customDirectory.GetFileReference(fileName + ".csv").UploadText(existingFiletxt + Environment.NewLine + valueLine.ToString());
}
}
else
{
customDirectory.GetFileReference(fileName + ".csv").UploadText(builder.ToString());
}
}
}
}
}
catch (Exception ex)
{
log.LogInformation("Some error occured " + ex);
}
}
public async Task <FileStreamResult> Get(int id)
{
var claimsIdentity = User.Identity as ClaimsIdentity;
var userId = Convert.ToInt64(claimsIdentity.Claims.FirstOrDefault(claim => claim.Type == "Id").Value);
var user = _multiSourcePlaylistRepository.GetUser(userId);
var track = _multiSourcePlaylistRepository.GetTrack(id);
byte[] audioArray = new byte[0];
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
_configuration["Production:StorageConnectionString"]);
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
// Get a reference to the file share we created previously.
CloudFileShare share = fileClient.GetShareReference(user.FileFolder);
CloudFileDirectory userDir = null;
// Ensure that the share exists.
if (await share.ExistsAsync())
{
// Get a reference to the root directory for the share.
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
// Get a reference to the directory we created previously.
userDir = rootDir.GetDirectoryReference("audio");
// Ensure that the directory exists.
if (await userDir.ExistsAsync())
{
var audiofile = userDir.GetFileReference(track.Address);
if (await audiofile.ExistsAsync())
{
await audiofile.FetchAttributesAsync();
audioArray = new byte[audiofile.Properties.Length];
await audiofile.DownloadToByteArrayAsync(audioArray, 0);
}
}
else
{
return(null);
}
}
else
{
return(null);
}
long fSize = audioArray.Length;
long startbyte = 0;
long endbyte = fSize - 1;
int statusCode = 200;
var rangeRequest = Request.Headers["Range"].ToString();
if (rangeRequest != "")
{
string[] range = Request.Headers["Range"].ToString().Split(new char[] { '=', '-' });
startbyte = Convert.ToInt64(range[1]);
if (range.Length > 2 && range[2] != "")
{
endbyte = Convert.ToInt64(range[2]);
}
if (startbyte != 0 || endbyte != fSize - 1 || range.Length > 2 && range[2] == "")
{
statusCode = 206;
}
}
long desSize = endbyte - startbyte + 1;
Response.StatusCode = statusCode;
Response.ContentType = "audio/mp3";
Response.Headers.Add("Content-Accept", Response.ContentType);
Response.Headers.Add("Content-Length", desSize.ToString());
Response.Headers.Add("Content-Range", string.Format("bytes {0}-{1}/{2}", startbyte, endbyte, fSize));
Response.Headers.Add("Accept-Ranges", "bytes");
Response.Headers.Remove("Cache-Control");
var stream = new MemoryStream(audioArray, (int)startbyte, (int)desSize);
return(new FileStreamResult(stream, "audio/mp3")
{
FileDownloadName = track.Name
});
}
public async static Task Run([TimerTrigger("0 0 0 * * Sun"
#if DEBUG
, RunOnStartup = true
#endif
)] TimerInfo myTimer, ILogger log, ExecutionContext context)
{
log.LogInformation($"SimpleGhostBackup function started execution at: {DateTime.Now}");
var config = new ConfigurationBuilder()
.SetBasePath(context.FunctionAppDirectory)
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.Build();
var clientId = config["ClientId"];
if (String.IsNullOrEmpty(clientId))
{
throw new ArgumentNullException("ClientId is Required!");
}
var clientSecret = config["ClientSecret"];
if (String.IsNullOrEmpty(clientSecret))
{
throw new ArgumentNullException("ClientSecret is Required!");
}
var blogUrl = config["BlogUrl"];
if (String.IsNullOrEmpty(blogUrl))
{
throw new ArgumentNullException("BlogUrl is Required!");
}
var storageShareName = config["StorageShareName"];
if (String.IsNullOrEmpty(storageShareName))
{
throw new ArgumentNullException("StorageShareName is Required!");
}
var storageConnection = config["StorageConnectionString"];
if (String.IsNullOrEmpty(storageConnection))
{
throw new ArgumentNullException("storageConnection is Required!");
}
// Let get the number of snapshots that we should keep, default to last 4
int maxSnapshots = 4;
Int32.TryParse(config["MaxSnapshots"], out maxSnapshots);
var client = new HttpClient(new HttpRetryMessageHandler(new HttpClientHandler()))
{
BaseAddress = new Uri(String.Format("https://{0}", blogUrl))
};
log.LogInformation($"Requesting Ghost Backup");
var response = await client.PostAsync(String.Format("/ghost/api/v0.1/db/backup?client_id={0}&client_secret={1}", clientId, clientSecret), null);
if (response.StatusCode == System.Net.HttpStatusCode.OK)
{
// Get our response content which contains the created backup file name
var content = await response.Content.ReadAsStringAsync();
var json = JObject.Parse(content);
// Connect to our Azure Storage Account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnection);
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference(storageShareName);
CloudFileDirectory root = share.GetRootDirectoryReference();
CloudFileDirectory data = root.GetDirectoryReference("data");
//Does the data folder exist
if (await data.ExistsAsync())
{
log.LogInformation($"Data folder exists.");
// get the backup file name
var filename = System.IO.Path.GetFileName((string)json["db"][0]["filename"]);
CloudFile file = data.GetFileReference(filename);
//Confirm that the backup file exists
if (await file.ExistsAsync())
{
// Create the snapshotg of the file share
log.LogInformation($"Backup file created - {filename}");
log.LogInformation($"Creating Azure Fileshare Snapshot");
var s = await share.SnapshotAsync();
if (s != null)
{
//Lets get all the current shares/snapshots
FileContinuationToken token = null;
var snapshots = new List <CloudFileShare>();
do
{
ShareResultSegment resultSegment = await fileClient.ListSharesSegmentedAsync(storageShareName, ShareListingDetails.Snapshots, 5, token, null, null);
snapshots.AddRange(resultSegment.Results);
token = resultSegment.ContinuationToken;
}while (token != null);
//lets delete the old ones
var toDelete = snapshots.Where(os => os.IsSnapshot).OrderByDescending(oos => oos.SnapshotTime).Skip(maxSnapshots).ToList();
foreach (var snapshot in toDelete)
{
try
{
log.LogInformation($"Deleting snapshot - {snapshot.Name}, Created at {snapshot.SnapshotTime}");
await snapshot.DeleteAsync();
}
catch (Exception ex)
{
log.LogError($"Failed to delete snapshot - '{ex}'");
}
}
}
}
}
}
log.LogInformation($"SimpleGhostBackup function ended execution at: {DateTime.Now}");
}
public JsonResult GetSingleUserTaxReturn(string userId, int taxYear)
{
bool downloadSuccessful = true;
var results = new List <Tuple <string, string, byte[]> >();
using (var db = new WorktripEntities())
{
try
{
Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)");
var user = db.Users.FirstOrDefault(u => u.Id == userId);
var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear);
taxReturn = taxReturn.OrderBy(d => d.Id);
var fileUrls = new List <UserTaxReturn>();
if (taxReturn.Count() != 0)
{
fileUrls.Add(taxReturn.AsEnumerable().Last());
byte[] bytes = new byte[64000];
var parsedFilePaths = new List <Tuple <string, string> >();
foreach (var url in fileUrls)
{
Match match = filePattern.Match(url.Path);
if (match.Success)
{
var newTuple = new Tuple <string, string>(
match.Groups["directory"].Value,
match.Groups["filename"].Value
);
parsedFilePaths.Add(newTuple);
}
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("worktripdocs");
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
CloudFileDirectory userDir = null;
var userDirName = "";
foreach (var parsedPath in parsedFilePaths)
{
if (userDirName != parsedPath.Item1)
{
userDir = rootDir.GetDirectoryReference(parsedPath.Item1);
if (!userDir.Exists())
{
continue;
}
userDirName = parsedPath.Item1;
}
var filename = parsedPath.Item2;
CloudFile file = userDir.GetFileReference(filename);
if (!file.Exists())
{
continue;
}
file.FetchAttributes();
string fileContents = "";
if (file.Properties.ContentType.ToLower() == "application/pdf")
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
bytes = fileStream.ToArray();
fileContents = ConvertStreamToBase64String(fileStream);
}
else
{
fileContents = file.DownloadText();
}
results.Add(
new Tuple <string, string, byte[]>(filename, file.Properties.ContentType, bytes)
);
}
}
}
catch (Exception e)
{
//Do some error logging here..
downloadSuccessful = false;
}
}
if (downloadSuccessful && results.Count > 0)
{
return(Json(new MyJsonResult
{
status = 0,
fileName = results.ElementAtOrDefault(0).Item1,
fileContentType = results.ElementAtOrDefault(0).Item2,
fileContents = results.ElementAtOrDefault(0).Item3
}));
}
else
{
return(Json(new { status = -1, message = "Error in downloading files" }));
}
}
/// <summary>
/// Test some of the file storage operations.
/// </summary>
public async Task RunFileStorageAdvancedOpsAsync()
{
// Keep a list of the file shares so you can compare this list
// against the list of shares that we retrieve .
List <string> fileShareNames = new List <string>();
// Create a file client for interacting with the file service.
CloudFileClient cloudFileClient = null;
try
{
//***** Setup *****//
Console.WriteLine("Getting reference to the storage account.");
// Retrieve storage account information from connection string
// How to create a storage connection string - http://msdn.microsoft.com/en-us/library/azure/ee758697.aspx
CloudStorageAccount storageAccount = Common.CreateStorageAccountFromConnectionString(CloudConfigurationManager.GetSetting("StorageConnectionString"));
Console.WriteLine("Instantiating file client.");
Console.WriteLine(string.Empty);
// Create a file client for interacting with the file service.
cloudFileClient = storageAccount.CreateCloudFileClient();
// Create 3 file shares.
// Create the share name -- use a guid in the name so it's unique.
// This will also be used as the container name for blob storage when copying the file to blob storage.
string baseShareName = "demotest-" + System.Guid.NewGuid().ToString();
for (int i = 0; i < 3; i++)
{
// Set the name of the share, then add it to the generic list.
string shareName = baseShareName + "-0" + i;
fileShareNames.Add(shareName);
// Create the share with this name.
Console.WriteLine("Creating share with name {0}", shareName);
CloudFileShare cloudFileShare = cloudFileClient.GetShareReference(shareName);
try
{
await cloudFileShare.CreateIfNotExistsAsync();
Console.WriteLine(" Share created successfully.");
}
catch (StorageException exStorage)
{
Common.WriteException(exStorage);
Console.WriteLine("Please make sure your storage account has storage file endpoint enabled and specified correctly in the app.config - then restart the sample.");
Console.WriteLine("Press any key to exit");
Console.ReadLine();
throw;
}
catch (Exception ex)
{
Console.WriteLine(" Exception thrown creating share.");
Common.WriteException(ex);
throw;
}
}
Console.WriteLine(string.Empty);
Console.WriteLine("List of shares in the storage account:");
// List the file shares for this storage account
IEnumerable <CloudFileShare> cloudShareList = cloudFileClient.ListShares();
try
{
foreach (CloudFileShare cloudShare in cloudShareList)
{
Console.WriteLine("Cloud Share name = {0}", cloudShare.Name);
}
}
catch (Exception ex)
{
Console.WriteLine(" Exception thrown listing shares.");
Common.WriteException(ex);
throw;
}
}
catch (Exception ex)
{
Console.WriteLine(" Exception thrown. Message = {0}{1} Strack Trace = {2}", ex.Message, Environment.NewLine, ex.StackTrace);
}
finally
{
// If it created the file shares, remove them (cleanup).
if (fileShareNames != null && cloudFileClient != null)
{
// Now clean up after yourself, using the list of shares that you created in case there were other shares in the account.
foreach (string fileShareName in fileShareNames)
{
CloudFileShare cloudFileShare = cloudFileClient.GetShareReference(fileShareName);
cloudFileShare.DeleteIfExists();
}
}
}
}
public JsonResult UploadUserTaxReturn(int taxYear, string subFolder, string userId)
{
var curUserId = userId == null?User.Identity.GetUserId() : userId;
bool uploadedSuccessfully = false;
string uploadedURI = "";
using (var db = new WorktripEntities())
{
try
{
var user = db.Users.FirstOrDefault(u => u.Id == curUserId);
HttpPostedFileBase uploadedFile = Request.Files.Get(0);
var compressedStream = uploadedFile.InputStream;
if (uploadedFile.ContentType.StartsWith("image/"))
{
compressedStream = ResizePictureForBandwidth(uploadedFile);
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("worktripdocs");
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
CloudFileDirectory userDir = rootDir.GetDirectoryReference(user.FirstName + " " + user.LastName + " " + user.PhoneNumber);
userDir.CreateIfNotExists();
if (!string.IsNullOrEmpty(subFolder))
{
userDir = userDir.GetDirectoryReference(subFolder);
userDir.CreateIfNotExists();
}
var newFileName = uploadedFile.FileName;
var fileExtension = Path.GetExtension(newFileName);
CloudFile file = userDir.GetFileReference(newFileName);
int fileDuplicateCount = 1;
while (file.Exists())
{
//generate a file name that doesn't exist yet
newFileName = Path.GetFileNameWithoutExtension(newFileName) + "(" + (fileDuplicateCount++) + ")" + fileExtension;
file = userDir.GetFileReference(newFileName);;
}
file.Properties.ContentType = uploadedFile.ContentType;
file.UploadFromStream(compressedStream);
uploadedURI = file.Uri.ToString();
UserTaxReturn newReturn = new UserTaxReturn()
{
UserId = curUserId,
Date = DateTime.UtcNow,
Path = uploadedURI,
Year = taxYear
};
db.UserTaxReturns.Add(newReturn);
db.SaveChanges();
uploadedSuccessfully = true;
UserInfoViewModel.UpdateUserActionsLog(curUserId, "uploaded " + taxYear + " " + (fileExtension == ".pdf" ? "tax return" : "tax form(s)"));
}
catch (Exception e)
{
//Do some error logging here..
uploadedSuccessfully = false;
}
}
if (uploadedSuccessfully)
{
return(Json(new { status = 0 }));
}
else
{
return(Json(new { status = -1, message = "Error in saving file" }));
}
}
public static void GetAthleteUpdates(TextWriter log)
{
Console.WriteLine("Checking the Ironman Athlete Tracker!");
string urlpattern = ConfigurationManager.AppSettings["trackURL"];
string allReferences = ConfigurationManager.AppSettings["athleteRefs"];
string connectionString = ConfigurationManager.ConnectionStrings["AzureCloudStorage"].ConnectionString;
// In format raceid-name:bib-name;bib-name/
string pushoverAPIkey = ConfigurationManager.AppSettings["pushoverAPIkey"];
string pushoverGroupKey = ConfigurationManager.AppSettings["pushoverGroupKey"];
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
// Create a reference to the file client.
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("tracking");
// Create a reference to the Azure path
CloudFileDirectory cloudFileDirectory = share.GetRootDirectoryReference();
foreach (string trackinfo in allReferences.Split('/'))
{
string raceinfo = trackinfo.Split(':')[0];
string raceid = raceinfo.Split('-')[0];
string racename = raceinfo.Split('-')[1];
string[] athletes = trackinfo.Split(':')[1].Split(';');
foreach (string athlete in athletes)
{
string bib = athlete.Split('-')[0];
string athleteName = athlete.Split('-')[1];
string url = urlpattern.Replace("$rid$", raceid).Replace("$bib$", bib);
try
{
Console.WriteLine("About to download athelete url: " + url);
WebClient client = new WebClient();
string newFile = client.DownloadString(url);
Console.WriteLine(newFile);
int pFrom = newFile.IndexOf(@"<!-- Begin: main content area -->");
int pTo = newFile.IndexOf(@"<!-- End: main content area -->");
string latestTrackingData = newFile.Substring(pFrom, pTo - pFrom);
string filename = raceid + "-" + bib + ".html";
try
{
CloudFile cloudFile = cloudFileDirectory.GetFileReference(filename);
bool isUpdated = false;
if (cloudFile.Exists())
{
StreamReader sr = new StreamReader(cloudFile.OpenRead());
string previousTrackingData = sr.ReadToEnd();
if (!latestTrackingData.Equals(previousTrackingData, StringComparison.InvariantCultureIgnoreCase))
{
isUpdated = true;
}
sr.Close();
}
cloudFile.UploadTextAsync(latestTrackingData);
if (isUpdated)
{
Pushover pclient = new Pushover(pushoverAPIkey);
PushResponse response = pclient.Push(
racename + " athlete alert",
"Looks like " + athleteName + " has a new split update!",
pushoverGroupKey
);
}
}
catch (Exception ex1)
{
Console.WriteLine("Error!");
Console.WriteLine(ex1.Message);
Console.WriteLine(ex1.StackTrace);
}
}
catch (Exception ex2)
{
Console.WriteLine("Error!");
Console.WriteLine(ex2.Message);
Console.WriteLine(ex2.StackTrace);
}
}
}
}
static void Main(string[] args)
{
string shareName = "\\\\unipertest.file.core.windows.net\\SecretDocuments";
string driveLetterAndColon = "z:";
string username = "******";
string password = "******";
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create a CloudFileClient object for credentialed access to File storage.
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
// Get a reference to the file share we created previously.
CloudFileShare share = fileClient.GetShareReference("secretdocuments");
// Ensure that the share exists.
if (share.Exists())
{
// Get a reference to the root directory for the share.
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
// Get a reference to the directory we created previously.
CloudFileDirectory sampleDir = rootDir.GetDirectoryReference("CustomLogs");
// Ensure that the directory exists.
if (sampleDir.Exists())
{
// Get a reference to the file we created previously.
CloudFile file = sampleDir.GetFileReference("Log1.txt");
// Ensure that the file exists.
if (file.Exists())
{
// Write the contents of the file to the console window.
Console.WriteLine(file.DownloadTextAsync().Result);
Console.WriteLine("Successfully connected to file share");
System.Threading.Thread.Sleep(300);
}
}
if (!String.IsNullOrEmpty(driveLetterAndColon))
{
// Make sure we aren't using this driveLetter for another mapping
WNetCancelConnection2(driveLetterAndColon, 0, true);
}
NETRESOURCE nr = new NETRESOURCE();
nr.dwType = ResourceType.RESOURCETYPE_DISK;
nr.lpRemoteName = shareName;
nr.lpLocalName = driveLetterAndColon;
int result = WNetAddConnection2(nr, password, username, 0);
if (result != 0)
{
throw new Exception("WNetAddConnection2 failed with error " + result);
}
}
}
public void CloudStorageAccountWithStorageUri()
{
StorageUri blobEndpoint = new StorageUri(
new Uri("http://" + AccountName + BlobService + EndpointSuffix),
new Uri("http://" + AccountName + SecondarySuffix + BlobService + EndpointSuffix));
StorageUri queueEndpoint = new StorageUri(
new Uri("http://" + AccountName + QueueService + EndpointSuffix),
new Uri("http://" + AccountName + SecondarySuffix + QueueService + EndpointSuffix));
StorageUri tableEndpoint = new StorageUri(
new Uri("http://" + AccountName + TableService + EndpointSuffix),
new Uri("http://" + AccountName + SecondarySuffix + TableService + EndpointSuffix));
StorageUri fileEndpoint = new StorageUri(
new Uri("http://" + AccountName + FileService + EndpointSuffix),
new Uri("http://" + AccountName + SecondarySuffix + FileService + EndpointSuffix));
#if WINDOWS_RT || ASPNET_K
CloudStorageAccount account = CloudStorageAccount.Create(new StorageCredentials(), blobEndpoint, queueEndpoint, tableEndpoint, fileEndpoint);
#else
CloudStorageAccount account = new CloudStorageAccount(new StorageCredentials(), blobEndpoint, queueEndpoint, tableEndpoint, fileEndpoint);
#endif
Assert.IsTrue(blobEndpoint.Equals(account.BlobStorageUri));
Assert.IsTrue(queueEndpoint.Equals(account.QueueStorageUri));
Assert.IsTrue(tableEndpoint.Equals(account.TableStorageUri));
Assert.IsTrue(fileEndpoint.Equals(account.FileStorageUri));
account = new CloudStorageAccount(new StorageCredentials(AccountName, TestBase.StorageCredentials.ExportBase64EncodedKey()), false);
Assert.IsTrue(blobEndpoint.Equals(account.BlobStorageUri));
Assert.IsTrue(queueEndpoint.Equals(account.QueueStorageUri));
Assert.IsTrue(tableEndpoint.Equals(account.TableStorageUri));
Assert.IsTrue(fileEndpoint.Equals(account.FileStorageUri));
account = CloudStorageAccount.Parse(string.Format("DefaultEndpointsProtocol=http;AccountName={0};AccountKey=", AccountName));
Assert.IsTrue(blobEndpoint.Equals(account.BlobStorageUri));
Assert.IsTrue(queueEndpoint.Equals(account.QueueStorageUri));
Assert.IsTrue(tableEndpoint.Equals(account.TableStorageUri));
Assert.IsTrue(fileEndpoint.Equals(account.FileStorageUri));
Assert.IsTrue(blobEndpoint.Equals(account.CreateCloudBlobClient().StorageUri));
Assert.IsTrue(queueEndpoint.Equals(account.CreateCloudQueueClient().StorageUri));
Assert.IsTrue(tableEndpoint.Equals(account.CreateCloudTableClient().StorageUri));
Assert.IsTrue(fileEndpoint.Equals(account.CreateCloudFileClient().StorageUri));
Assert.IsTrue(blobEndpoint.PrimaryUri.Equals(account.BlobEndpoint));
Assert.IsTrue(queueEndpoint.PrimaryUri.Equals(account.QueueEndpoint));
Assert.IsTrue(tableEndpoint.PrimaryUri.Equals(account.TableEndpoint));
Assert.IsTrue(fileEndpoint.PrimaryUri.Equals(account.FileEndpoint));
}
private static async Task mainAsync(string[] args)
{
try
{
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true);
IConfigurationRoot configuration = builder.Build();
var allCitDebtsLines = new string[10000000];
var allSettlementsLines = new string[10000000];
var allPaymentLines = new string[10000000];
while (true)
{
if (DateTime.Now.AddHours(3).ToShortTimeString() == "12:00 PM")
{
List <string> fileNames = new List <string>();
List <string[]> lstCloudFilesdata = new List <string[]>();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse($"{configuration["ConnectionString1"]}");
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare fileShare = fileClient.GetShareReference("import");
//looks for a file share in the cloud
bool fileShareExists = await fileShare.ExistsAsync();
if (fileShareExists)
{
List <CloudFile> lstCloudFiles = new List <CloudFile>();
CloudFileDirectory rootDir = fileShare.GetRootDirectoryReference();
//for each file in my fileshare
FileContinuationToken token = null;
FileResultSegment k = await rootDir.ListFilesAndDirectoriesSegmentedAsync(token);
token = k.ContinuationToken;
//var context_ = new Db.Data.();
List <string> sl = new List <string>();
foreach (IListFileItem fiile in k.Results)
{
//if the file exists
CloudFile file = (CloudFile)fiile;
bool asd = await file.ExistsAsync();
if (asd)
{
//adds new datasting array
sl = await ReadDataAsync(lstCloudFilesdata, file, fileNames);
}
foreach (string y in sl)
{
Console.WriteLine("From list new : " + y);
}
;
IEnumerable <CitizenDepts> o = from eachLine in (
from inner in sl
select inner.Split(';')
)
select new CitizenDepts
{
VAT = eachLine[0],
FirstName = eachLine[1],
LastName = eachLine[2],
Email = eachLine[3],
Phone = eachLine[4],
Address = eachLine[5],
County = eachLine[6],
BillId = eachLine[7],
Bill_description = eachLine[8],
Amount = Decimal.Parse(eachLine[9]),
DueDate = DateTime.ParseExact(eachLine[10],
"yyyyMMdd", CultureInfo.InvariantCulture)
};
foreach (var p in o)
{
Console.WriteLine(p.FirstName + " - " + p.BillId + ", - " + p.DueDate);
}
;
//string s = context_.Database.ProviderName;
// Console.WriteLine(s);
/// var all = from c in context_.CitizenDepts select c;
//context_.CitizenDepts.RemoveRange(all);
/// context_.SaveChanges();
foreach (var p in o)
{
//Add Student object into Students DBset
//if (p.VAT!=null)
//context_.Add(p);
}
//// call SaveChanges method to save student into database
//context_.SaveChanges();
}
}
if (lstCloudFilesdata != null && fileNames != null)
{
ProccessData(lstCloudFilesdata, fileNames);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
finally
{
}
}
public void CloudStorageAccountClientUriVerify()
{
StorageCredentials cred = new StorageCredentials(TestBase.TargetTenantConfig.AccountName, TestBase.TargetTenantConfig.AccountKey);
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(cred, true);
CloudBlobClient blobClient = cloudStorageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("container1");
Assert.AreEqual(cloudStorageAccount.BlobEndpoint.ToString() + "container1", container.Uri.ToString());
CloudQueueClient queueClient = cloudStorageAccount.CreateCloudQueueClient();
CloudQueue queue = queueClient.GetQueueReference("queue1");
Assert.AreEqual(cloudStorageAccount.QueueEndpoint.ToString() + "queue1", queue.Uri.ToString());
CloudTableClient tableClient = cloudStorageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("table1");
Assert.AreEqual(cloudStorageAccount.TableEndpoint.ToString() + "table1", table.Uri.ToString());
CloudFileClient fileClient = cloudStorageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("share1");
Assert.AreEqual(cloudStorageAccount.FileEndpoint.ToString() + "share1", share.Uri.ToString());
}
async Task <bool> ExecuteLoadFileSharesAsync(bool force = false)
{
if (IsBusy)
{
return(false);
}
var realm = App.GetRealm();
try
{
IsBusy = true;
NoFileSharesFound = false;
var realmFileShares = realm.All <RealmCloudFileShare>();
if (realmFileShares.Count() > 0 && force == false)
{
var storageAccounts = realm.All <StorageAccountExt>().Where(sa => sa.IsStorageAccountOn);
List <ASECloudFileShare> aseFileShares = new List <ASECloudFileShare>();
if (storageAccounts.Count() > 0)
{
foreach (var fShare in realmFileShares)
{
StorageAccountsExist = true;
var storageAccount = storageAccounts.Where((arg) => arg.Name == fShare.StorageAccountName).FirstOrDefault();
if (storageAccount != null)
{
var te = new CloudFileShare(new Uri(fShare.FileShareUri),
new Microsoft.WindowsAzure.Storage.Auth.StorageCredentials(storageAccount.Name, storageAccount.PrimaryKey));
aseFileShares.Add(new ASECloudFileShare(te, storageAccount.Name));
}
}
FileShares.Clear();
FileShares.AddRange(aseFileShares);
}
}
else
{
var storageAccounts = realm.All <StorageAccountExt>().Where(sa => sa.IsStorageAccountOn);
FileShares.Clear();
foreach (var account in storageAccounts)
{
string connectionString = Constants.StorageConnectionString;
connectionString = connectionString.Replace("<ACCOUNTNAME>", account.Name);
connectionString = connectionString.Replace("<ACCOUNTKEY>", account.PrimaryKey);
CloudStorageAccount sa = CloudStorageAccount.Parse(connectionString);
var fileClient = sa.CreateCloudFileClient();
var fileShares = await fileClient.ListFileSharesAsync();
List <ASECloudFileShare> aseFileShares = new List <ASECloudFileShare>();
for (int i = 0; i < fileShares.Count; i++)
{
aseFileShares.Add(new ASECloudFileShare(fileShares[i]));
}
aseFileShares.All(c => { c.StorageAccountName = account.Name; return(true); });
FileShares.AddRange(aseFileShares);
}
if (storageAccounts.Count() > 0)
{
StorageAccountsExist = true;
}
else
{
StorageAccountsExist = false;
}
await realm.WriteAsync(temprealm =>
{
temprealm.RemoveAll <RealmCloudFileShare>();
foreach (var fShare in FileShares)
{
temprealm.Add(new RealmCloudFileShare(fShare.FileShareName, fShare.StorageAccountName, fShare.BaseFileShare.Uri.ToString()));
}
});
realm.All <RealmCloudFileShare>().SubscribeForNotifications((sender, changes, error) =>
{
Console.WriteLine("Change to CloudFileShares");
});
}
SortFileShares();
if (FileShares.Count == 0)
{
NoFileSharesFound = true;
}
else
{
NoFileSharesFound = false;
}
}
catch (Exception ex)
{
Logger.Report(ex, "Method", "ExecuteLoadFileSharesAsync");
MessagingService.Current.SendMessage(MessageKeys.Error, ex);
}
finally
{
IsBusy = false;
}
return(true);
}
