private static List<CloudFile> GetAll(ICloudProvider provider) { List<CloudFile> cloudFiles = null; var cloudItems = provider.Dir(); if (cloudItems != null && cloudItems.Count > 0) { cloudFiles = new List<CloudFile>(); foreach (var cloudItem in cloudItems) { if (cloudItem is CloudFolder) { var allChildren = (cloudItem as CloudFolder).GetAllFileChildren(); if (allChildren != null) { cloudFiles.AddRange(allChildren); } } else { cloudFiles.Add(cloudItem as CloudFile); } } } return cloudFiles; }
/// <summary> /// Create a CachedDirectory /// </summary> /// <param name="CloudProvider">the implimentation for interfacing with the cloud</param> /// <param name="Catalog">name of catalog (folder in blob storage)</param> /// <param name="CacheDirectory">local Directory object to use for local cache</param> public CachedDirectory( ICloudProvider CloudProvider, string Catalog = null, Directory CacheDirectory = null ) { if ( CloudProvider == null ) { throw new ArgumentNullException( "cloudProvider" ); } this.cloudProvider = CloudProvider; string catalog = string.IsNullOrEmpty( Catalog ) ? "lucene" : Catalog.ToLower(); if ( CacheDirectory != null ) { // save it off this.CacheDirectory = CacheDirectory; } else { string cachePath = Path.Combine( Environment.ExpandEnvironmentVariables( "%temp%" ), "LuceneCache" ); DirectoryInfo cacheDir = new DirectoryInfo( cachePath ); if ( !cacheDir.Exists ) { cacheDir.Create(); } string catalogPath = Path.Combine( cachePath, catalog ); DirectoryInfo catalogDir = new DirectoryInfo( catalogPath ); if ( !catalogDir.Exists ) { catalogDir.Create(); } this.CacheDirectory = FSDirectory.Open( catalogPath ); } this.cloudProvider.InitializeStorage(); }
public static List<ChangeItem> Compare(ICloudProvider localProvider, ICloudProvider cloudProvider, PipelineDirections direction) { var pares = GetPares(localProvider, cloudProvider); List<ChangeItem> changeset = new List<ChangeItem>(); foreach (var pare in pares) { var compare = pare.Compare; if (compare < 0 && direction != PipelineDirections.Backup) { var changeItem = new UpdateItem(); changeItem.Item = pare.CloudFile; changeItem.Target = localProvider; changeItem.Source = cloudProvider; changeset.Add(changeItem); } if (compare > 0 && direction != PipelineDirections.Restore) { var changeItem = new UpdateItem(); changeItem.Item = pare.LocalFile; changeItem.Target = cloudProvider; changeItem.Source = localProvider; changeset.Add(changeItem); } } return changeset; }
public Pipeline(ICloudProvider localProvider, ICloudProvider cloudProvider, PipelineDirections direction, Coordinator coordinator) { this.localProvider = localProvider; this.cloudProvider = cloudProvider; this.direction = direction; this.coordinator = coordinator; }
public string DeleteInfrastructure(CloudEnvironments infrastructureName) { ICloudProvider cloudProvider = _cloudProviderFactory.GetAbstractCloudProvider(Provider); var cloudManager = cloudProvider.GetCloudEnvironment(infrastructureName); var deleted = cloudManager.DeleteInfrastructure(); return(deleted ? $"{infrastructureName} deleted successfully" : $"{infrastructureName} deletion failed"); }
public CloudFileBaseService(string baseRoot, ICloudProvider provider) { _providerPath = CreateDirectory(provider.Name, baseRoot); _jsonSerializerSettings = new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.All }; }
/// <summary> /// Initializes a new instance of the <see cref="Startup"/> class. /// </summary> /// <param name="configuration">App configuration</param> /// <param name="env">Hosting environment</param> /// <param name="logger">Logger used for tracing</param> public Startup(IConfiguration configuration, IHostingEnvironment env, ILogger <Startup> logger) { var builder = new ConfigurationBuilder() .AddJsonFile("appsettings.json", optional: false) .AddJsonFile("secrets.json", optional: true) .AddJsonFile("/flexvol/secrets.json", optional: true) .AddEnvironmentVariables(); this.Configuration = builder.Build(); this.logger = logger; CloudLogger.SetLogger(logger); this.cloudProvider = new CloudFactory(this.Configuration).Provider(); this.currentEnvironment = env; }
public string CreateDatabaseServer(DatabaseServer databaseServer, CloudEnvironments environment) { var isValid = ValidateDatabaseServer(databaseServer); if (!string.IsNullOrEmpty(isValid)) { return(isValid); } ICloudProvider cloudProvider = _cloudProviderFactory.GetAbstractCloudProvider(Provider); var cloudManager = cloudProvider.GetCloudEnvironment(environment); var created = cloudManager.CreateDatabaseServer(databaseServer.Type); return(created ? $"{environment}-{databaseServer.Type} Database server created successfully" : "Database server creation failed"); }
public void RunIndexOperations( ICloudProvider CloudProvider ) { try { // default CachedDirectory stores cache in local temp folder using ( CachedDirectory cachedDirectory = new CachedDirectory( CloudProvider ) ) { bool findexExists = IndexReader.IndexExists( cachedDirectory ); using ( StandardAnalyzer analyzer = new StandardAnalyzer( Version.LUCENE_CURRENT ) ) { using ( IndexWriter indexWriter = GetIndexWriter( cachedDirectory, analyzer ) ) { indexWriter.SetRAMBufferSizeMB( 10.0 ); //indexWriter.SetUseCompoundFile(false); //indexWriter.SetMaxMergeDocs(10000); //indexWriter.SetMergeFactor(100); Console.WriteLine( "Total existing docs is {0}", indexWriter.NumDocs() ); Console.WriteLine( "Creating docs:" ); int maxDocs = 10000; for ( int iDoc = 0; iDoc < maxDocs; iDoc++ ) { if ( iDoc % 1000 == 0 ) { Console.WriteLine( "- " + iDoc + "/" + maxDocs ); } Document doc = new Document(); doc.Add( new Field( "id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) ); doc.Add( new Field( "Title", GeneratePhrase( 10 ), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) ); doc.Add( new Field( "Body", GeneratePhrase( 40 ), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) ); indexWriter.AddDocument( doc ); } Console.WriteLine( "Total docs is {0}", indexWriter.NumDocs() ); } } using ( new AutoStopWatch( "Creating searcher" ) ) { using ( IndexSearcher searcher = new IndexSearcher( cachedDirectory ) ) { SearchForPhrase( searcher, "dog" ); SearchForPhrase( searcher, _random.Next( 32768 ).ToString() ); SearchForPhrase( searcher, _random.Next( 32768 ).ToString() ); } } } } catch ( Exception ex ) { Console.WriteLine( "Error: " + ex.Message ); Console.WriteLine( ex.StackTrace ); } Console.WriteLine( "Push return to exit" ); Console.Read(); }
public void RunIndexOperations(ICloudProvider CloudProvider) { try { // default CachedDirectory stores cache in local temp folder using (CachedDirectory cachedDirectory = new CachedDirectory(CloudProvider)) { bool findexExists = IndexReader.IndexExists(cachedDirectory); using (StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT)) { using (IndexWriter indexWriter = GetIndexWriter(cachedDirectory, analyzer)) { indexWriter.SetRAMBufferSizeMB(10.0); //indexWriter.SetUseCompoundFile(false); //indexWriter.SetMaxMergeDocs(10000); //indexWriter.SetMergeFactor(100); Console.WriteLine("Total existing docs is {0}", indexWriter.NumDocs()); Console.WriteLine("Creating docs:"); int maxDocs = 10000; for (int iDoc = 0; iDoc < maxDocs; iDoc++) { if (iDoc % 1000 == 0) { Console.WriteLine("- " + iDoc + "/" + maxDocs); } Document doc = new Document(); doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); indexWriter.AddDocument(doc); } Console.WriteLine("Total docs is {0}", indexWriter.NumDocs()); } } using (new AutoStopWatch("Creating searcher")) { using (IndexSearcher searcher = new IndexSearcher(cachedDirectory)) { SearchForPhrase(searcher, "dog"); SearchForPhrase(searcher, _random.Next(32768).ToString()); SearchForPhrase(searcher, _random.Next(32768).ToString()); } } } } catch (Exception ex) { Console.WriteLine("Error: " + ex.Message); Console.WriteLine(ex.StackTrace); } Console.WriteLine("Push return to exit"); Console.Read(); }
public CachedIndexOutput( ICloudProvider CloudProvider, Directory CacheDirectory, string Name ) { this.cloudProvider = CloudProvider; this.cacheDirectory = CacheDirectory; this.name = Name; this.fileMutex = BlobMutexManager.GrabMutex( this.name ); this.fileMutex.WaitOne(); try { // create the local cache one we will operate against... this.indexOutput = this.cacheDirectory.CreateOutput( this.name ); } finally { this.fileMutex.ReleaseMutex(); } }
public string CreateVirtualMachine(VirtualMachine virtualMachine, CloudEnvironments environment) { var isValid = ValidateVirtualMachine(virtualMachine); if (!string.IsNullOrEmpty(isValid)) { return(isValid); } ICloudProvider cloudProvider = _cloudProviderFactory.GetAbstractCloudProvider(Provider); var cloudManager = cloudProvider.GetCloudEnvironment(environment); var created = cloudManager.CreateVirtualMachine(virtualMachine.Type); return(created ? $"{environment}-{virtualMachine.Type} Virtual machine created successfully" : "Virtual machine creation failed"); }
public CachedIndexOutput(ICloudProvider CloudProvider, Directory CacheDirectory, string Name) { this.cloudProvider = CloudProvider; this.cacheDirectory = CacheDirectory; this.name = Name; this.fileMutex = BlobMutexManager.GrabMutex(this.name); this.fileMutex.WaitOne(); try { // create the local cache one we will operate against... this.indexOutput = this.cacheDirectory.CreateOutput(this.name); } finally { this.fileMutex.ReleaseMutex(); } }
/// <summary> /// Registers a provider for a particular drive and user account. Reregistration of provider throws exception of type ProviderRegistrationException. /// </summary> /// <param name="newProvider"></param> private static void RegisterProvider(ICloudProvider newProvider) { if (!DriveProviderCollection.ContainsKey(newProvider.Drive)) { DriveProviderCollection.Add(newProvider.Drive, new List <ICloudProvider>()); } List <ICloudProvider> providers = DriveProviderCollection[newProvider.Drive]; if (providers.Find(provider => provider.UserName == newProvider.UserName) != null) { string message = string.Format( "The provider with username {0} is already registered with {1}. Please register with a different user id. Multiple accounts from same cloud provider is supported.", newProvider.UserName, newProvider.Drive); throw new ProviderRegistrationException(message); } providers.Add(newProvider); }
/// <summary> /// Authenticates the given drive. If one account is already authenticated then the method forces authentication of the drive on a different account. This also registers a provider for the supported drive. /// </summary> /// <param name="drive"></param> public static async Task <AuthenticationResult> AuthenticateAsync(SupportedDrive drive) { CloudAuthenticator authenticator = CloudAuthenticator.GetAuthenticator(drive); AuthenticationResult result = await authenticator.AuthenticateAsync(); if (result == AuthenticationResult.Success) { ICloudProvider provider = null; switch (drive) { case SupportedDrive.SkyDrive: provider = new SkyDriveProvider((LiveConnectSession)authenticator.GetSession()); break; //case SupportedDrive.DropBox: // provider = new SkyDriveProvider((LiveConnectSession) authenticator.GetSession()); // break; } RegisterProvider(provider); } return(result); }
public void OnApplicationStarted() { string archiveFileAndPath = Path.Combine(ArchivePath, ArchiveFile); this.logger.Debug("PlaynitePersistPlugin initialization!"); this.testProvider = new GoogleDriveProvider(this.logger); this.testProvider.Connect(); this.logger.Debug("Attempting to download archive."); this.testProvider.SyncFrom(ArchiveFile, archiveFileAndPath); this.logger.Debug("Games data archive downloaded from cloud provider."); this.logger.Debug("Attempting to extract games data from archive."); Archive.ExtractGamesArchive(archiveFileAndPath); Archive.DeleteGamesArchive(archiveFileAndPath); this.logger.Debug("Extracted games data from archive."); this.logger.Debug("Updating games from persist plugin..."); this.updateGames(); this.api.Notifications.Add(PersistPluginNotificationId, "Synced games data from Google Drive.", NotificationType.Info); }
public static List<Pare> GetPares(ICloudProvider localHDProvider, ICloudProvider cloudProvider) { var pares = new Dictionary<string, Pare>(); //load local files var cloudFiles = GetAll(localHDProvider); if (cloudFiles != null) { foreach (var cloudFile in cloudFiles) { var pare = new Pare(); pare.LocalFile = cloudFile; pares[cloudFile.FullName()] = pare; } } cloudFiles = GetAll(cloudProvider); if (cloudFiles != null) { foreach (var cloudFile in cloudFiles) { var key = cloudFile.FullName(); Pare pare = null; if (pares.ContainsKey(key)) { pare = pares[key]; } else { pare = new Pare(); pares[key] = pare; } pare.CloudFile = cloudFile; } } return pares.Values.ToList(); }
/// <summary> /// Create a CachedDirectory /// </summary> /// <param name="CloudProvider">the implimentation for interfacing with the cloud</param> /// <param name="Catalog">name of catalog (folder in blob storage)</param> /// <param name="CacheDirectory">local Directory object to use for local cache</param> public CachedDirectory(ICloudProvider CloudProvider, string Catalog = null, Directory CacheDirectory = null) { if (CloudProvider == null) { throw new ArgumentNullException("cloudProvider"); } this.cloudProvider = CloudProvider; string catalog = string.IsNullOrEmpty(Catalog) ? "lucene" : Catalog.ToLower(); if (CacheDirectory != null) { // save it off this.CacheDirectory = CacheDirectory; } else { string cachePath = Path.Combine(Environment.ExpandEnvironmentVariables("%temp%"), "LuceneCache"); DirectoryInfo cacheDir = new DirectoryInfo(cachePath); if (!cacheDir.Exists) { cacheDir.Create(); } string catalogPath = Path.Combine(cachePath, catalog); DirectoryInfo catalogDir = new DirectoryInfo(catalogPath); if (!catalogDir.Exists) { catalogDir.Create(); } this.CacheDirectory = FSDirectory.Open(catalogPath); } this.cloudProvider.InitializeStorage(); }
/// <summary> /// Uploads a file to cloud provider /// </summary> /// <param name="filename">file name</param> /// <param name="content">file content</param> /// <param name="cloudProvider">cloud provider</param> private void UploadFile(string filename, string content, ICloudProvider cloudProvider) { cloudProvider.StorageProvider().UploadFile(filename, new MemoryStream(Encoding.ASCII.GetBytes(content))); }
public CachedIndexInput( ICloudProvider CloudProvider, Directory CacheDirectory, string Name ) { this.name = Name; #if FULLDEBUG Debug.WriteLine( "Opening " + this.name ); #endif this.fileMutex = BlobMutexManager.GrabMutex( this.name ); this.fileMutex.WaitOne(); try { bool fFileNeeded = false; FileMetadata cloudMetadata = CloudProvider.FileMetadata( this.name ); if ( !cloudMetadata.Exists ) { fFileNeeded = false; // TODO: Delete local if it doesn't exist on cloud? /* if (CacheDirectory.FileExists(this.name)) { CacheDirectory.DeleteFile(this.name); } */ } else if ( !CacheDirectory.FileExists( this.name ) ) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength( this.name ); long blobLength = cloudMetadata.Length; DateTime blobLastModifiedUTC = cloudMetadata.LastModified.ToUniversalTime(); if ( !cloudMetadata.Exists || cachedLength != blobLength ) { fFileNeeded = true; } else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length DateTime cachedLastModifiedUTC = new DateTime( CacheDirectory.FileModified( this.name ), DateTimeKind.Local ).ToUniversalTime(); if ( cachedLastModifiedUTC < blobLastModifiedUTC ) { TimeSpan timeSpan = blobLastModifiedUTC.Subtract( cachedLastModifiedUTC ); if ( timeSpan.TotalSeconds > 1 ) { fFileNeeded = true; } else { #if FULLDEBUG Debug.WriteLine( "Using cache for " + this.name + ": " + timeSpan.TotalSeconds ); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if ( fFileNeeded ) { using ( StreamOutput fileStream = new StreamOutput( CacheDirectory.CreateOutput( this.name ) ) ) { Stream blobStream = CloudProvider.Download( this.name ); blobStream.CopyTo( fileStream ); fileStream.Flush(); Debug.WriteLine( "GET {0} RETREIVED {1} bytes", this.name, fileStream.Length ); } } else { #if FULLDEBUG if ( !cloudMetadata.Exists ) { Debug.WriteLine( "Cloud doesn't have " + this.name ); } else { Debug.WriteLine( "Using cached file for " + this.name ); } #endif } // open the file in read only mode this.indexInput = CacheDirectory.OpenInput( this.name ); } finally { this.fileMutex.ReleaseMutex(); } }
/// <summary> /// Initializes a new instance of the <see cref="DownloadController"/> class. /// </summary> /// <param name="cloudProvider">Cloud specific uploader</param> public DownloadController(ICloudProvider cloudProvider) { this.cloudProvider = cloudProvider; }
public CachedLock(ICloudProvider CloudProvider, string LockFile) { this.cloudProvider = CloudProvider; this.lockFile = LockFile; }
/// <summary> /// Initializes a new instance of the <see cref="QueueWatcher"/> class. /// </summary> /// <param name="cloudProvider">cloud provider</param> /// <param name="sleepTime">Time between polls</param> public QueueWatcher(ICloudProvider cloudProvider, int sleepTime) { this.cloud = cloudProvider; this.sleepTime = sleepTime; }
/// <summary> /// Initializes a new instance of the <see cref="NotificationHandler"/> class. /// </summary> /// <param name="cloudProvider">cloud provider</param> public NotificationHandler(ICloudProvider cloudProvider) { this.cloud = cloudProvider; }
public CachedIndexInput(ICloudProvider CloudProvider, Directory CacheDirectory, string Name) { this.name = Name; #if FULLDEBUG Debug.WriteLine("Opening " + this.name); #endif this.fileMutex = BlobMutexManager.GrabMutex(this.name); this.fileMutex.WaitOne(); try { bool fFileNeeded = false; FileMetadata cloudMetadata = CloudProvider.FileMetadata(this.name); if (!cloudMetadata.Exists) { fFileNeeded = false; // TODO: Delete local if it doesn't exist on cloud? /* * if (CacheDirectory.FileExists(this.name)) { * CacheDirectory.DeleteFile(this.name); * } */ } else if (!CacheDirectory.FileExists(this.name)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(this.name); long blobLength = cloudMetadata.Length; DateTime blobLastModifiedUTC = cloudMetadata.LastModified.ToUniversalTime(); if (!cloudMetadata.Exists || cachedLength != blobLength) { fFileNeeded = true; } else { // there seems to be an error of 1 tick which happens every once in a while // for now we will say that if they are within 1 tick of each other and same length DateTime cachedLastModifiedUTC = new DateTime(CacheDirectory.FileModified(this.name), DateTimeKind.Local).ToUniversalTime(); if (cachedLastModifiedUTC < blobLastModifiedUTC) { TimeSpan timeSpan = blobLastModifiedUTC.Subtract(cachedLastModifiedUTC); if (timeSpan.TotalSeconds > 1) { fFileNeeded = true; } else { #if FULLDEBUG Debug.WriteLine("Using cache for " + this.name + ": " + timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { using (StreamOutput fileStream = new StreamOutput(CacheDirectory.CreateOutput(this.name))) { Stream blobStream = CloudProvider.Download(this.name); blobStream.CopyTo(fileStream); fileStream.Flush(); Debug.WriteLine("GET {0} RETREIVED {1} bytes", this.name, fileStream.Length); } } else { #if FULLDEBUG if (!cloudMetadata.Exists) { Debug.WriteLine("Cloud doesn't have " + this.name); } else { Debug.WriteLine("Using cached file for " + this.name); } #endif } // open the file in read only mode this.indexInput = CacheDirectory.OpenInput(this.name); } finally { this.fileMutex.ReleaseMutex(); } }
public CachedLock( ICloudProvider CloudProvider, string LockFile ) { this.cloudProvider = CloudProvider; this.lockFile = LockFile; }
/// <summary> /// Initializes a new instance of the <see cref="UploadController"/> class. /// </summary> /// <param name="cloudProvider">Cloud specific uploader</param> /// <param name="logger">Application Insights logger</param> public UploadController(ICloudProvider cloudProvider, ILogger <UploadController> logger) { this.cloudProvider = cloudProvider; this.logger = logger; }
public SevenZipProvider(ICloudProvider provider, string password = null) { this.provider = provider; this.password = password; }
public void Init() { this.cloudProvider = new MockCloudProvider(true); }