예제 #1
0
        /// <summary>
        /// Ensure that the current item is pushed onto the stack.
        /// </summary>
        public void EnsureStack()
        {
            switch (_storage.Location)
            {
            case ItemLocation.Stack:
                // Already on the stack
                return;

            case ItemLocation.Parameter:
            case ItemLocation.Local:
            case ItemLocation.Current:
                PushValue();
                break;

            case ItemLocation.Global:
                // Call method that computes the value of this global value
                _helper.LoadQueryRuntime();
                _helper.Call(_storage.GlobalLocation !);
                break;

            default:
                Debug.Fail($"Invalid location: {_storage.Location}");
                break;
            }

            _storage = _storage.ToStack();
        }
        private DataConnection CreateSourceDataConnection(StorageDescriptor storageDescriptor)
        {
            var connectionString = storageDescriptor.Tenant.HasValue
                ? _connectionStringSettings.GetConnectionString(
                storageDescriptor.ConnectionStringIdentity, storageDescriptor.Tenant.Value)
                : _connectionStringSettings.GetConnectionString(
                storageDescriptor.ConnectionStringIdentity);

            if (string.IsNullOrWhiteSpace(connectionString))
            {
                throw new Exception($"No connection string for {storageDescriptor.ConnectionStringIdentity.GetType().Name} for tenant {storageDescriptor.Tenant} is defined");
            }

            using var scope = new TransactionScope(TransactionScopeOption.Suppress);

            // Creating connection to source that will NOT be enlisted in transactions
            var connection = CreateDataConnection(
                connectionString,
                storageDescriptor.MappingSchema,
                storageDescriptor.CommandTimeout);

            if (connection.Connection.State != ConnectionState.Open)
            {
                connection.Connection.Open();
            }

            scope.Complete();

            return(connection);
        }
예제 #3
0
        public static TableEntity ToData(CdmEntityDefinition instance, CdmCorpusContext ctx, ResolveOptions resOpt, CopyOptions options)
        {
            var properties = CreateTablePropertyBags(instance, resOpt, options);
            var columns    = new List <DataColumn>();

            foreach (var attribute in instance.Attributes)
            {
                columns.Add(TypeAttributePersistence.ToData(attribute as CdmTypeAttributeDefinition, ctx, resOpt, options));
            }

            var storageDescriptor = new StorageDescriptor
            {
                Source  = new DataSource(),
                Format  = new FormatInfo(),
                Columns = columns
            };

            TableProperties teProperties = new TableProperties
            {
                Properties        = properties,
                Partitioning      = new TablePartitioning(),
                StorageDescriptor = storageDescriptor
            };

            return(new TableEntity
            {
                Name = instance.EntityName,
                Type = SASEntityType.TABLE,
                Properties = teProperties
            });
        }
        private void ReplaceInBulk(IReadOnlyCollection <Type> dataObjectTypes, StorageDescriptor sourceStorageDescriptor, DataConnection targetConnection, IReadOnlyCollection <ICommand> replicationCommands)
        {
            DataConnection sourceConnection;

            // Creating connection to source that will NOT be enlisted in transactions
            using (var scope = new TransactionScope(TransactionScopeOption.Suppress))
            {
                sourceConnection = CreateDataConnection(sourceStorageDescriptor);
                if (sourceConnection.Connection.State != ConnectionState.Open)
                {
                    sourceConnection.Connection.Open();
                }

                scope.Complete();
            }

            using (sourceConnection)
            {
                var actorsFactory = new ReplaceDataObjectsInBulkActorFactory(dataObjectTypes, sourceConnection, targetConnection);
                var actors        = actorsFactory.Create();

                foreach (var actor in actors)
                {
                    var sw = Stopwatch.StartNew();
                    actor.ExecuteCommands(replicationCommands);
                    sw.Stop();

                    Console.WriteLine($"[{DateTime.Now}] [{Environment.CurrentManagedThreadId}] {actor.GetType().GetFriendlyName()}: {sw.Elapsed.TotalSeconds:F3} seconds");
                }
            }
        }
예제 #5
0
 /// <summary>
 /// Discard the current item if it is pushed onto the stack.
 /// </summary>
 public void DiscardStack()
 {
     if (_storage.Location == ItemLocation.Stack)
     {
         _helper.Emit(OpCodes.Pop);
         _storage = StorageDescriptor.None();
     }
 }
예제 #6
0
 private static bool IsConfigured(StorageDescriptor descriptor) =>
 !string.IsNullOrWhiteSpace(
     descriptor.Tenant.HasValue
             ? ConnectionStringSettings.GetConnectionString(
         descriptor.ConnectionStringIdentity,
         descriptor.Tenant.Value)
             : ConnectionStringSettings.GetConnectionString(
         descriptor.ConnectionStringIdentity));
 /// <summary>
 /// В databaseManagementMode исключен updatestatistics - причина, т.к. будет выполнен rebuild индексов, то
 /// статистика для индексов при этом будет автоматически пересчитана с FULLSCAN, нет смысла после этого делать updatestatistics
 /// с меньшим SampleRate потенциально ухудшая качество статистики
 /// </summary>
 private static ReplicateInBulkCommand ReplicateFromDbToDbCommand(
     IReadOnlyCollection <Type> typesToReplicate, StorageDescriptor from, StorageDescriptor to) =>
 new ReplicateInBulkCommand(
     typesToReplicate,
     from,
     to,
     executionMode: ParallelReplication,
     databaseManagementMode: DbManagementMode.DropAndRecreateConstraints |
     DbManagementMode.EnableIndexManagment);
        private DataConnection CreateDataConnection(StorageDescriptor storageDescriptor)
        {
            var connectionString = _connectionStringSettings.GetConnectionString(storageDescriptor.ConnectionStringIdentity);
            var connection       = SqlServerTools.CreateDataConnection(connectionString);

            connection.AddMappingSchema(storageDescriptor.MappingSchema);
            connection.CommandTimeout = (int)storageDescriptor.CommandTimeout.TotalMilliseconds;
            return(connection);
        }
예제 #9
0
 /// <summary>
 /// Ensure that current item is saved to the specified local variable.
 /// </summary>
 public void EnsureLocal(LocalBuilder bldr)
 {
     if (_storage.LocalLocation != bldr)
     {
         // Push value onto stack and then save to bldr
         EnsureStack();
         _helper.Emit(OpCodes.Stloc, bldr);
         _storage = _storage.ToLocal(bldr);
     }
 }
예제 #10
0
        /// <summary>
        /// Set this iterator to be the same as the specified iterator.
        /// </summary>
        public void SetIterator(IteratorDescriptor iterInfo)
        {
            if (iterInfo.HasLabelNext)
            {
                _lblNext = iterInfo.GetLabelNext();
                _hasNext = true;
            }

            _storage = iterInfo.Storage;
        }
예제 #11
0
        /// <summary>
        /// Continue iteration until it is complete.  Branch to "lblOnEnd" when iteration is complete.
        /// </summary>
        /// <remarks>
        /// goto LabelNextCtxt;
        /// LabelOnEnd:
        /// </remarks>
        public void LoopToEnd(Label lblOnEnd)
        {
            if (_hasNext)
            {
                _helper.BranchAndMark(_lblNext, lblOnEnd);
                _hasNext = false;
            }

            // After looping is finished, storage is N/A
            _storage = StorageDescriptor.None();
        }
예제 #12
0
        private static bool CreateStorageDescriptor(string value, out object result)
        {
            string dir = Path.GetDirectoryName(value);

            if (string.IsNullOrEmpty(dir))
            {
                dir = Application.WorkerPath;
            }

            result = new StorageDescriptor(new PathDescriptor(dir), Path.GetFileName(value));
            return(true);
        }
예제 #13
0
        public override void Execute()
        {
            StorageDescriptor pluginCache = new StorageDescriptor(Application.CacheDirectory, PluginCacheFileName, path.Name);

            if (pluginCache.Load())
            {
                foreach (StorageContainer node in pluginCache.Nodes.Values)
                {
                    if (!node.Validate())
                    {
                        pluginCache.Clear();
                        break;
                    }
                }
            }
            StorageContainer assemblyNode; if (!pluginCache.TryGetNode(PluginAssemblyStorageAlias, out assemblyNode))

            {
                List <FileSystemDescriptor> files = Application.GetPluginFiles(path);
                if (files.Count > 0)
                {
                    FileDescriptor assemblyFile = Compile(files);
                    assemblyNode = pluginCache.Store(assemblyFile, PluginAssemblyStorageAlias);

                    foreach (FileDescriptor mixin in files)
                    {
                        pluginCache.Store(mixin);
                    }
                    pluginCache.Save();
                }
                else
                {
                    Application.Warning(SeverityFlags.None, "Plugin '{0}' has no code files to compile", path.Name);
                }
                files = null;
            }
            if (assemblyNode != null && assemblyNode.Element.Exists())
            {
                Assembly assembly = Assembly.LoadFrom(assemblyNode.Element.GetAbsolutePath());
                Type     type     = assembly.GetType <IPlugin>();
                if (type != null)
                {
                    instance = type.CreateInstance <IPlugin>();

                    if (instance != null)
                    {
                        instance.Load();
                        return;
                    }
                }
                Application.Warning(SeverityFlags.None, "'{0}' is not a plugin", path.GetAbsolutePath());
            }
        }
 public ReplicateInBulkCommand(
     StorageDescriptor sourceStorageDescriptor,
     StorageDescriptor targetStorageDescriptor,
     DbManagementMode databaseManagementMode = DbManagementMode.DropAndRecreateConstraints | DbManagementMode.EnableIndexManagment | DbManagementMode.UpdateTableStatistics,
     ExecutionMode executionMode             = null,
     TimeSpan?bulkCopyTimeout = null)
 {
     SourceStorageDescriptor = sourceStorageDescriptor;
     TargetStorageDescriptor = targetStorageDescriptor;
     DbManagementMode        = databaseManagementMode;
     ExecutionMode           = executionMode ?? ExecutionMode.Parallel;
     BulkCopyTimeout         = bulkCopyTimeout ?? DefaultBulkCopyTimeout;
 }
예제 #15
0
        /// <summary>
        /// If the iterator has been fully cached, then iterate the values one-by-one.
        /// </summary>
        public void EnsureNoCache()
        {
            if (_storage.IsCached)
            {
                if (!HasLabelNext)
                {
                    // If no Next label, this must be a singleton cache
                    EnsureStack();
                    _helper.LoadInteger(0);
                    _helper.CallCacheItem(_storage.ItemStorageType);

                    _storage = StorageDescriptor.Stack(_storage.ItemStorageType, false);
                }
                else
                {
                    // int idx;
                    LocalBuilder locIdx = _helper.DeclareLocal("$$$idx", typeof(int));
                    Label        lblNext;

                    // Make sure cache is not on the stack
                    EnsureNoStack("$$$cache");

                    // idx = -1;
                    _helper.LoadInteger(-1);
                    _helper.Emit(OpCodes.Stloc, locIdx);

                    // LabelNext:
                    lblNext = _helper.DefineLabel();
                    _helper.MarkLabel(lblNext);

                    // idx++;
                    _helper.Emit(OpCodes.Ldloc, locIdx);
                    _helper.LoadInteger(1);
                    _helper.Emit(OpCodes.Add);
                    _helper.Emit(OpCodes.Stloc, locIdx);

                    // if (idx >= cache.Count) goto LabelNextCtxt;
                    _helper.Emit(OpCodes.Ldloc, locIdx);
                    CacheCount();
                    _helper.Emit(OpCodes.Bge, GetLabelNext());

                    // item = cache[idx];
                    PushValue();
                    _helper.Emit(OpCodes.Ldloc, locIdx);
                    _helper.CallCacheItem(_storage.ItemStorageType);

                    SetIterator(lblNext, StorageDescriptor.Stack(_storage.ItemStorageType, false));
                }
            }
        }
 private void ExecuteInTransactionScope(StorageDescriptor storageDescriptor,
                                        Action <DataConnection, SequentialPipelineActor> action)
 {
     using (var transation = new TransactionScope(TransactionScopeOption.RequiresNew, TransactionOptions))
     {
         using (var targetConnection = CreateDataConnection(storageDescriptor))
         {
             var schemaManagenentActor =
                 CreateDbSchemaManagementActor((SqlConnection)targetConnection.Connection,
                                               storageDescriptor.CommandTimeout);
             action(targetConnection, schemaManagenentActor);
             transation.Complete();
         }
     }
 }
        private DataConnection CreateTransactionlessDataConnection(StorageDescriptor storageDescriptor)
        {
            // Creating connection to source that will NOT be enlisted in transactions
            using (var scope = new TransactionScope(TransactionScopeOption.Suppress))
            {
                var connection = CreateDataConnection(storageDescriptor);
                if (connection.Connection.State != ConnectionState.Open)
                {
                    connection.Connection.Open();
                }

                scope.Complete();

                return(connection);
            }
        }
예제 #18
0
        public static CdmDataPartitionDefinition FromData(CdmCorpusContext ctx, StorageDescriptor obj, string symsRootPath, FormatType formatType)
        {
            var newPartition = ctx.Corpus.MakeObject <CdmDataPartitionDefinition>(CdmObjectType.DataPartitionDef);

            var symsPath = Utils.CreateSymsAbsolutePath(symsRootPath, obj.Source.Location);

            newPartition.Location = Utils.SymsPathToCorpusPath(symsPath, ctx.Corpus.Storage);

            var trait = Utils.CreatePartitionTrait(obj.Format.Properties, ctx, formatType);

            if (trait != null)
            {
                newPartition.ExhibitsTraits.Add(trait);
            }
            else
            {
                Logger.Error(ctx, Tag, nameof(FromData), null, CdmLogCode.ErrPersistSymsUnsupportedTableFormat);
                return(null);
            }

            var properties = obj.Properties;

            if (properties != null)
            {
                if (properties.ContainsKey("cdm:name"))
                {
                    newPartition.Name = (string)properties["cdm:name"];
                }
                if (properties.ContainsKey("cdm:lastFileStatusCheckTime"))
                {
                    newPartition.LastFileStatusCheckTime = DateTimeOffset.Parse(properties["cdm:lastFileStatusCheckTime"].ToString());
                }
                if (properties.ContainsKey("cdm:lastFileModifiedTime"))
                {
                    newPartition.LastFileModifiedTime = DateTimeOffset.Parse(properties["cdm:lastFileModifiedTime"].ToString());
                }
                if (properties.ContainsKey("cdm:traits"))
                {
                    Utils.AddListToCdmCollection(newPartition.ExhibitsTraits, Utils.CreateTraitReferenceList(ctx, properties["cdm:traits"]));
                }
            }

            return(newPartition);
        }
예제 #19
0
        public static Project CreateProject(Project project, List <FileSystemDescriptor> files)
        {
            StorageDescriptor cache = project.Cache;

            files.ParallelFor((file) =>
            {
                CppReferenceNode node = new CppReferenceNode(project.Target, file as FileDescriptor);
                node.FetchImports();

                lock (project)
                    project.Files.Nodes.Add(node);

                StorageContainer element = cache.Store(file);
                if (node.Flags.Count != 0)
                {
                    switch ((OutputAssemblyType)node.Flags[0])
                    {
                    case OutputAssemblyType.Library: element.Tag = LibraryStorageTag; break;

                    case OutputAssemblyType.Console: element.Tag = ConsoleStorageTag; break;

                    case OutputAssemblyType.Executable: element.Tag = ExeStorageTag; break;
                    }
                    lock (project)
                        project.AssemblyType = (OutputAssemblyType)node.Flags[0];
                }
            });

            ProjectAnalyzer.AddProjectFiles(project);

            Filter filter = new Filter();

            filter.Add(".*").Exclude = true;

            project.Location.FindDirectories(filter).ParallelFor((directory) =>
            {
                cache.Store(directory);
            });

            Analyzer.SetProjectType(project);
            Analyzer.SetProjectDependencies(project);
            return(project);
        }
예제 #20
0
        private static void LoadMixins()
        {
            StorageDescriptor mixinCache = new StorageDescriptor(Application.CacheDirectory, ModuleCacheFileName);

            if (mixinCache.Load())
            {
                foreach (StorageContainer node in mixinCache.Nodes.Values)
                {
                    if (!node.Validate())
                    {
                        mixinCache.Clear();
                        break;
                    }
                }
            }
            StorageContainer assemblyNode; if (!mixinCache.TryGetNode(MixinAssemblyStorageAlias, out assemblyNode))

            {
                Filter filter = new Filter();

                SetMixinFilter(filter);
                List <FileSystemDescriptor> mixins = Application.SdkDirectory.FindFiles(filter);
                if (mixins.Count > 0)
                {
                    FileDescriptor assemblyFile = Compile(mixins);
                    assemblyNode = mixinCache.Store(assemblyFile, MixinAssemblyStorageAlias);

                    foreach (FileDescriptor mixin in mixins)
                    {
                        mixinCache.Store(mixin);
                    }
                    mixinCache.Save();
                }
                filter = null;
                mixins = null;
            }
            MixinManager.RegisterMixins(Assembly.GetExecutingAssembly());
            if (assemblyNode != null && assemblyNode.Element.Exists())
            {
                MixinManager.RegisterMixins(Assembly.LoadFrom(assemblyNode.Element.GetAbsolutePath()));
            }
        }
        private void UpdateAccountsCombobox()
        {
            m_cbAccounts.BeginUpdate();

            ImageComboBoxItem item;
            StorageDescriptor descriptor = null;

            foreach (var account in m_configService.Accounts.OrderBy(_ => _.Type).ThenBy(_ => _.Name))
            {
                if (descriptor == null || (account.Type != descriptor.Type))
                {
                    descriptor = StorageRegistry.Descriptors.SingleOrDefault(_ => _.Type == account.Type);
                    if (descriptor == null)
                    {
                        continue;
                    }

                    item = new ImageComboBoxItem()
                    {
                        Text        = descriptor.FriendlyName,
                        ImageIndex  = m_ilProviderIcons.Images.IndexOfKey(descriptor.Type.ToString()),
                        IndentLevel = 0,
                        Font        = new Font(FontFamily.GenericSansSerif, 9, FontStyle.Bold),
                    };

                    m_cbAccounts.Items.Add(item);
                }

                item = new ImageComboBoxItem()
                {
                    Text        = account.Name,
                    ImageIndex  = m_ilProviderIcons.Images.IndexOfKey(account.Type.ToString()),
                    Item        = account,
                    IndentLevel = 1,
                };

                m_cbAccounts.Items.Add(item);
            }
            ;

            m_cbAccounts.EndUpdate();
        }
예제 #22
0
        public static bool FromCache(BuildTarget target, UInt32 tag, PathDescriptor location, out Project project)
        {
            Project tmp = new Project(tag, location);

            tmp.AssemblyType = OutputAssemblyType.Static;
            tmp.Target       = target;
            project          = tmp;

            if (Application.Rebuild)
            {
                return(false);
            }

            PathDescriptor cacheDirectory = new PathDescriptor(Application.CacheDirectory, tag.ToString());

            if (!cacheDirectory.Exists())
            {
                return(false);
            }

            StorageDescriptor cache = tmp.Cache;

            if (cache.Nodes.Count > 0)
            {
                Int32 rebuildFlag = 0;
                cache.Nodes.Values.ParallelFor((node) =>
                {
                    if (Interlocked.CompareExchange(ref rebuildFlag, 0, 0) == 0)
                    {
                        switch (node.Tag)
                        {
                        case 0: break;

                        default:
                            {
                                if (node.Tag == LibraryStorageTag)
                                {
                                    lock (tmp)
                                        tmp.AssemblyType = OutputAssemblyType.Library;
                                }
                                else if (node.Tag == ConsoleStorageTag)
                                {
                                    lock (tmp)
                                        tmp.AssemblyType = OutputAssemblyType.Console;
                                }
                                else if (node.Tag == ExeStorageTag)
                                {
                                    lock (tmp)
                                        tmp.AssemblyType = OutputAssemblyType.Executable;
                                }
                                else
                                {
                                    return;
                                }
                            }
                            break;
                        }
                        if (node.Validate())
                        {
                            if (node.Element is FileDescriptor)
                            {
                                lock (tmp)
                                    tmp.Files.Nodes.Add(new CppReferenceNode(target, node.Element as FileDescriptor));
                            }
                        }
                        else
                        {
                            Interlocked.Exchange(ref rebuildFlag, 1);
                        }
                    }
                });
                if (rebuildFlag == 0)
                {
                    project.IsCached = true;
                    return(true);
                }
            }
            project.Files.Clear();
            project.AssemblyType = OutputAssemblyType.Library;
            return(false);
        }
예제 #23
0
 private SqlConnection CreateSqlConnection(StorageDescriptor storageDescriptor)
 => new SqlConnection(_connectionStringSettings.GetConnectionString(storageDescriptor.ConnectionStringIdentity));
예제 #24
0
        /// <summary>
        /// Each XmlQueryType has multiple legal CLR representations.  Ensure that all items returned by this iterator are in
        /// the Clr representation specified by "storageTypeDest".
        /// </summary>
        public void EnsureItemStorageType(XmlQueryType xmlType, Type storageTypeDest)
        {
            // If source type = destination type, then done
            if (_storage.ItemStorageType == storageTypeDest)
            {
                goto SetStorageType;
            }

            Debug.Assert(_storage.ItemStorageType == typeof(XPathItem) || storageTypeDest == typeof(XPathItem),
                         "EnsureItemStorageType must convert to or from Item");

            // If items are cached,
            if (_storage.IsCached)
            {
                // Check for special case of IList<XPathNavigator> -> IList<XPathItem>
                if (_storage.ItemStorageType == typeof(XPathNavigator))
                {
                    EnsureStack();
                    _helper.Call(XmlILMethods.NavsToItems);
                    goto SetStorageType;
                }

                // Check for special case of IList<XPathItem> -> IList<XPathNavigator>
                if (storageTypeDest == typeof(XPathNavigator))
                {
                    EnsureStack();
                    _helper.Call(XmlILMethods.ItemsToNavs);
                    goto SetStorageType;
                }
            }

            // Iterate over each item, and convert each to the destination type
            EnsureStackNoCache();

            // If source type is Item,
            if (_storage.ItemStorageType == typeof(XPathItem))
            {
                // Then downcast to Navigator
                if (storageTypeDest == typeof(XPathNavigator))
                {
                    _helper.Emit(OpCodes.Castclass, typeof(XPathNavigator));
                }
                else
                {
                    // Call ValueAs methods for atomic types
                    _helper.CallValueAs(storageTypeDest);
                }
                goto SetStorageType;
            }
            else if (_storage.ItemStorageType == typeof(XPathNavigator))
            {
                // No-op if converting from XPathNavigator to XPathItem
                Debug.Assert(storageTypeDest == typeof(XPathItem), "Must be converting from XPathNavigator to XPathItem");
                goto SetStorageType;
            }

            // Destination type must be item, so generate code to create an XmlAtomicValue
            _helper.LoadInteger(_helper.StaticData.DeclareXmlType(xmlType));
            _helper.LoadQueryRuntime();
            _helper.Call(XmlILMethods.StorageMethods[_storage.ItemStorageType].ToAtomicValue !);

SetStorageType:
            _storage = _storage.ToStorageType(storageTypeDest);
        }
 private DataConnection CreateDataConnection(StorageDescriptor storageDescriptor)
 => CreateDataConnection(
     _connectionStringSettings.GetConnectionString(storageDescriptor.ConnectionStringIdentity),
     storageDescriptor.MappingSchema,
     storageDescriptor.CommandTimeout);
예제 #26
0
        /// <summary>
        /// Creates or updates a glue table for the new CUR files. This makes sure any changes in the columns are captured
        /// and applied to the table. This will end up creating a new table for each billing period.
        /// </summary>
        /// <param name="manifest"></param>
        /// <param name="context"></param>
        /// <returns>The table name</returns>
        private static async Task <string> CreateOrUpdateGlueTable(Manifest manifest, ILambdaContext context)
        {
            if (String.IsNullOrEmpty(_GlueDatabaseName))
            {
                string Message = "No Glue database name defined, cannot create a table.";
                context.LogWarning(Message);
                await SNSNotify(Message, context);

                return(String.Empty);
            }

            string Date = manifest.BillingPeriod.Start.ToString("yyyy-MM-dd");

            string Format = manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1);

            Dictionary <string, string> Parameters;
            StorageDescriptor           Descriptor;

            switch (Format)
            {
            case "csv":
            {
                Parameters = new Dictionary <string, string>()
                {
                    { "EXTERNAL", "TRUE" },
                    { "skip.header.line.count", "1" },
                    { "columnsOrdered", "true" },
                    { "compressionType", manifest.Compression.ToString().ToLower() },
                    { "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
                };

                Descriptor = new StorageDescriptor()
                {
                    Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
                        {
                            Name = $"{x.Category}/{x.Name}", Type = "string"
                        }).ToList(),
                    InputFormat  = "org.apache.hadoop.mapred.TextInputFormat",
                    OutputFormat = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
                    Location     = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
                    SerdeInfo    = new SerDeInfo()
                    {
                        Name = "OpenCSVSerde",
                        SerializationLibrary = "org.apache.hadoop.hive.serde2.OpenCSVSerde",
                        Parameters           = new Dictionary <string, string>()
                        {
                            { "escapeChar", "\\" },
                            { "quoteChar", "\"" },
                            { "separatorChar", "," }
                        }
                    }
                };

                break;
            }

            case "parquet":
            {
                Parameters = new Dictionary <string, string>()
                {
                    { "EXTERNAL", "TRUE" },
                    { "compressionType", manifest.Compression.ToString().ToLower() },
                    { "classification", manifest.ContentType.ToLower().Substring(manifest.ContentType.LastIndexOf("/") + 1) }
                };

                Descriptor = new StorageDescriptor()
                {
                    Columns = manifest.Columns.Select(x => new Amazon.Glue.Model.Column()
                        {
                            Name = x.Name, Type = (!String.IsNullOrEmpty(x.Type) ? x.Type.ToLower() : "string")
                        }).ToList(),
                    InputFormat  = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
                    OutputFormat = "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
                    Location     = $"s3://{_DestinationBucket}/{GetDestinationPrefix(manifest)}",
                    SerdeInfo    = new SerDeInfo()
                    {
                        Name = "ParquetHiveSerDe",
                        SerializationLibrary = "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
                        Parameters           = new Dictionary <string, string>()
                        {
                            { "serialization.format", "1" }
                        }
                    }
                };
                break;
            }

            default:
            {
                string Message = $"Failed to create or update the database {_GlueDatabaseName} table. Unknown format type ${manifest.ContentType}.";
                await SNSNotify(Message, context);

                return(String.Empty);
            }
            }

            // The updated table input for this particular CUR
            TableInput TblInput = new TableInput()
            {
                Description       = Date,
                Name              = Date,
                TableType         = "EXTERNAL_TABLE",
                Parameters        = Parameters,
                StorageDescriptor = Descriptor
            };

            // Make sure the database exists
            GetDatabaseRequest GetDb = new GetDatabaseRequest()
            {
                Name = _GlueDatabaseName
            };

            try
            {
                await _GlueClient.GetDatabaseAsync(GetDb);

                context.LogInfo($"Database {_GlueDatabaseName} already exists.");
            }
            catch (EntityNotFoundException)
            {
                try
                {
                    CreateDatabaseRequest DbRequest = new CreateDatabaseRequest()
                    {
                        DatabaseInput = new DatabaseInput()
                        {
                            Name = _GlueDatabaseName
                        }
                    };

                    CreateDatabaseResponse Response = await _GlueClient.CreateDatabaseAsync(DbRequest);

                    if (Response.HttpStatusCode == HttpStatusCode.OK)
                    {
                        context.LogInfo($"Successfully CREATED database {_GlueDatabaseName}.");
                    }
                    else
                    {
                        context.LogError($"Failed to CREATE database with status code {(int)Response.HttpStatusCode}.");
                    }
                }
                catch (Exception ex)
                {
                    string Message = $"Failed to create the database {_GlueDatabaseName}.";
                    context.LogError(Message, ex);
                    await SNSNotify(Message + $" {ex.Message}", context);

                    return(String.Empty);
                }
            }

            // Make sure the table exists
            GetTableRequest GetTable = new GetTableRequest()
            {
                DatabaseName = _GlueDatabaseName,
                Name         = Date
            };

            try
            {
                GetTableResponse TableResponse = await _GlueClient.GetTableAsync(GetTable);

                UpdateTableRequest UpdateReq = new UpdateTableRequest()
                {
                    TableInput   = TblInput,
                    DatabaseName = _GlueDatabaseName
                };

                UpdateTableResponse Response = await _GlueClient.UpdateTableAsync(UpdateReq);

                if (Response.HttpStatusCode == HttpStatusCode.OK)
                {
                    context.LogInfo($"Successfully UPDATED table {TblInput.Name} in database {_GlueDatabaseName}.");
                    return(TblInput.Name);
                }
                else
                {
                    string Message = $"Failed to UPDATE table with status code {(int)Response.HttpStatusCode}.";
                    context.LogError(Message);
                    await SNSNotify(Message, context);

                    return(String.Empty);
                }
            }
            catch (EntityNotFoundException) // This means the table does not exist
            {
                CreateTableRequest CreateReq = new CreateTableRequest()
                {
                    TableInput   = TblInput,
                    DatabaseName = _GlueDatabaseName
                };

                CreateTableResponse Response = await _GlueClient.CreateTableAsync(CreateReq);

                if (Response.HttpStatusCode == HttpStatusCode.OK)
                {
                    context.LogInfo($"Successfully CREATED table {TblInput.Name} in database {_GlueDatabaseName}.");
                    return(TblInput.Name);
                }
                else
                {
                    string Message = $"Failed to CREATE table with status code {(int)Response.HttpStatusCode}.";
                    context.LogError(Message);
                    await SNSNotify(Message, context);

                    return(String.Empty);
                }
            }
        }
예제 #27
0
        public static StorageDescriptor ToData(CdmDataPartitionDefinition instance, StorageDescriptor obj, ResolveOptions resOpt, CopyOptions options)
        {
            obj.Properties = new Dictionary <string, JToken>();

            if (instance.Name != null)
            {
                obj.Properties["cdm:name"] = instance.Name;
            }

            if (instance.LastFileStatusCheckTime != null)
            {
                obj.Properties["cdm:lastFileStatusCheckTime"] = instance.LastFileStatusCheckTime;
            }

            if (instance.LastFileModifiedTime != null)
            {
                obj.Properties["cdm:lastFileModifiedTime"] = instance.LastFileModifiedTime;
            }

            if (instance.ExhibitsTraits != null)
            {
                TraitToPropertyMap tpm = new TraitToPropertyMap(instance);
                var csvTrait           = tpm.FetchTraitReference("is.partition.format.CSV");
                if (csvTrait != null)
                {
                    instance.ExhibitsTraits.Remove("is.partition.format.CSV");
                }

                if (instance.ExhibitsTraits.Count > 0)
                {
                    obj.Properties["cdm:traits"] = JToken.FromObject(Utils.ListCopyData <TraitReferenceDefinition>(resOpt, instance.ExhibitsTraits, options),
                                                                     new JsonSerializer {
                        NullValueHandling = NullValueHandling.Ignore, ContractResolver = new CamelCasePropertyNamesContractResolver()
                    });
                }
                if (csvTrait != null)
                {
                    instance.ExhibitsTraits.Add(csvTrait);
                }
            }

            var properties = FillPropertyBagFromCsvTrait(instance);

            if (properties != null)
            {
                obj.Format = new FormatInfo
                {
                    InputFormat  = InputFormat.OrgapachehadoopmapredSequenceFileInputFormat,
                    OutputFormat = OutputFormat.OrgapachehadoophiveqlioHiveSequenceFileOutputFormat,
                    SerializeLib = SerializeLib.Orgapachehadoophiveserde2lazyLazySimpleSerDe,
                    FormatType   = FormatType.Csv,
                    Properties   = properties
                };
            }
            else
            {
                // error
                return(null);
            }

            return(obj);
        }
예제 #28
0
        public static void DisplayManual()
        {
            Application.Log(SeverityFlags.None, "Forge {0}, distributed under the Schroedinger Entertainment EULA (See EULA.md for details)", Application.Version);

            StorageDescriptor mixinCache = new StorageDescriptor(Application.CacheDirectory, ModuleCacheFileName);

            if (mixinCache.Load() && mixinCache.Nodes.Count > 0)
            {
                Application.Log(SeverityFlags.None, "Loaded {0} Mixin(s) from root", mixinCache.Nodes.Count - 1);
                foreach (StorageContainer mixin in mixinCache.Nodes.Values)
                {
                    if (mixin.Tag == 0)
                    {
                        Application.Log(SeverityFlags.None, "  {0}", mixin.Element.GetRelativePath(Application.SdkDirectory));
                    }
                }
            }
            mixinCache = null;

            Application.Log(SeverityFlags.None, "Loaded {0} Plugin(s) from root", plugins.Count);
            foreach (Plugin plugin in plugins)
            {
                Application.Log(SeverityFlags.None, "  {0}", plugin.Path.GetRelativePath(Application.SdkDirectory));
            }

            Application.Log(SeverityFlags.None, string.Empty);

            PageFormatter page = new PageFormatter();

            AutoConfig.GetAttributePage <Application>(page);
            page.Sort();

            PageFormatter subPage = new PageFormatter();

            AutoConfig.GetAttributePage(Application.Profile.GetType(), subPage);
            AutoConfig.GetAttributePage(Application.Profile, subPage);
            subPage.Sort();

            if (subPage.Rows.Count > 0)
            {
                page.AddRow("");
                page.AddRow(string.Format("{0} Profile", Application.Profile.Name.ToTitleCase()));
                page.Rows.AddRange(subPage.Rows);
            }

            Dictionary <string, PageFormatter> subSystems = new Dictionary <string, PageFormatter>();

            AppDomain.CurrentDomain.GetAssemblies().ParallelFor((assembly) =>
            {
                foreach (Type receiver in assembly.GetTypes <IConfigReceiver>())
                {
                    try
                    {
                        PageFormatter systemPage = new PageFormatter();
                        AutoConfig.GetAttributePage(receiver, systemPage);

                        IConfigReceiver instance; if (receiver.IsAbstract && MixinType.IsAssignableFrom(receiver))
                        {
                            instance = MixinManager.Create(receiver) as IConfigReceiver;
                        }
                        else if (!receiver.IsAbstract)
                        {
                            instance = receiver.CreateInstance <IConfigReceiver>();
                        }
                        else
                        {
                            continue;
                        }

                        if (instance != null && instance.Display)
                        {
                            instance.GetAdditionalManualInfo(systemPage);
                            if (systemPage.Rows.Count > 0)
                            {
                                lock (subSystems)
                                {
                                    PageFormatter tmp; if (subSystems.TryGetValue(instance.DisplayName, out tmp))
                                    {
                                        tmp.AddRows(systemPage.Rows);
                                    }
                                    else
                                    {
                                        subSystems.Add(instance.DisplayName, systemPage);
                                    }
                                }
                            }
                        }
                    }
                    catch
                    { }
                }
            });
            foreach (KeyValuePair <string, PageFormatter> systems in subSystems.OrderBy(x => x.Key))
            {
                systems.Value.Sort();

                page.AddRow("");
                page.AddRow(string.Format("{0} System", systems.Key));
                page.Rows.AddRange(systems.Value.Rows);
            }
            foreach (Plugin plugin in plugins)
            {
                subPage = new PageFormatter();
                plugin.GetManualPage(subPage);
                subPage.Sort();

                if (subPage.Rows.Count > 0)
                {
                    page.AddRow("");
                    page.AddRow(string.Format("{0} Plugin", plugin.Path.Name.ToTitleCase()));
                    page.Rows.AddRange(subPage.Rows);
                }
            }

            Application.Log(SeverityFlags.None, page.ToString());
        }
예제 #29
0
        public override void Execute()
        {
            LoadInput();

            List <Project> projects = new List <Project>();

            foreach (TaskPin inputPin in inputPins)
            {
                projects.Add(inputPin.Data as Project);
            }

            Analyzer.GetModulePaths().ParallelFor((location) =>
            {
                foreach (TaskPin inputPin in inputPins)
                {
                    if ((inputPin.Data as FileDescriptor).Location.Contains(location))
                    {
                        return;
                    }
                }

                foreach (BuildTarget target in CompilerServices.Targets)
                {
                    Project module; if (!ProjectLookup.FromCache(target, SystemTags.Cpp, location, out module))
                    {
                        List <FileSystemDescriptor> files = Application.GetProjectFiles(location, ProjectLookup.Extensions);
                        if (files.Count > 0)
                        {
                            module = ProjectLookup.CreateProject(module, files);
                        }
                    }

                    module.AssemblyType = OutputAssemblyType.Static;
                    module.IsModule     = true;

                    lock (projects)
                        if (!projects.Contains(module))
                        {
                            Application.Log(SeverityFlags.Full, "Loaded {0} module {1}::{2}", ToolDisplayName, module.Name, module.Target.TechnicalName);
                            projects.Add(module);
                        }
                }
            });
            projects.ParallelFor((project) =>
            {
                StorageDescriptor cache = project.Cache;
                if (project.IsCached)
                {
                    cache.Nodes.Values.ParallelFor((element) =>
                    {
                        if (element.Tag == ReferenceStorageTag)
                        {
                            projects.Where(x => x.Target.Id == project.Target.Id).ParallelFor((reference) =>
                            {
                                if (reference == element.Element as FileDescriptor)
                                {
                                    Application.Log(SeverityFlags.Full, "Cached {0} module {1}::{2}", ToolDisplayName, reference.Name, reference.Target.TechnicalName);
                                    lock (project)
                                        project.References.Add(reference);
                                }
                            });
                        }
                        else if (element.Tag == ExternalStorageTag)
                        {
                            project.Externals.Add(element.Element as FileDescriptor);
                        }
                    });
                }
                else
                {
                    DependencyResolver.ConnectFiles(project, projects);

                    foreach (Project referene in project.References)
                    {
                        cache.Store <ReferenceContainer>(referene).Tag = ReferenceStorageTag;
                    }
                    foreach (FileDescriptor referene in project.Externals)
                    {
                        cache.Store <ReferenceContainer>(referene).Tag = ExternalStorageTag;
                    }

                    cache.Save();
                }
            });

            foreach (TaskPin inputPin in inputPins)
            {
                outputPins.Add(new FlaggedPin(this, inputPin.Data, SystemTags.Cpp));
            }

            for (int i = 0; i < inputPins.Count; i++)
            {
                foreach (Project reference in (outputPins[i].Data as Project).References)
                {
                    outputPins.Add(new FlaggedPin(this, reference, SystemTags.Cpp));
                }
            }

            if (printProjectStack)
            {
                PathDescriptor outputPath = Application.GetDeploymentPath(SystemTags.Analytics);
                if (!outputPath.Exists())
                {
                    try
                    {
                        outputPath.Create();
                    }
                    catch { }
                }

                ReferenceGraph moduleGraph = new ReferenceGraph();
                foreach (Project project in projects)
                {
                    project.Files.Nodes.ParallelFor((file) =>
                    {
                        file.Rect = Rectangle.Empty;
                        file.Slot = null;
                    });
                    project.Files.Save(outputPath, string.Format("{0}.cpp", project.Name));
                    AddProjectToGraph(moduleGraph, project);
                }
                moduleGraph.Save(outputPath, "CppModules");
            }
        }
예제 #30
0
 /// <summary>
 /// Set this iterator's next label and storage.  This iterator will range over a set of values located in
 /// "storage".  To get the next value, jump to "lblNext".
 /// </summary>
 public void SetIterator(Label lblNext, StorageDescriptor storage)
 {
     _lblNext = lblNext;
     _hasNext = true;
     _storage = storage;
 }