Example #1
0
        public PqlCompilerState(
            IExpressionEvaluatorRuntime parentRuntime,
            ParsedRequest parsedRequest,
            DataRequestParams requestParams,
            DataContainerDescriptor containerDescriptor,
            Type contextType,
            Type returnType)
            : base(parentRuntime, contextType, returnType, null)
        {
            if (parsedRequest == null)
            {
                throw new ArgumentNullException("parsedRequest");
            }

            if (containerDescriptor == null)
            {
                throw new ArgumentNullException("containerDescriptor");
            }

            if (requestParams == null)
            {
                throw new ArgumentNullException("requestParams");
            }

            ParsedRequest = parsedRequest;
            RequestParameters = requestParams;
            ContainerDescriptor = containerDescriptor;
            FieldRefs = new Dictionary<int, Tuple<ParameterExpression, Expression>>();
            ParamRefs = new Dictionary<int, Tuple<ParameterExpression, Expression>>();
        }
Example #2
0
        public void AttachContainerDescriptor(DataContainerDescriptor containerDescriptor)
        {
            if (containerDescriptor == null)
            {
                throw new ArgumentNullException("containerDescriptor");
            }

            ContainerDescriptor = containerDescriptor;
        }
        public QueryPreprocessor(DataContainerDescriptor containerDescriptor)
        {
            if (containerDescriptor == null)
            {
                throw new ArgumentNullException("containerDescriptor");
            }

            m_containerDescriptor  = containerDescriptor;
            m_identifierAliasParts = new ConcurrentDictionary <string, ParseTreeNode>(StringComparer.OrdinalIgnoreCase);
        }
        private static void CompileInsertUpdateClauses(
            IStorageDriver storageDriver, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, DriverChangeType changeType)
        {
            var updates = cacheInfo.ParsedRequest.Modify.UpdateAssignments;
            var clauses = cacheInfo.ParsedRequest.Modify.InsertUpdateSetClauses;
            var fields = cacheInfo.ParsedRequest.Modify.ModifiedFields;

            if (clauses.Count != fields.Count)
            {
                throw new Exception(string.Format("Internal error: insert/update clauses count ({0}) does not match count of modified fields ({1})",
                    clauses.Count, fields.Count));
            }

            // compile field assignment clauses (SET clauses or value inserts)
            for (var ordinal = 0; ordinal < clauses.Count; ordinal++)
            {
                var clause = clauses[ordinal];
                var field = fields[ordinal];

                // for bulk requests, primary key is there but it is only used to lookup the record
                // for non-bulk requests, primary key should not be in the list of UPDATE clauses
                if (changeType == DriverChangeType.Update && !cacheInfo.ParsedRequest.IsBulk)
                {
                    if (!storageDriver.CanUpdateField(field.FieldId))
                    {
                        throw new Exception(string.Format("Cannot update field {0}/{1} on entity {2}", field.FieldId, field.Name, cacheInfo.ParsedRequest.TargetEntity.Name));
                    }
                }

                // prepare Action compilation context
                var compilerState = QueryParser.PrepareCompilerState(containerDescriptor, cacheInfo, null);
                compilerState.CompileToAction = true;

                // extractor has signature like Func<ClauseEvaluationContext, T>
                var extractor = clause == null
                                    ? QueryParser.CompileFieldValueExtractorClause(compilerState, field, containerDescriptor, cacheInfo, MakeNullableType(field.DbType))
                                    : QueryParser.CompileClause(compilerState, clause, containerDescriptor, cacheInfo, MakeNullableType(field.DbType));
                // get the value into local variable, to prevent multiple invokations when row writer checks for null
                var extractedValue = Expression.Variable(extractor.Type);

                // now take the extractor and create another method, that will take the value and then put it into the changebuffer's data
                var changeBufferData = Expression.Field(Expression.Field(compilerState.Context, "ChangeBuffer"), "Data");
                var blockBody = Expression.Block(
                    new[] {extractedValue},
                    Expression.Assign(extractedValue, extractor),
                    DriverRowData.CreateWriteAccessor(extractedValue, changeBufferData, field.DbType, ordinal));

                updates.Add(
                    new ParsedRequest.FieldAssignment
                    {
                        Field = field,
                        CompiledExpression = (Action<ClauseEvaluationContext>)QueryParser.CompileClause(blockBody, compilerState)
                    });
            }
        }
Example #5
0
        public DataContainerDescriptor Commit()
        {
            if (m_descriptor == null)
            {
                throw new InvalidOperationException("Cannot invoke Commit more than once");
            }

            var result = m_descriptor;

            m_descriptor = null;
            return(result);
        }
        public DocumentDataContainer(
            DataContainerDescriptor dataContainerDescriptor,
            DocumentTypeDescriptor documentTypeDescriptor,
            IUnmanagedAllocator allocator,
            ITracer tracer)
        {
            if (tracer == null)
            {
                throw new ArgumentNullException("tracer");
            }

            if (dataContainerDescriptor == null)
            {
                throw new ArgumentNullException("dataContainerDescriptor");
            }

            if (documentTypeDescriptor == null)
            {
                throw new ArgumentNullException("documentTypeDescriptor");
            }

            if (allocator == null)
            {
                throw new ArgumentNullException("allocator");
            }

            m_logger = tracer;

            m_allocator = allocator;
            DocDesc = documentTypeDescriptor;
            DataContainerDescriptor = dataContainerDescriptor;

            ColumnStores = new ColumnDataBase[DocDesc.Fields.Length];
            DocumentKeys = new ExpandableArrayOfKeys(m_allocator);
            FieldIdToColumnStore = new Dictionary<int, int>(ColumnStores.Length * 2);
            PrimaryKeyFieldId = dataContainerDescriptor.RequireField(documentTypeDescriptor.DocumentType, documentTypeDescriptor.PrimaryKeyFieldName).FieldId;

            for (var i = 0; i < DocDesc.Fields.Length; i++)
            {
                var field = dataContainerDescriptor.RequireField(DocDesc.Fields[i]);
                ColumnStores[i] = CreateColumnStore(field.DbType, m_allocator, null);
                FieldIdToColumnStore.Add(field.FieldId, i);
            }

            DocumentIdToIndex = new ConcurrentHashmapOfKeys(m_allocator);
            ValidDocumentsBitmap = new BitVector(m_allocator);
            SortIndexManager = new SortIndexManager(this);
            StructureLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);
        }
        private static void CompileSelectClauses(DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo)
        {
            var parsedRequest = cacheInfo.ParsedRequest;

            if (parsedRequest.IsBulk)
            {
                foreach (var field in parsedRequest.Select.SelectFields)
                {
                    var exprType = MakeNullableType(field.DbType);

                    parsedRequest.Select.OutputColumns.Add(
                        new ParsedRequest.SelectOutputColumn
                            {
                                Label = field.Name,
                                CompiledExpression = QueryParser.CompileFieldValueExtractorClause(field, containerDescriptor, cacheInfo, exprType),
                                DbType = field.DbType,
                                IsNullable = exprType.IsNullableType()
                            });
                }
            }
            else
            {
                foreach (var clause in parsedRequest.Select.SelectClauses)
                {
                    // under column item, there is a "columnSource" element (with a single child), and optional "Id" element for alias
                    var columnExpressionNode = clause.RequireChild("columnSource", 0).RequireChild(null, 0);
                    var compiled = QueryParser.CompileClause(columnExpressionNode, containerDescriptor, cacheInfo, null);
                    var returnType = compiled.GetType().GetGenericArguments()[1];
                    var dbType = DriverRowData.DeriveDataType(returnType.GetUnderlyingType());

                    // get alias
                    var aliasNode = clause.TryGetChild("Id", 1) ?? columnExpressionNode;
                    var span = aliasNode.Span;
                    var label = span.Length > 0
                                    ? cacheInfo.CommandText.Substring(span.Location.Position, span.Length)
                                    : aliasNode.FindTokenAndGetText();

                    parsedRequest.Select.OutputColumns.Add(
                        new ParsedRequest.SelectOutputColumn
                            {
                                Label = label,
                                CompiledExpression = compiled,
                                DbType = dbType,
                                IsNullable = returnType.IsNullableType()
                            });
                }
            }
        }
Example #8
0
        public static DataContainerDescriptor BuildContainerDescriptor()
        {
            var result = new DataContainerDescriptor();
            result.AddDocumentTypeName("testDoc");

            var testDocId = result.RequireDocumentTypeName("testDoc");

            var fieldId = 1;
            for (var i = 0; i < 2; i++)
            {
                result.AddField(new FieldMetadata(fieldId, "FieldByte" + fieldId, "Byte" + fieldId, DbType.Byte, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "FieldGuid" + fieldId, "Guid" + fieldId, DbType.Guid, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "FieldString" + fieldId, "String" + fieldId, DbType.String, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "FieldBinary" + fieldId, "Binary" + fieldId, DbType.Binary, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "FieldDecimal" + fieldId, "Decimal" + fieldId, DbType.Decimal, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "FieldBool" + fieldId, "Bool" + fieldId, DbType.Boolean, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "FieldDate" + fieldId, "Date" + fieldId, DbType.DateTime, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "FieldTime" + fieldId, "Time" + fieldId, DbType.Time, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "FieldDateTimeOffset" + fieldId, "DateTimeOffset" + fieldId, DbType.DateTimeOffset, testDocId));
                fieldId++;
            }

            result.AddField(new FieldMetadata(fieldId + 1, "id", "primary key", DbType.Int64, testDocId));

            var fieldIds = result.EnumerateFields().Select(x => x.FieldId).ToArray();
            result.AddDocumentTypeDescriptor(new DocumentTypeDescriptor("testDoc", "testDoc", 1, "id", fieldIds));

            result.AddIdentifierAlias("testDoc", new List<string> { "id", "ALIAS1" }, new[] { "id" });

            return result;
        }
Example #9
0
        public static void WriteDescriptorToStore(DataContainerDescriptor descriptor, string storageRoot)
        {
            if (string.IsNullOrEmpty(storageRoot))
            {
                throw new ArgumentNullException(storageRoot);
            }

            if (descriptor == null)
            {
                throw new ArgumentNullException("descriptor");
            }

            Directory.CreateDirectory(storageRoot);

            var file = new DataContainerDescriptorFile(RamDriverFactory.CurrentStoreVersion(), descriptor);
            using (var writer = new StreamWriter(
                new FileStream(Path.Combine(storageRoot, "descriptor.json"), FileMode.Create, FileAccess.ReadWrite)))
            {
                var serializer = new JsonSerializer();
                serializer.Serialize(writer, file);
                writer.Flush();
            }
        }
        public void AttachContainerDescriptor(DataContainerDescriptor containerDescriptor)
        {
            if (containerDescriptor == null)
            {
                throw new ArgumentNullException("containerDescriptor");
            }

            ContainerDescriptor = containerDescriptor;
        }
Example #11
0
 public static object CompileClause(ParseTreeNode parseTreeNode, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, Type returnType)
 {
     var compilerState = PrepareCompilerState(containerDescriptor, cacheInfo, returnType);
     var exprBody = CompileClause(compilerState, parseTreeNode, containerDescriptor, cacheInfo, returnType);
     return s_expressionRuntime.Compile(exprBody, compilerState);
 }
Example #12
0
 public SchemaBuilder()
 {
     m_descriptor = new DataContainerDescriptor();
 }
Example #13
0
        public static Expression CompileClause(PqlCompilerState compilerState, ParseTreeNode parseTreeNode, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, Type returnType)
        {
            var exprBody = s_expressionRuntime.Analyze(parseTreeNode, compilerState);

            // now add our field references to the expression
            if (compilerState.FieldRefs.Count > 0 || compilerState.ParamRefs.Count > 0)
            {
                // variable declarations
                var localVariables = new ParameterExpression[compilerState.FieldRefs.Count + compilerState.ParamRefs.Count];
                var exprList = new Expression[1 + localVariables.Length];

                var ix = 0;
                foreach (var pair in compilerState.FieldRefs)
                {
                    localVariables[ix] = pair.Value.Item1;
                    exprList[ix] = Expression.Assign(pair.Value.Item1, pair.Value.Item2);
                    ix++;
                }
                foreach (var pair in compilerState.ParamRefs)
                {
                    localVariables[ix] = pair.Value.Item1;
                    exprList[ix] = Expression.Assign(pair.Value.Item1, pair.Value.Item2);
                    ix++;
                }

                // and the expression code itself
                exprList[ix] = exprBody;
                // ready to go
                exprBody = Expression.Block(localVariables, exprList);
            }

            return s_expressionRuntime.AdjustReturnType(exprBody, returnType);
        }
Example #14
0
 public static object CompileFieldValueExtractorClause(FieldMetadata field, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, Type returnType)
 {
     var compilerState = PrepareCompilerState(containerDescriptor, cacheInfo, returnType);
     var fieldAccessor = CompileFieldValueExtractorClause(compilerState, field, containerDescriptor, cacheInfo, returnType);
     return s_expressionRuntime.Compile(fieldAccessor, compilerState);
 }
Example #15
0
 public static Expression CompileFieldValueExtractorClause(PqlCompilerState compilerState, FieldMetadata field, DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, Type returnType)
 {
     var ordinal = GetFieldOrdinalInDriverFetchFields(cacheInfo.ParsedRequest, field);
     var rowData = Expression.Field(compilerState.Context, "InputRow");
     return s_expressionRuntime.AdjustReturnType(DriverRowData.CreateReadAccessor(rowData, field.DbType, ordinal), returnType);
 }
        private void CompileClauses(DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo)
        {
            if (cacheInfo.ParsedRequest.SpecialCommand.IsSpecialCommand)
            {
                return;
            }

            if (cacheInfo.ParsedRequest.BaseDataset.WhereClauseRoot != null)
            {
                cacheInfo.ParsedRequest.BaseDataset.WhereClauseProcessor =
                    (Func<ClauseEvaluationContext, bool>) QueryParser.CompileWhereClause(
                        containerDescriptor,
                        cacheInfo.ParsedRequest.BaseDataset.WhereClauseRoot,
                        cacheInfo);
            }

            switch (cacheInfo.ParsedRequest.StatementType)
            {
                case StatementType.Select:
                    CompileSelectClauses(containerDescriptor, cacheInfo);
                    break;
                case StatementType.Update:
                    CompileInsertUpdateClauses(m_storageDriver, containerDescriptor, cacheInfo, DriverChangeType.Update);
                    break;
                case StatementType.Insert:
                    CompileInsertUpdateClauses(m_storageDriver, containerDescriptor, cacheInfo, DriverChangeType.Insert);
                    break;
                case StatementType.Delete:
                    break;
                default:
                    throw new Exception("Invalid statement type: " + cacheInfo.ParsedRequest.StatementType);
            }
        }
Example #17
0
 public static object CompileWhereClause(DataContainerDescriptor containerDescriptor, ParseTreeNode parseTreeNode, RequestExecutionContextCacheInfo cacheInfo)
 {
     return CompileClause(parseTreeNode, containerDescriptor, cacheInfo, typeof (bool));
 }
Example #18
0
        public DataContainerDescriptor Commit()
        {
            if (m_descriptor == null)
            {
                throw new InvalidOperationException("Cannot invoke Commit more than once");
            }

            var result = m_descriptor;
            m_descriptor = null;
            return result;
        }
Example #19
0
 /// <summary>
 /// Ctr.
 /// </summary>
 public RamDriverSettings(RamDriverSettings settings)
 {
     if (settings != null)
     {
         InitializationCommand = settings.InitializationCommand;
         StorageRoot = settings.StorageRoot;
         Descriptor = settings.Descriptor;
     }
 }
Example #20
0
        private static DataContainerDescriptor BuildDemoContainerDescriptor()
        {
            var result = new DataContainerDescriptor();
            result.AddDocumentTypeName("testDoc");

            var testDocId = result.RequireDocumentTypeName("testDoc");
            var count = 12;

            for (var fieldId = 1; fieldId <= count; )
            {
                result.AddField(new FieldMetadata(fieldId, "Field" + fieldId, "Byte" + fieldId, DbType.Byte, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "Field" + fieldId, "Guid" + fieldId, DbType.Guid, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "Field" + fieldId, "String" + fieldId, DbType.String, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "Field" + fieldId, "Binary" + fieldId, DbType.Binary, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "Field" + fieldId, "Decimal" + fieldId, DbType.Decimal, testDocId));
                fieldId++;
                result.AddField(new FieldMetadata(fieldId, "Field" + fieldId, "Bool" + fieldId, DbType.Boolean, testDocId));
                fieldId++;
            }

            result.AddField(new FieldMetadata(count + 1, "id", "primary key", DbType.Int64, testDocId));

            var fieldIds = result.EnumerateFields().Select(x => x.FieldId).ToArray();
            result.AddDocumentTypeDescriptor(new DocumentTypeDescriptor("testDoc", "testDoc", 1, "id", fieldIds));

            return result;
        }
Example #21
0
        /// <summary>
        /// Ctr.
        /// </summary>
        public QueryParser(DataContainerDescriptor containerDescriptor, int maxConcurrency)
        {
            if (containerDescriptor == null)
            {
                throw new ArgumentNullException("containerDescriptor");
            }

            m_parsers = new ObjectPool<Irony.Parsing.Parser>(maxConcurrency, null);
            for (var i = 0; i < maxConcurrency; i++)
            {
                m_parsers.Return(new Irony.Parsing.Parser(LangData, PqlNonTerminal));
            }

            m_containerDescriptor = containerDescriptor;
            m_preprocessor = new QueryPreprocessor(containerDescriptor);

            // these predefined instances of ParseTreeNode are substituted when parsing "select * from .." statement,
            // in order to avoid allocating them every time
            m_simpleFieldAccessorNodes = new Dictionary<int, ParseTreeNode>();
            foreach (var field in m_containerDescriptor.EnumerateFields())
            {
                // generate columnItem -> columnSource -> Id -> id_simple hierarchy, exactly same structure as it comes out of grammar-based parser
                var idNode = new ParseTreeNode(new NonTerminal("Id"), new SourceSpan());
                idNode.ChildNodes.Add(new ParseTreeNode(new Token(new Terminal("id_simple"), new SourceLocation(), field.Name, field.Name)));

                var columnSourceNode = new ParseTreeNode(new NonTerminal("columnSource"), new SourceSpan());
                columnSourceNode.ChildNodes.Add(idNode);

                var columnItemNode = new ParseTreeNode(new NonTerminal("columnItem"), new SourceSpan());
                columnItemNode.ChildNodes.Add(columnSourceNode);

                m_simpleFieldAccessorNodes.Add(field.FieldId, columnItemNode);
            }
        }
Example #22
0
        private static FieldMetadata TryGetFieldByIdentifierNode(ParseTreeNode node, DataContainerDescriptor containerDescriptor, int docType)
        {
            // we support simple and dot-separated identifiers
            // for a dot-separated identifier, field name is the part before first dot, and it MUST be an object type field
            if (node.ChildNodes.Count >= 1)
            {
                var idsimple = node.ChildNodes[0];
                if (0 == StringComparer.Ordinal.Compare(idsimple.Term.Name, "id_simple"))
                {
                    var fieldName = idsimple.Token.ValueString;
                    var field = containerDescriptor.TryGetField(docType, fieldName);
                    if (field != null)
                    {
                        if (node.ChildNodes.Count > 1)
                        {
                            throw new CompilationException("Dotted notation is supported, but not allowed in Pql server context");
                            //if (field.DbType != DbType.Object)
                            //{
                            //    throw new CompilationException("Dotted notation is only allowed on object-type fields", node);
                            //}
                        }
                    }

                    return field;
                }
            }

            return null;
        }
Example #23
0
 private static TreeIteratorContext GetTreeIteratorContext(ParsedRequest parsedRequest, DataContainerDescriptor containerDescriptor)
 {
     var result = new TreeIteratorContext();
     result.ContainerDescriptor = containerDescriptor;
     result.ParsedRequest = parsedRequest;
     return result;
 }
Example #24
0
 public static PqlCompilerState PrepareCompilerState(DataContainerDescriptor containerDescriptor, RequestExecutionContextCacheInfo cacheInfo, Type returnType)
 {
     return new PqlCompilerState(
         s_expressionRuntime,
         cacheInfo.ParsedRequest,
         cacheInfo.RequestParams,
         containerDescriptor,
         typeof(ClauseEvaluationContext), returnType);
 }
Example #25
0
        private void CheckInitialized()
        {
            if (m_initialized)
            {
                return;
            }

            lock (m_thisLock)
            {
                if (m_initialized)
                {
                    return;
                }

                if (0 == StringComparer.OrdinalIgnoreCase.Compare("demo", m_settings.InitializationCommand))
                {
                    m_descriptor = BuildDemoContainerDescriptor();
                }
                else if (m_settings.Descriptor != null)
                {
                    m_descriptor = m_settings.Descriptor;
                }
                else
                {
                    m_descriptor = GetDescriptorFromStore();
                }

                if (m_descriptor != null)
                {
                    m_dataContainer = new DataContainer(m_tracer, m_descriptor, m_settings.StorageRoot);
                    m_initialized = true;
                }
            }
        }
Example #26
0
 public SchemaBuilder()
 {
     m_descriptor = new DataContainerDescriptor();
 }
Example #27
-1
        public DataEngine(ITracer tracer, string instanceName, int maxConcurrency, IStorageDriver storageDriver, DataContainerDescriptor containerDescriptor)
        {
            if (tracer == null)
            {
                throw new ArgumentNullException("tracer");
            }

            if (maxConcurrency <= 0 || maxConcurrency > 10000)
            {
                throw new ArgumentOutOfRangeException("maxConcurrency");
            }

            if (storageDriver == null)
            {
                throw new ArgumentNullException("storageDriver");
            }

            if (containerDescriptor == null)
            {
                throw new ArgumentNullException("containerDescriptor");
            }

            m_tracer = tracer;
            m_containerDescriptor = containerDescriptor;
            m_maxConcurrency = maxConcurrency;
            m_parsedRequestCache = new ParsedRequestCache(instanceName);
            m_storageDriver = storageDriver;
            m_parser = new QueryParser(containerDescriptor, maxConcurrency);
            m_activeProcessors = new ConcurrentDictionary<RequestExecutionContext, Task>(m_maxConcurrency, m_maxConcurrency);
            m_utcLastUsedAt = DateTime.UtcNow;
        }
Example #28
-1
        public DataContainer(ITracer tracer, DataContainerDescriptor dataContainerDescriptor, string storageRoot)
        {
            if (tracer == null)
            {
                throw new ArgumentNullException("tracer");
            }

            if (dataContainerDescriptor == null)
            {
                throw new ArgumentNullException("dataContainerDescriptor");
            }

            // intentionally allowed to be null - in case if we don't want to use any persistence
            m_storageRoot = storageRoot;
            m_tracer = tracer;
            m_memoryPool = new DynamicMemoryPool();

            m_dataContainerDescriptor = dataContainerDescriptor;
            m_documentDataContainers = new Dictionary<int, DocumentDataContainer>(50);
            m_documentDataContainerLocks = new Dictionary<int, object>(50);

            foreach (var item in dataContainerDescriptor.EnumerateDocumentTypes())
            {
                m_documentDataContainerLocks.Add(item.DocumentType, new object());
            }
        }