Many updates to grammar including support for arrays, ilike, natural joins with no predicates... (#49)

* Got ISNULL working

* Allow function calls with no arguments

* Added no-else cases and arrays

* Added more operations including ilike, concat

* Added optional all to set operations and fixed natural join

* Ran astyle

* Used the appropriate star expression

Instead of using a ColumnRef with star literal.

* NULL expressions now returns true isLiteral

* Fixed structure for no-else case clauses

* Fixed up grammar conflicts

.gitignore

@@ -41,4 +41,7 @@ cmake-build-debug/
 *.cpp.orig
 *.h.orig
 
+# Vim Swap Files
+*.swp
+
 *.csv

src/parser/bison_parser.h

@@ -48,7 +48,7 @@
 extern int hsql_debug;
 #endif
 /* "%code requires" blocks.  */
-#line 34 "bison_parser.y" /* yacc.c:1915  */
+#line 35 "bison_parser.y" /* yacc.c:1909  */
 
 // %code requires block
 
@@ -71,7 +71,7 @@ extern int hsql_debug;
 				} \
 		}
 
-#line 75 "bison_parser.h" /* yacc.c:1915  */
+#line 75 "bison_parser.h" /* yacc.c:1909  */
 
 /* Token type.  */
 #ifndef HSQL_TOKENTYPE
@@ -198,14 +198,17 @@ extern int hsql_debug;
     SQL_ON = 375,
     SQL_OR = 376,
     SQL_TO = 377,
-    SQL_EQUALS = 378,
+    SQL_ARRAY = 378,
-    SQL_NOTEQUALS = 379,
+    SQL_CONCAT = 379,
-    SQL_LESS = 380,
+    SQL_ILIKE = 380,
-    SQL_GREATER = 381,
+    SQL_EQUALS = 381,
-    SQL_LESSEQ = 382,
+    SQL_NOTEQUALS = 382,
-    SQL_GREATEREQ = 383,
+    SQL_LESS = 383,
-    SQL_NOTNULL = 384,
+    SQL_GREATER = 384,
-    SQL_UMINUS = 385
+    SQL_LESSEQ = 385,
+    SQL_GREATEREQ = 386,
+    SQL_NOTNULL = 387,
+    SQL_UMINUS = 388
   };
 #endif
 
@@ -214,7 +217,7 @@ extern int hsql_debug;
 
 union HSQL_STYPE
 {
-#line 92 "bison_parser.y" /* yacc.c:1915  */
+#line 93 "bison_parser.y" /* yacc.c:1909  */
 
 	double fval;
 	int64_t ival;
@@ -251,7 +254,7 @@ union HSQL_STYPE
 	std::vector<hsql::Expr*>* expr_vec;
 	std::vector<hsql::OrderDescription*>* order_vec;
 
-#line 255 "bison_parser.h" /* yacc.c:1915  */
+#line 258 "bison_parser.h" /* yacc.c:1909  */
 };
 
 typedef union HSQL_STYPE HSQL_STYPE;

src/parser/bison_parser.y

@@ -15,6 +15,7 @@
 #include "flex_lexer.h"
 
 #include <stdio.h>
+#include <string.h>
 
 using namespace hsql;
 
@@ -165,6 +166,7 @@ int yyerror(YYLTYPE* llocp, SQLParserResult* result, yyscan_t scanner, const cha
 %token LEFT LIKE LOAD NULL PART PLAN SHOW TEXT THEN TIME
 %token VIEW WHEN WITH ADD ALL AND ASC CSV END FOR INT KEY
 %token NOT OFF SET TBL TOP AS BY IF IN IS OF ON OR TO
+%token ARRAY CONCAT ILIKE
 
 /*********************************
  ** Non-Terminal types (http://www.gnu.org/software/bison/manual/html_node/Type-Decl.html)
@@ -173,7 +175,7 @@ int yyerror(YYLTYPE* llocp, SQLParserResult* result, yyscan_t scanner, const cha
 %type <statement> 	statement preparable_statement
 %type <exec_stmt>	execute_statement
 %type <prep_stmt>	prepare_statement
-%type <select_stmt> select_statement select_with_paren select_no_paren select_clause
+%type <select_stmt> select_statement select_with_paren select_no_paren select_clause select_paren_or_clause
 %type <import_stmt> import_statement
 %type <create_stmt> create_statement
 %type <insert_stmt> insert_statement
@@ -183,12 +185,13 @@ int yyerror(YYLTYPE* llocp, SQLParserResult* result, yyscan_t scanner, const cha
 %type <sval> 		table_name opt_alias alias file_path prepare_target_query
 %type <bval> 		opt_not_exists opt_distinct
 %type <uval>		import_file_type opt_join_type column_type
-%type <table> 		from_clause table_ref table_ref_atomic table_ref_name
+%type <table> 		from_clause table_ref table_ref_atomic table_ref_name nonjoin_table_ref_atomic
 %type <table>		join_clause table_ref_name_no_alias
 %type <expr> 		expr operand scalar_expr unary_expr binary_expr logic_expr exists_expr
-%type <expr>		function_expr between_expr star_expr expr_alias param_expr
+%type <expr>		function_expr between_expr expr_alias param_expr
 %type <expr> 		column_name literal int_literal num_literal string_literal
 %type <expr> 		comp_expr opt_where join_condition opt_having case_expr in_expr hint
+%type <expr> 		array_expr array_index null_literal
 %type <limit>		opt_limit opt_top
 %type <order>		order_desc
 %type <order_type>	opt_order_type
@@ -210,7 +213,7 @@ int yyerror(YYLTYPE* llocp, SQLParserResult* result, yyscan_t scanner, const cha
 %left		OR
 %left		AND
 %right		NOT
-%right		'=' EQUALS NOTEQUALS LIKE
+%nonassoc	'=' EQUALS NOTEQUALS LIKE ILIKE
 %nonassoc	'<' '>' LESS GREATER LESSEQ GREATEREQ
 
 %nonassoc	NOTNULL
@@ -219,6 +222,7 @@ int yyerror(YYLTYPE* llocp, SQLParserResult* result, yyscan_t scanner, const cha
 %left		'+' '-'
 %left		'*' '/' '%'
 %left		'^'
+%left		CONCAT
 
 /* Unary Operators */
 %right  UMINUS
@@ -510,25 +514,7 @@ update_clause:
 select_statement:
 		select_with_paren
 	|	select_no_paren
-	;
+	|	select_with_paren set_operator select_paren_or_clause opt_order opt_limit {
-
-select_with_paren:
-		'(' select_no_paren ')' { $$ = $2; }
-	|	'(' select_with_paren ')' { $$ = $2; }
-	;
-
-select_no_paren:
-		select_clause opt_order opt_limit {
-			$$ = $1;
-			$$->order = $2;
-
-			// Limit could have been set by TOP.
-			if ($3 != nullptr) {
-				delete $$->limit;
-				$$->limit = $3;
-			}
-		}
-	|	select_clause set_operator select_clause opt_order opt_limit {
 			// TODO: allow multiple unions (through linked list)
 			// TODO: capture type of set_operator
 			// TODO: might overwrite order and limit of first select here
@@ -542,7 +528,33 @@ select_no_paren:
 				$$->limit = $5;
 			}
 		}
-	|	select_clause set_operator select_with_paren opt_order opt_limit {
+	;
+
+select_with_paren:
+		'(' select_no_paren ')' { $$ = $2; }
+	|	'(' select_with_paren ')' { $$ = $2; }
+	;
+
+select_paren_or_clause:
+		select_with_paren
+	|	select_clause
+	;
+
+select_no_paren:
+		select_clause opt_order opt_limit {
+			$$ = $1;
+			$$->order = $2;
+
+			// Limit could have been set by TOP.
+			if ($3 != nullptr) {
+				delete $$->limit;
+				$$->limit = $3;
+			}
+		}
+	|	select_clause set_operator select_paren_or_clause opt_order opt_limit {
+			// TODO: allow multiple unions (through linked list)
+			// TODO: capture type of set_operator
+			// TODO: might overwrite order and limit of first select here
 			$$ = $1;
 			$$->unionSelect = $3;
 			$$->order = $4;
@@ -556,11 +568,20 @@ select_no_paren:
 	;
 
 set_operator:
+		set_type opt_all
+	;
+
+set_type:
 		UNION
 	|	INTERSECT
 	|	EXCEPT
 	;
 
+opt_all:
+		ALL
+	|	/* empty */
+	;
+
 select_clause:
 		SELECT opt_top opt_distinct select_list from_clause opt_where opt_group {
 			$$ = new SelectStatement();
@@ -663,28 +684,32 @@ expr:
 	|	between_expr
 	|	logic_expr
 	|	exists_expr
-	|	case_expr
 	|	in_expr
 	;
 
 operand:
 		'(' expr ')' { $$ = $2; }
+	|	array_index
 	|	scalar_expr
 	|	unary_expr
 	|	binary_expr
+	|	case_expr
 	|	function_expr
+	|	array_expr
 	|	'(' select_no_paren ')' { $$ = Expr::makeSelect($2); }
 	;
 
 scalar_expr:
 		column_name
-	|	star_expr
 	|	literal
 	;
 
 unary_expr:
 		'-' operand { $$ = Expr::makeOpUnary(kOpUnaryMinus, $2); }
 	|	NOT operand { $$ = Expr::makeOpUnary(kOpNot, $2); }
+	|	operand ISNULL { $$ = Expr::makeOpUnary(kOpIsNull, $1); }
+	|	operand IS NULL { $$ = Expr::makeOpUnary(kOpIsNull, $1); }
+	|	operand IS NOT NULL { $$ = Expr::makeOpUnary(kOpNot, Expr::makeOpUnary(kOpIsNull, $1)); }
 	;
 
 binary_expr:
@@ -697,6 +722,8 @@ binary_expr:
 	|	operand '^' operand			{ $$ = Expr::makeOpBinary($1, kOpCaret, $3); }
 	|	operand LIKE operand		{ $$ = Expr::makeOpBinary($1, kOpLike, $3); }
 	|	operand NOT LIKE operand	{ $$ = Expr::makeOpBinary($1, kOpNotLike, $4); }
+	|	operand ILIKE operand		{ $$ = Expr::makeOpBinary($1, kOpILike, $3); }
+	|	operand CONCAT operand	{ $$ = Expr::makeOpBinary($1, kOpConcat, $3); }
 	;
 
 logic_expr:
@@ -713,6 +740,8 @@ in_expr:
 
 // TODO: allow no else specified
 case_expr:
+		CASE WHEN expr THEN operand END { $$ = Expr::makeCase($3, $5); }
+	|
 		CASE WHEN expr THEN operand ELSE operand END { $$ = Expr::makeCase($3, $5, $7); }
 	;
 
@@ -731,7 +760,16 @@ comp_expr:
 	;
 
 function_expr:
-		IDENTIFIER '(' opt_distinct expr_list ')' { $$ = Expr::makeFunctionRef($1, $4, $3); }
+		IDENTIFIER '(' ')' { $$ = Expr::makeFunctionRef($1, new std::vector<Expr*>(), false); }
+	|	IDENTIFIER '(' opt_distinct expr_list ')' { $$ = Expr::makeFunctionRef($1, $4, $3); }
+	;
+
+array_expr:
+	  	ARRAY '[' expr_list ']' { $$ = Expr::makeArray($3); }
+	;
+
+array_index:
+	   	operand '[' int_literal ']' { $$ = Expr::makeArrayIndex($1, $3->ival); }
 	;
 
 between_expr:
@@ -741,11 +779,14 @@ between_expr:
 column_name:
 		IDENTIFIER { $$ = Expr::makeColumnRef($1); }
 	|	IDENTIFIER '.' IDENTIFIER { $$ = Expr::makeColumnRef($1, $3); }
+	|	'*' { $$ = Expr::makeStar(); }
+	|	IDENTIFIER '.' '*' { $$ = Expr::makeStar($1); }
 	;
 
 literal:
 		string_literal
 	|	num_literal
+	|	null_literal
 	|	param_expr
 	;
 
@@ -763,8 +804,8 @@ int_literal:
 		INTVAL { $$ = Expr::makeLiteral($1); }
 	;
 
-star_expr:
+null_literal:
-		'*' { $$ = Expr::make(kExprStar); }
+	    	NULL { $$ = Expr::makeNullLiteral(); }
 	;
 
 param_expr:
@@ -791,6 +832,11 @@ table_ref:
 
 
 table_ref_atomic:
+		nonjoin_table_ref_atomic
+	|	join_clause
+	;
+
+nonjoin_table_ref_atomic: 
 		table_ref_name
 	|	'(' select_statement ')' opt_alias {
 			auto tbl = new TableRef(kTableSelect);
@@ -798,10 +844,8 @@ table_ref_atomic:
 			tbl->alias = $4;
 			$$ = tbl;
 		}
-	|	join_clause
 	;
 
-
 table_ref_commalist:
 		table_ref_atomic { $$ = new std::vector<TableRef*>(); $$->push_back($1); }
 	|	table_ref_commalist ',' table_ref_atomic { $1->push_back($3); $$ = $1; }
@@ -847,7 +891,15 @@ opt_alias:
  ******************************/
 
 join_clause:
-		table_ref_atomic opt_join_type JOIN table_ref_atomic ON join_condition
+		table_ref_atomic NATURAL JOIN nonjoin_table_ref_atomic
+		{
+			$$ = new TableRef(kTableJoin);
+			$$->join = new JoinDefinition();
+			$$->join->type = kJoinNatural;
+			$$->join->left = $1;
+			$$->join->right = $4;
+		}
+	|	table_ref_atomic opt_join_type JOIN table_ref_atomic ON join_condition
 		{ 
 			$$ = new TableRef(kTableJoin);
 			$$->join = new JoinDefinition();
@@ -856,6 +908,23 @@ join_clause:
 			$$->join->right = $4;
 			$$->join->condition = $6;
 		}
+	|
+		table_ref_atomic opt_join_type JOIN table_ref_atomic USING '(' column_name ')'
+		{ 
+			$$ = new TableRef(kTableJoin);
+			$$->join = new JoinDefinition();
+			$$->join->type = (JoinType) $2;
+			$$->join->left = $1;
+			$$->join->right = $4;
+			auto left_col = Expr::makeColumnRef(strdup($7->name));
+			if ($7->alias != nullptr) left_col->alias = strdup($7->alias);
+			if ($1->getName() != nullptr) left_col->table = strdup($1->getName());
+			auto right_col = Expr::makeColumnRef(strdup($7->name));
+			if ($7->alias != nullptr) right_col->alias = strdup($7->alias);
+			if ($4->getName() != nullptr) right_col->table = strdup($4->getName());
+			$$->join->condition = Expr::makeOpBinary(left_col, kOpEquals, right_col);
+			delete $7;
+		}
 	;
 
 opt_join_type:
@@ -866,7 +935,6 @@ opt_join_type:
 	|	LEFT		{ $$ = kJoinLeft; }
 	|	RIGHT		{ $$ = kJoinRight; }
 	|	CROSS		{ $$ = kJoinCross; }
-	|	NATURAL		{ $$ = kJoinNatural; }
 	|	/* empty, default */	{ $$ = kJoinInner; }
 	;
 

src/parser/flex_lexer.h

@@ -729,7 +729,7 @@ extern int yylex \
 #undef yyTABLES_NAME
 #endif
 
-#line 213 "flex_lexer.l"
+#line 217 "flex_lexer.l"
 
 
 #line 735 "flex_lexer.h"

src/parser/flex_lexer.l

@@ -107,6 +107,7 @@ UPDATE		TOKEN(UPDATE)
 VALUES		TOKEN(VALUES)
 AFTER		TOKEN(AFTER)
 ALTER		TOKEN(ALTER)
+ARRAY		TOKEN(ARRAY)
 CROSS		TOKEN(CROSS)
 DELTA		TOKEN(DELTA)
 GROUP		TOKEN(GROUP)
@@ -138,6 +139,7 @@ INTO		TOKEN(INTO)
 JOIN		TOKEN(JOIN)
 LEFT		TOKEN(LEFT)
 LIKE		TOKEN(LIKE)
+ILIKE		TOKEN(ILIKE)
 LOAD		TOKEN(LOAD)
 NULL		TOKEN(NULL)
 PART		TOKEN(PART)
@@ -173,11 +175,13 @@ ON			TOKEN(ON)
 OR			TOKEN(OR)
 TO			TOKEN(TO)
 
+"!="		TOKEN(NOTEQUALS)
 "<>" 		TOKEN(NOTEQUALS)
 "<="		TOKEN(LESSEQ)
 ">="		TOKEN(GREATEREQ)
+"||"		TOKEN(CONCAT)
 
-[-+*/(){},.;<>=^%:?]	{ return yytext[0]; }
+[-+*/(){},.;<>=^%:?[\]|]	{ return yytext[0]; }
 
 -?[0-9]+"."[0-9]* |
 "."[0-9]*	{

src/parser/sql_keywords.txt

@@ -127,6 +127,7 @@ DEFAULT
 CALL
 FOR
 TO
+ARRAY
 
 
 // Expressions

src/sql/Expr.cpp

@@ -6,8 +6,8 @@
 
 namespace hsql {
 
-  Expr::Expr(ExprType type) :
+  Expr::Expr(ExprType type)
-    type(type),
+    : type(type),
       expr(nullptr),
       expr2(nullptr),
       exprList(nullptr),
@@ -63,6 +63,15 @@ namespace hsql {
     return e;
   }
 
+  Expr* Expr::makeCase(Expr* expr, Expr* then) {
+    Expr* e = new Expr(kExprOperator);
+    e->expr = expr;
+    e->opType = kOpCase;
+    e->exprList = new std::vector<Expr*>();
+    e->exprList->push_back(then);
+    return e;
+  }
+
   Expr* Expr::makeCase(Expr* expr, Expr* then, Expr* other) {
     Expr* e = new Expr(kExprOperator);
     e->expr = expr;
@@ -91,6 +100,10 @@ namespace hsql {
     return e;
   }
 
+  Expr* Expr::makeNullLiteral() {
+    Expr* e = new Expr(kExprLiteralNull);
+    return e;
+  }
 
   Expr* Expr::makeColumnRef(char* name) {
     Expr* e = new Expr(kExprColumnRef);
@@ -105,7 +118,19 @@ namespace hsql {
     return e;
   }
 
-  Expr* Expr::makeFunctionRef(char* func_name, std::vector<Expr*>* exprList, bool distinct) {
+  Expr* Expr::makeStar(void) {
+    Expr* e = new Expr(kExprStar);
+    return e;
+  }
+
+  Expr* Expr::makeStar(char* table) {
+    Expr* e = new Expr(kExprStar);
+    e->table = table;
+    return e;
+  }
+
+  Expr* Expr::makeFunctionRef(char* func_name, std::vector<Expr*>* exprList,
+                              bool distinct) {
     Expr* e = new Expr(kExprFunctionRef);
     e->name = func_name;
     e->exprList = exprList;
@@ -113,6 +138,19 @@ namespace hsql {
     return e;
   }
 
+  Expr* Expr::makeArray(std::vector<Expr*>* exprList) {
+    Expr* e = new Expr(kExprArray);
+    e->exprList = exprList;
+    return e;
+  }
+
+  Expr* Expr::makeArrayIndex(Expr* expr, int64_t index) {
+    Expr* e = new Expr(kExprArrayIndex);
+    e->expr = expr;
+    e->ival = index;
+    return e;
+  }
+
   Expr* Expr::makeParameter(int id) {
     Expr* e = new Expr(kExprParameter);
     e->ival = id;
@@ -155,7 +193,9 @@ namespace hsql {
   }
 
   bool Expr::isLiteral() const {
-    return isType(kExprLiteralInt) || isType(kExprLiteralFloat) || isType(kExprLiteralString) || isType(kExprParameter);
+    return isType(kExprLiteralInt) || isType(kExprLiteralFloat) ||
+           isType(kExprLiteralString) || isType(kExprParameter) ||
+           isType(kExprLiteralNull);
   }
 
   bool Expr::hasAlias() const {
@@ -167,13 +207,16 @@ namespace hsql {
   }
 
   const char* Expr::getName() const {
-    if (alias != nullptr) return alias;
+    if (alias != nullptr)
-    else return name;
+      return alias;
+    else
+      return name;
   }
 
   char* substr(const char* source, int from, int to) {
     int len = to - from;
-    char* copy = (char*) malloc(len + 1);;
+    char* copy = (char*)malloc(len + 1);
+    ;
     strncpy(copy, source + from, len);
     copy[len] = '\0';
     return copy;

src/sql/Expr.h

@@ -8,27 +8,30 @@
 namespace hsql {
   struct SelectStatement;
 
-  // Helper function used by the lexer.
+// Helper function used by the lexer.
-  // TODO: move to more appropriate place.
+// TODO: move to more appropriate place.
   char* substr(const char* source, int from, int to);
 
   enum ExprType {
     kExprLiteralFloat,
     kExprLiteralString,
     kExprLiteralInt,
+    kExprLiteralNull,
     kExprStar,
     kExprParameter,
     kExprColumnRef,
     kExprFunctionRef,
     kExprOperator,
     kExprSelect,
-    kExprHint
+    kExprHint,
+    kExprArray,
+    kExprArrayIndex
   };
 
-  // Operator types. These are important for expressions of type kExprOperator.
+// Operator types. These are important for expressions of type kExprOperator.
-  // Trivial types are those that can be described by a single character e.g:
+// Trivial types are those that can be described by a single character e.g:
-  // + - * / < > = %
+// + - * / < > = %
-  // Non-trivial are: <> <= >= LIKE ISNULL NOT
+// Non-trivial are: <> <= >= LIKE ISNULL NOT
   enum OperatorType {
     kOpNone,
 
@@ -52,9 +55,11 @@ namespace hsql {
     kOpGreaterEq,
     kOpLike,
     kOpNotLike,
+    kOpILike,
     kOpAnd,
     kOpOr,
     kOpIn,
+    kOpConcat,
 
     // Unary operators.
     kOpNot,
@@ -65,10 +70,10 @@ namespace hsql {
 
   typedef struct Expr Expr;
 
-  // Represents SQL expressions (i.e. literals, operators, column_refs).
+// Represents SQL expressions (i.e. literals, operators, column_refs).
-  // TODO: When destructing a placeholder expression, we might need to alter the placeholder_list.
+// TODO: When destructing a placeholder expression, we might need to alter the
+// placeholder_list.
   struct Expr {
-
     Expr(ExprType type);
     virtual ~Expr();
 
@@ -89,7 +94,6 @@ namespace hsql {
     OperatorType opType;
     bool distinct;
 
-
     // Convenience accessor methods.
 
     bool isType(ExprType exprType) const;
@@ -112,6 +116,8 @@ namespace hsql {
 
     static Expr* makeBetween(Expr* expr, Expr* left, Expr* right);
 
+    static Expr* makeCase(Expr* expr, Expr* then);
+
     static Expr* makeCase(Expr* expr, Expr* then, Expr* other);
 
     static Expr* makeLiteral(int64_t val);
@@ -120,11 +126,22 @@ namespace hsql {
 
     static Expr* makeLiteral(char* val);
 
+    static Expr* makeNullLiteral();
+
     static Expr* makeColumnRef(char* name);
 
     static Expr* makeColumnRef(char* table, char* name);
 
-    static Expr* makeFunctionRef(char* func_name, std::vector<Expr*>* exprList, bool distinct);
+    static Expr* makeStar(void);
+
+    static Expr* makeStar(char* table);
+
+    static Expr* makeFunctionRef(char* func_name, std::vector<Expr*>* exprList,
+                                 bool distinct);
+
+    static Expr* makeArray(std::vector<Expr*>* exprList);
+
+    static Expr* makeArrayIndex(Expr* expr, int64_t index);
 
     static Expr* makeParameter(int id);
 
@@ -147,10 +164,9 @@ namespace hsql {
         Expr zero = {type};               \
         var = (Expr*)malloc(sizeof *var); \
         *var = zero;                      \
-	} while(0);
+    } while (0);
 #undef ALLOC_EXPR
 
-
 }  // namespace hsql
 
 #endif